From 3abecf2ee9fc724af571f8c7da4302f7bee9eadb Mon Sep 17 00:00:00 2001 From: Frederik Gladhorn Date: Sat, 27 Jul 2013 00:11:20 +0200 Subject: Accessibility for Android This enables both modes for TalkBack, explore-by-touch and the normal swiping mode. It is partially inspired by the BarGraphView example of the Google/Android Eyes-Free project. Note that for any accessibility to work you'll need a device with api level 16 at least. Using reflection we should be able to dynamically pick up the classes if we have the high enough api level. Change-Id: I11b93bead451483782a1711434d45c8f9a35996f Reviewed-by: Eskil Abrahamsen Blomfeldt --- src/android/accessibility/accessibility.pro | 2 + src/android/accessibility/jar/AndroidManifest.xml | 7 + src/android/accessibility/jar/bundledjar.pro | 3 + src/android/accessibility/jar/distributedjar.pro | 2 + src/android/accessibility/jar/jar.pri | 14 + src/android/accessibility/jar/jar.pro | 2 + .../accessibility/QtAccessibilityDelegate.java | 343 +++++++++++++++++++++ .../accessibility/QtNativeAccessibility.java | 58 ++++ src/android/android.pro | 2 +- .../src/org/qtproject/qt5/android/QtSurface.java | 46 ++- 10 files changed, 477 insertions(+), 2 deletions(-) create mode 100644 src/android/accessibility/accessibility.pro create mode 100644 src/android/accessibility/jar/AndroidManifest.xml create mode 100644 src/android/accessibility/jar/bundledjar.pro create mode 100644 src/android/accessibility/jar/distributedjar.pro create mode 100644 src/android/accessibility/jar/jar.pri create mode 100644 src/android/accessibility/jar/jar.pro create mode 100644 src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtAccessibilityDelegate.java create mode 100644 src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtNativeAccessibility.java (limited to 'src/android') diff --git a/src/android/accessibility/accessibility.pro b/src/android/accessibility/accessibility.pro new file mode 100644 index 0000000000..df5846945d --- /dev/null +++ b/src/android/accessibility/accessibility.pro @@ -0,0 +1,2 @@ +TEMPLATE = subdirs +SUBDIRS = jar diff --git a/src/android/accessibility/jar/AndroidManifest.xml b/src/android/accessibility/jar/AndroidManifest.xml new file mode 100644 index 0000000000..dc8343a55a --- /dev/null +++ b/src/android/accessibility/jar/AndroidManifest.xml @@ -0,0 +1,7 @@ + + + + diff --git a/src/android/accessibility/jar/bundledjar.pro b/src/android/accessibility/jar/bundledjar.pro new file mode 100644 index 0000000000..85ba810310 --- /dev/null +++ b/src/android/accessibility/jar/bundledjar.pro @@ -0,0 +1,3 @@ +TARGET = QtAndroidAccessibility-bundled +CONFIG += bundled_jar_file +include(jar.pri) diff --git a/src/android/accessibility/jar/distributedjar.pro b/src/android/accessibility/jar/distributedjar.pro new file mode 100644 index 0000000000..d161cf0cf6 --- /dev/null +++ b/src/android/accessibility/jar/distributedjar.pro @@ -0,0 +1,2 @@ +TARGET = QtAndroidAccessibility +include(jar.pri) diff --git a/src/android/accessibility/jar/jar.pri b/src/android/accessibility/jar/jar.pri new file mode 100644 index 0000000000..23b3ff7087 --- /dev/null +++ b/src/android/accessibility/jar/jar.pri @@ -0,0 +1,14 @@ +CONFIG += java +DESTDIR = $$[QT_INSTALL_PREFIX/get]/jar +API_VERSION = android-16 + +PATHPREFIX = $$PWD/src/org/qtproject/qt5/android/accessibility + +JAVACLASSPATH += $$PWD/src/ +JAVASOURCES += \ + $$PATHPREFIX/QtAccessibilityDelegate.java \ + $$PATHPREFIX/QtNativeAccessibility.java + +# install +target.path = $$[QT_INSTALL_PREFIX]/jar +INSTALLS += target \ No newline at end of file diff --git a/src/android/accessibility/jar/jar.pro b/src/android/accessibility/jar/jar.pro new file mode 100644 index 0000000000..8d19c1b7d6 --- /dev/null +++ b/src/android/accessibility/jar/jar.pro @@ -0,0 +1,2 @@ +TEMPLATE = subdirs +SUBDIRS += bundledjar.pro distributedjar.pro diff --git a/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtAccessibilityDelegate.java b/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtAccessibilityDelegate.java new file mode 100644 index 0000000000..669fbaab0d --- /dev/null +++ b/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtAccessibilityDelegate.java @@ -0,0 +1,343 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Android port of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + + +package org.qtproject.qt5.android.accessibility; + +import android.accessibilityservice.AccessibilityService; +import android.graphics.Rect; +import android.os.Bundle; +import android.util.Log; +import android.view.View; +import android.view.ViewGroup; +import android.view.ViewParent; +import android.text.TextUtils; + +import android.view.accessibility.*; +import android.view.MotionEvent; + +import android.content.Context; + +import java.util.LinkedList; +import java.util.List; + +public class QtAccessibilityDelegate extends View.AccessibilityDelegate +{ + private static final String TAG = "Qt A11Y"; + + // Qt uses the upper half of the unsiged integers + // all low positive ints should be fine. + public static final int INVALID_ID = 333; // half evil + + // The platform might ask for the class implementing the "view". + // Pretend to be an inner class of the QtSurface. + private static final String DEFAULT_CLASS_NAME = "$VirtualChild"; + + private final View m_view; + private final AccessibilityManager m_manager; + + // The accessible object that currently has the "accessibility focus" + // usually indicated by a yellow rectangle on screen. + private int m_focusedVirtualViewId = INVALID_ID; + // When exploring the screen by touch, the item "hovered" by the finger. + private int m_hoveredVirtualViewId = INVALID_ID; + + // Cache coordinates of the view to know the global offset + // this is because the Android platform window does not take + // the offset of the view on screen into account (eg status bar on top) + private final int[] m_globalOffset = new int[2]; + + public QtAccessibilityDelegate(View host) + { + m_view = host; + m_manager = (AccessibilityManager) host.getContext() + .getSystemService(Context.ACCESSIBILITY_SERVICE); + + // Enable Qt Accessibility so that notifications are enabled + QtNativeAccessibility.setActive(true); + } + + @Override + public AccessibilityNodeProvider getAccessibilityNodeProvider(View host) + { + return m_nodeProvider; + } + + // For "explore by touch" we need all movement events here first + // (user moves finger over screen to discover items on screen). + public boolean dispatchHoverEvent(MotionEvent event) + { + if (!m_manager.isTouchExplorationEnabled()) { + return false; + } + + int virtualViewId = QtNativeAccessibility.hitTest(event.getX(), event.getY()); + if (virtualViewId == INVALID_ID) { + virtualViewId = View.NO_ID; + } + + switch (event.getAction()) { + case MotionEvent.ACTION_HOVER_ENTER: + case MotionEvent.ACTION_HOVER_MOVE: + setHoveredVirtualViewId(virtualViewId); + break; + case MotionEvent.ACTION_HOVER_EXIT: + setHoveredVirtualViewId(virtualViewId); + break; + } + + return true; + } + + public boolean sendEventForVirtualViewId(int virtualViewId, int eventType) + { + if ((virtualViewId == INVALID_ID) || !m_manager.isEnabled()) { + Log.w(TAG, "sendEventForVirtualViewId for invalid view"); + return false; + } + + final ViewGroup group = (ViewGroup) m_view.getParent(); + if (group == null) { + Log.w(TAG, "Could not send AccessibilityEvent because group was null. This should really not happen."); + return false; + } + + final AccessibilityEvent event; + event = getEventForVirtualViewId(virtualViewId, eventType); + return group.requestSendAccessibilityEvent(m_view, event); + } + + public void invalidateVirtualViewId(int virtualViewId) + { + sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_WINDOW_CONTENT_CHANGED); + } + + private void setHoveredVirtualViewId(int virtualViewId) + { + if (m_hoveredVirtualViewId == virtualViewId) { + return; + } + + final int previousVirtualViewId = m_hoveredVirtualViewId; + m_hoveredVirtualViewId = virtualViewId; + sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_HOVER_ENTER); + sendEventForVirtualViewId(previousVirtualViewId, AccessibilityEvent.TYPE_VIEW_HOVER_EXIT); + } + + private AccessibilityEvent getEventForVirtualViewId(int virtualViewId, int eventType) + { + final AccessibilityEvent event = AccessibilityEvent.obtain(eventType); + + event.setEnabled(true); + event.setClassName(m_view.getClass().getName() + DEFAULT_CLASS_NAME); + + event.setContentDescription(QtNativeAccessibility.descriptionForAccessibleObject(virtualViewId)); + if (event.getText().isEmpty() && TextUtils.isEmpty(event.getContentDescription())) + Log.w(TAG, "AccessibilityEvent with empty description"); + + event.setPackageName(m_view.getContext().getPackageName()); + event.setSource(m_view, virtualViewId); + return event; + } + + private void dumpNodes(int parentId) + { + Log.i(TAG, "A11Y hierarchy: " + parentId + " parent: " + QtNativeAccessibility.parentId(parentId)); + Log.i(TAG, " desc: " + QtNativeAccessibility.descriptionForAccessibleObject(parentId) + " rect: " + QtNativeAccessibility.screenRect(parentId)); + Log.i(TAG, " NODE: " + getNodeForVirtualViewId(parentId)); + int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(parentId); + for (int i = 0; i < ids.length; ++i) { + Log.i(TAG, parentId + " has child: " + ids[i]); + dumpNodes(ids[i]); + } + } + + private AccessibilityNodeInfo getNodeForView() + { + // Since we don't want the parent to be focusable, but we can't remove + // actions from a node, copy over the necessary fields. + final AccessibilityNodeInfo result = AccessibilityNodeInfo.obtain(m_view); + final AccessibilityNodeInfo source = AccessibilityNodeInfo.obtain(m_view); + m_view.onInitializeAccessibilityNodeInfo(source); + + // Get the actual position on screen, taking the status bar into account. + m_view.getLocationOnScreen(m_globalOffset); + final int offsetX = m_globalOffset[0]; + final int offsetY = m_globalOffset[1]; + + // Copy over parent and screen bounds. + final Rect m_tempParentRect = new Rect(); + source.getBoundsInParent(m_tempParentRect); + result.setBoundsInParent(m_tempParentRect); + + final Rect m_tempScreenRect = new Rect(); + source.getBoundsInScreen(m_tempScreenRect); + m_tempScreenRect.offset(offsetX, offsetY); + result.setBoundsInScreen(m_tempScreenRect); + + // Set up the parent view, if applicable. + final ViewParent parent = m_view.getParent(); + if (parent instanceof View) { + result.setParent((View) parent); + } + + result.setVisibleToUser(source.isVisibleToUser()); + result.setPackageName(source.getPackageName()); + result.setClassName(source.getClassName()); + +// Spit out the entire hierarchy for debugging purposes +// dumpNodes(-1); + + int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(-1); + for (int i = 0; i < ids.length; ++i) + result.addChild(m_view, ids[i]); + + return result; + } + + private AccessibilityNodeInfo getNodeForVirtualViewId(int virtualViewId) + { + final AccessibilityNodeInfo node = AccessibilityNodeInfo.obtain(); + + node.setClassName(m_view.getClass().getName() + DEFAULT_CLASS_NAME); + node.setPackageName(m_view.getContext().getPackageName()); + + node.setSource(m_view, virtualViewId); + QtNativeAccessibility.populateNode(virtualViewId, node); + if (TextUtils.isEmpty(node.getText()) && TextUtils.isEmpty(node.getContentDescription())) + Log.w(TAG, "AccessibilityNodeInfo with empty contentDescription: " + virtualViewId); + + int parentId = QtNativeAccessibility.parentId(virtualViewId); + node.setParent(m_view, parentId); + + Rect screenRect = QtNativeAccessibility.screenRect(virtualViewId); + final int offsetX = m_globalOffset[0]; + final int offsetY = m_globalOffset[1]; + screenRect.offset(offsetX, offsetY); + node.setBoundsInScreen(screenRect); + + Rect rectInParent = screenRect; + Rect parentScreenRect = QtNativeAccessibility.screenRect(parentId); + rectInParent.offset(-parentScreenRect.left, -parentScreenRect.top); + node.setBoundsInParent(rectInParent); + + // Manage internal accessibility focus state. + if (m_focusedVirtualViewId == virtualViewId) { + node.setAccessibilityFocused(true); + node.addAction(AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS); + } else { + node.setAccessibilityFocused(false); + node.addAction(AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS); + } + + int[] ids = QtNativeAccessibility.childIdListForAccessibleObject(virtualViewId); + for (int i = 0; i < ids.length; ++i) + node.addChild(m_view, ids[i]); + return node; + } + + private AccessibilityNodeProvider m_nodeProvider = new AccessibilityNodeProvider() + { + @Override + public AccessibilityNodeInfo createAccessibilityNodeInfo(int virtualViewId) + { + if (virtualViewId == View.NO_ID) { + return getNodeForView(); + } + return getNodeForVirtualViewId(virtualViewId); + } + + @Override + public boolean performAction(int virtualViewId, int action, Bundle arguments) + { + boolean handled = false; + //Log.i(TAG, "PERFORM ACTION: " + action + " on " + virtualViewId); + switch (action) { + case AccessibilityNodeInfo.ACTION_ACCESSIBILITY_FOCUS: + // Only handle the FOCUS action if it's placing focus on + // a different view that was previously focused. + if (m_focusedVirtualViewId != virtualViewId) { + m_focusedVirtualViewId = virtualViewId; + m_view.invalidate(); + sendEventForVirtualViewId(virtualViewId, + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUSED); + handled = true; + } + break; + case AccessibilityNodeInfo.ACTION_CLEAR_ACCESSIBILITY_FOCUS: + if (m_focusedVirtualViewId == virtualViewId) { + m_focusedVirtualViewId = INVALID_ID; + } + // Since we're managing focus at the parent level, we are + // likely to receive a FOCUS action before a CLEAR_FOCUS + // action. We'll give the benefit of the doubt to the + // framework and always handle FOCUS_CLEARED. + m_view.invalidate(); + sendEventForVirtualViewId(virtualViewId, + AccessibilityEvent.TYPE_VIEW_ACCESSIBILITY_FOCUS_CLEARED); + handled = true; + break; + default: + // Let the node provider handle focus for the view node. + if (virtualViewId == View.NO_ID) { + return m_view.performAccessibilityAction(action, arguments); + } + } + handled |= performActionForVirtualViewId(virtualViewId, action, arguments); + + return handled; + } + }; + + protected boolean performActionForVirtualViewId(int virtualViewId, int action, Bundle arguments) + { +// Log.i(TAG, "ACTION " + action + " on " + virtualViewId); +// dumpNodes(virtualViewId); + switch (action) { + case AccessibilityNodeInfo.ACTION_CLICK: + boolean success = QtNativeAccessibility.clickAction(virtualViewId); + if (success) + sendEventForVirtualViewId(virtualViewId, AccessibilityEvent.TYPE_VIEW_CLICKED); + return success; + } + return false; + } +} diff --git a/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtNativeAccessibility.java b/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtNativeAccessibility.java new file mode 100644 index 0000000000..b1cc82c065 --- /dev/null +++ b/src/android/accessibility/jar/src/org/qtproject/qt5/android/accessibility/QtNativeAccessibility.java @@ -0,0 +1,58 @@ +/**************************************************************************** +** +** Copyright (C) 2013 Digia Plc and/or its subsidiary(-ies). +** Contact: http://www.qt-project.org/legal +** +** This file is part of the Android port of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and Digia. For licensing terms and +** conditions see http://qt.digia.com/licensing. For further information +** use the contact form at http://qt.digia.com/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Digia gives you certain additional +** rights. These rights are described in the Digia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +package org.qtproject.qt5.android.accessibility; + +import android.graphics.Rect; +import android.view.accessibility.AccessibilityNodeInfo; + +class QtNativeAccessibility +{ + static native void setActive(boolean enable); + static native int[] childIdListForAccessibleObject(int objectId); + static native int parentId(int objectId); + static native String descriptionForAccessibleObject(int objectId); + static native Rect screenRect(int objectId); + static native int hitTest(float x, float y); + static native boolean clickAction(int objectId); + + static native void populateNode(int objectId, AccessibilityNodeInfo node); +} diff --git a/src/android/android.pro b/src/android/android.pro index a5db78e32f..1850f012c0 100644 --- a/src/android/android.pro +++ b/src/android/android.pro @@ -1,2 +1,2 @@ TEMPLATE = subdirs -SUBDIRS = jar java +SUBDIRS = jar java accessibility diff --git a/src/android/jar/src/org/qtproject/qt5/android/QtSurface.java b/src/android/jar/src/org/qtproject/qt5/android/QtSurface.java index 6cc2f1e333..cd0bddf2c8 100644 --- a/src/android/jar/src/org/qtproject/qt5/android/QtSurface.java +++ b/src/android/jar/src/org/qtproject/qt5/android/QtSurface.java @@ -55,17 +55,23 @@ import android.view.MotionEvent; import android.view.SurfaceHolder; import android.view.SurfaceView; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + public class QtSurface extends SurfaceView implements SurfaceHolder.Callback { private Bitmap m_bitmap = null; private boolean m_started = false; private boolean m_usesGL = false; private GestureDetector m_gestureDetector; + private Object m_accessibilityDelegate = null; public QtSurface(Context context, int id) { super(context); - setFocusable(true); + setFocusable(false); + setFocusableInTouchMode(false); + getHolder().addCallback(this); getHolder().setType(SurfaceHolder.SURFACE_TYPE_GPU); setId(id); @@ -107,6 +113,44 @@ public class QtSurface extends SurfaceView implements SurfaceHolder.Callback if (m_usesGL) holder.setFormat(PixelFormat.RGBA_8888); + + + // Initialize Accessibility + // The accessibility code depends on android API level 16, so dynamically resolve it + try { + final String a11yDelegateClassName = "org.qtproject.qt5.android.accessibility.QtAccessibilityDelegate"; + Class qtDelegateClass = Class.forName(a11yDelegateClassName); + Constructor constructor = qtDelegateClass.getConstructor(Class.forName("android.view.View")); + m_accessibilityDelegate = constructor.newInstance(this); + + Class a11yDelegateClass = Class.forName("android.view.View$AccessibilityDelegate"); + Method setDelegateMethod = this.getClass().getMethod("setAccessibilityDelegate", a11yDelegateClass); + setDelegateMethod.invoke(this, m_accessibilityDelegate); + } catch (ClassNotFoundException e) { + // Class not found is fine since we are compatible with Android API < 16, but the function will + // only be available with that API level. + } catch (Exception e) { + // Unknown exception means something went wrong. + Log.w("Qt A11y", "Unknown exception: " + e.toString()); + } + } + + public boolean dispatchHoverEvent(MotionEvent event) { + // Always attempt to dispatch hover events to accessibility first. + if (m_accessibilityDelegate != null) { + try { + Method dispHoverA11y = m_accessibilityDelegate.getClass().getMethod("dispatchHoverEvent", MotionEvent.class); + boolean ret = (Boolean) dispHoverA11y.invoke(m_accessibilityDelegate, event); + if (ret) + return true; + SurfaceView view = (SurfaceView) this; + Method dispHoverView = view.getClass().getMethod("dispatchHoverEvent", MotionEvent.class); + return (Boolean) dispHoverView.invoke(view, event); + } catch (Exception e) { + Log.w("Qt A11y", "EXCEPTION in dispatchHoverEvent for Accessibility: " + e); + } + } + return false; } @Override -- cgit v1.2.3