diff options
Diffstat (limited to 'src/plugins/platforms/android/androidjniaccessibility.cpp')
-rw-r--r-- | src/plugins/platforms/android/androidjniaccessibility.cpp | 423 |
1 files changed, 367 insertions, 56 deletions
diff --git a/src/plugins/platforms/android/androidjniaccessibility.cpp b/src/plugins/platforms/android/androidjniaccessibility.cpp index d4b7f38bf6..adad9dde98 100644 --- a/src/plugins/platforms/android/androidjniaccessibility.cpp +++ b/src/plugins/platforms/android/androidjniaccessibility.cpp @@ -37,6 +37,7 @@ ** ****************************************************************************/ +#include "androiddeadlockprotector.h" #include "androidjniaccessibility.h" #include "androidjnimain.h" #include "qandroidplatformintegration.h" @@ -50,6 +51,7 @@ #include <QtCore/private/qjnihelpers_p.h> #include <QtCore/private/qjni_p.h> #include <QtGui/private/qhighdpiscaling_p.h> +#include <QtCore/QObject> #include "qdebug.h" @@ -65,27 +67,73 @@ namespace QtAndroidAccessibility static jmethodID m_setCheckedMethodID = 0; static jmethodID m_setClickableMethodID = 0; static jmethodID m_setContentDescriptionMethodID = 0; + static jmethodID m_setEditableMethodID = 0; static jmethodID m_setEnabledMethodID = 0; static jmethodID m_setFocusableMethodID = 0; static jmethodID m_setFocusedMethodID = 0; + static jmethodID m_setHeadingMethodID = 0; static jmethodID m_setScrollableMethodID = 0; static jmethodID m_setTextSelectionMethodID = 0; static jmethodID m_setVisibleToUserMethodID = 0; + static bool m_accessibilityActivated = false; + + // This object is needed to schedule the execution of the code that + // deals with accessibility instances to the Qt main thread. + // Because of that almost every method here is split into two parts. + // The _helper part is executed in the context of m_accessibilityContext + // on the main thread. The other part is executed in Java thread. + static QPointer<QObject> m_accessibilityContext = nullptr; + + // This method is called from the Qt main thread, and normally a + // QGuiApplication instance will be used as a parent. + void createAccessibilityContextObject(QObject *parent) + { + if (m_accessibilityContext) + m_accessibilityContext->deleteLater(); + m_accessibilityContext = new QObject(parent); + } + + template <typename Func, typename Ret> + void runInObjectContext(QObject *context, Func &&func, Ret *retVal) + { + AndroidDeadlockProtector protector; + if (!protector.acquire()) { + __android_log_print(ANDROID_LOG_WARN, m_qtTag, + "Could not run accessibility call in object context, accessing " + "main thread could lead to deadlock"); + return; + } + + if (!QtAndroid::blockEventLoopsWhenSuspended() + || QGuiApplication::applicationState() != Qt::ApplicationSuspended) { + QMetaObject::invokeMethod(context, func, Qt::BlockingQueuedConnection, retVal); + } else { + __android_log_print(ANDROID_LOG_WARN, m_qtTag, + "Could not run accessibility call in object context, event loop suspended."); + } + } + void initialize() { QJNIObjectPrivate::callStaticMethod<void>(QtAndroid::applicationClass(), "initializeAccessibility"); } + bool isActive() + { + return m_accessibilityActivated; + } + static void setActive(JNIEnv */*env*/, jobject /*thiz*/, jboolean active) { QMutexLocker lock(QtAndroid::platformInterfaceMutex()); QAndroidPlatformIntegration *platformIntegration = QtAndroid::androidPlatformIntegration(); + m_accessibilityActivated = active; if (platformIntegration) platformIntegration->accessibility()->setActive(active); else - __android_log_print(ANDROID_LOG_WARN, m_qtTag, "Could not activate platform accessibility."); + __android_log_print(ANDROID_LOG_WARN, m_qtTag, "Could not (yet) activate platform accessibility."); } QAccessibleInterface *interfaceFromId(jint objectId) @@ -101,7 +149,38 @@ namespace QtAndroidAccessibility return iface; } - static jintArray childIdListForAccessibleObject(JNIEnv *env, jobject /*thiz*/, jint objectId) + void notifyLocationChange(uint accessibilityObjectId) + { + QtAndroid::notifyAccessibilityLocationChange(accessibilityObjectId); + } + + static int parentId_helper(int objectId); // forward declaration + + void notifyObjectHide(uint accessibilityObjectId) + { + const auto parentObjectId = parentId_helper(accessibilityObjectId); + QtAndroid::notifyObjectHide(accessibilityObjectId, parentObjectId); + } + + void notifyObjectFocus(uint accessibilityObjectId) + { + QtAndroid::notifyObjectFocus(accessibilityObjectId); + } + + static jstring jvalueForAccessibleObject(int objectId); // forward declaration + + void notifyValueChanged(uint accessibilityObjectId) + { + jstring value = jvalueForAccessibleObject(accessibilityObjectId); + QtAndroid::notifyValueChanged(accessibilityObjectId, value); + } + + void notifyScrolledEvent(uint accessiblityObjectId) + { + QtAndroid::notifyScrolledEvent(accessiblityObjectId); + } + + static QVarLengthArray<int, 8> childIdListForAccessibleObject_helper(int objectId) { QAccessibleInterface *iface = interfaceFromId(objectId); if (iface && iface->isValid()) { @@ -113,6 +192,18 @@ namespace QtAndroidAccessibility if (child && child->isValid()) ifaceIdArray.append(QAccessible::uniqueId(child)); } + return ifaceIdArray; + } + return {}; + } + + static jintArray childIdListForAccessibleObject(JNIEnv *env, jobject /*thiz*/, jint objectId) + { + if (m_accessibilityContext) { + QVarLengthArray<jint, 8> ifaceIdArray; + runInObjectContext(m_accessibilityContext, [objectId]() { + return childIdListForAccessibleObject_helper(objectId); + }, &ifaceIdArray); jintArray jArray = env->NewIntArray(jsize(ifaceIdArray.count())); env->SetIntArrayRegion(jArray, 0, ifaceIdArray.count(), ifaceIdArray.data()); return jArray; @@ -121,7 +212,7 @@ namespace QtAndroidAccessibility return env->NewIntArray(jsize(0)); } - static jint parentId(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + static int parentId_helper(int objectId) { QAccessibleInterface *iface = interfaceFromId(objectId); if (iface && iface->isValid()) { @@ -135,21 +226,47 @@ namespace QtAndroidAccessibility return -1; } - static jobject screenRect(JNIEnv *env, jobject /*thiz*/, jint objectId) + static jint parentId(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + { + jint result = -1; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return parentId_helper(objectId); + }, &result); + } + return result; + } + + static QRect screenRect_helper(int objectId, bool clip = true) { QRect rect; QAccessibleInterface *iface = interfaceFromId(objectId); if (iface && iface->isValid()) { rect = QHighDpi::toNativePixels(iface->rect(), iface->window()); } + // If the widget is not fully in-bound in its parent then we have to clip the rectangle to draw + if (clip && iface && iface->parent() && iface->parent()->isValid()) { + const auto parentRect = QHighDpi::toNativePixels(iface->parent()->rect(), iface->parent()->window()); + rect = rect.intersected(parentRect); + } + return rect; + } + static jobject screenRect(JNIEnv *env, jobject /*thiz*/, jint objectId) + { + QRect rect; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return screenRect_helper(objectId); + }, &rect); + } jclass rectClass = env->FindClass("android/graphics/Rect"); jmethodID ctor = env->GetMethodID(rectClass, "<init>", "(IIII)V"); jobject jrect = env->NewObject(rectClass, ctor, rect.left(), rect.top(), rect.right(), rect.bottom()); return jrect; } - static jint hitTest(JNIEnv */*env*/, jobject /*thiz*/, jfloat x, jfloat y) + static int hitTest_helper(float x, float y) { QAccessibleInterface *root = interfaceFromId(-1); if (root && root->isValid()) { @@ -167,33 +284,106 @@ namespace QtAndroidAccessibility return -1; } - static jboolean clickAction(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + static jint hitTest(JNIEnv */*env*/, jobject /*thiz*/, jfloat x, jfloat y) + { + jint result = -1; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [x, y]() { + return hitTest_helper(x, y); + }, &result); + } + return result; + } + + static void invokeActionOnInterfaceInMainThread(QAccessibleActionInterface* actionInterface, + const QString& action) + { + // Queue the action and return back to Java thread, so that we do not + // block it for too long + QMetaObject::invokeMethod(qApp, [actionInterface, action]() { + actionInterface->doAction(action); + }, Qt::QueuedConnection); + } + + static bool clickAction_helper(int objectId) { -// qDebug() << "A11Y: CLICK: " << objectId; QAccessibleInterface *iface = interfaceFromId(objectId); - if (iface && iface->isValid() && iface->actionInterface()) { - if (iface->actionInterface()->actionNames().contains(QAccessibleActionInterface::pressAction())) - iface->actionInterface()->doAction(QAccessibleActionInterface::pressAction()); - else - iface->actionInterface()->doAction(QAccessibleActionInterface::toggleAction()); + if (!iface || !iface->isValid() || !iface->actionInterface()) + return false; + + const auto& actionNames = iface->actionInterface()->actionNames(); + + if (actionNames.contains(QAccessibleActionInterface::pressAction())) { + invokeActionOnInterfaceInMainThread(iface->actionInterface(), + QAccessibleActionInterface::pressAction()); + } else if (actionNames.contains(QAccessibleActionInterface::toggleAction())) { + invokeActionOnInterfaceInMainThread(iface->actionInterface(), + QAccessibleActionInterface::toggleAction()); + } else { + return false; } - return false; + return true; } - static jboolean scrollForward(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + static jboolean clickAction(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + { + bool result = false; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return clickAction_helper(objectId); + }, &result); + } + return result; + } + + static bool scroll_helper(int objectId, const QString &actionName) { QAccessibleInterface *iface = interfaceFromId(objectId); if (iface && iface->isValid()) - return QAccessibleBridgeUtils::performEffectiveAction(iface, QAccessibleActionInterface::increaseAction()); + return QAccessibleBridgeUtils::performEffectiveAction(iface, actionName); return false; } + static jboolean scrollForward(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) + { + bool result = false; + + const auto& ids = childIdListForAccessibleObject_helper(objectId); + if (ids.isEmpty()) + return false; + + const int firstChildId = ids.first(); + const QRect oldPosition = screenRect_helper(firstChildId, false); + + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return scroll_helper(objectId, QAccessibleActionInterface::increaseAction()); + }, &result); + } + + // Don't check for position change if the call was not successful + return result && oldPosition != screenRect_helper(firstChildId, false); + } + static jboolean scrollBackward(JNIEnv */*env*/, jobject /*thiz*/, jint objectId) { - QAccessibleInterface *iface = interfaceFromId(objectId); - if (iface && iface->isValid()) - return QAccessibleBridgeUtils::performEffectiveAction(iface, QAccessibleActionInterface::decreaseAction()); - return false; + bool result = false; + + const auto& ids = childIdListForAccessibleObject_helper(objectId); + if (ids.isEmpty()) + return false; + + const int firstChildId = ids.first(); + const QRect oldPosition = screenRect_helper(firstChildId, false); + + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return scroll_helper(objectId, QAccessibleActionInterface::decreaseAction()); + }, &result); + } + + // Don't check for position change if the call was not successful + return result && oldPosition != screenRect_helper(firstChildId, false); } @@ -206,81 +396,198 @@ if (!clazz) { \ //__android_log_print(ANDROID_LOG_FATAL, m_qtTag, m_methodErrorMsg, METHOD_NAME, METHOD_SIGNATURE); + static QString textFromValue(QAccessibleInterface *iface) + { + QString valueStr; + QAccessibleValueInterface *valueIface = iface->valueInterface(); + if (valueIface) { + const QVariant valueVar = valueIface->currentValue(); + const auto type = static_cast<QMetaType::Type>(valueVar.type()); + if (type == QMetaType::Double || type == QMetaType::Float) { + // QVariant's toString() formats floating-point values with + // FloatingPointShortest, which is not an accessible + // representation; nor, in many cases, is it suitable to the UI + // element whose value we're looking at. So roll our own + // A11Y-friendly conversion to string. + const double val = valueVar.toDouble(); + // Try to use minimumStepSize() to determine precision + bool stepIsValid = false; + const double step = qAbs(valueIface->minimumStepSize().toDouble(&stepIsValid)); + if (!stepIsValid || qFuzzyIsNull(step)) { + // Ignore step, use default precision + valueStr = qFuzzyIsNull(val) ? QStringLiteral("0") : QString::number(val, 'f'); + } else { + const int precision = [](double s) { + int count = 0; + while (s < 1. && !qFuzzyCompare(s, 1.)) { + ++count; + s *= 10; + } + // If s is now 1.25, we want to show some more digits, + // but don't want to get silly with a step like 1./7; + // so only include a few extra digits. + const int stop = count + 3; + const auto fractional = [](double v) { + double whole = 0.0; + std::modf(v + 0.5, &whole); + return qAbs(v - whole); + }; + s = fractional(s); + while (count < stop && !qFuzzyIsNull(s)) { + ++count; + s = fractional(s * 10); + } + return count; + }(step); + valueStr = qFuzzyIsNull(val / step) ? QStringLiteral("0") + : QString::number(val, 'f', precision); + } + } else { + valueStr = valueVar.toString(); + } + } + return valueStr; + } + static jstring jvalueForAccessibleObject(int objectId) + { + QAccessibleInterface *iface = interfaceFromId(objectId); + const QString value = textFromValue(iface); + QJNIEnvironmentPrivate env; + jstring jstr = env->NewString((jchar*)value.constData(), (jsize)value.size()); +#ifdef QT_DEBUG + env->ExceptionDescribe(); +#endif // QT_DEBUG + env->ExceptionClear(); + return jstr; + } - static jstring descriptionForAccessibleObject_helper(JNIEnv *env, QAccessibleInterface *iface) + static QString descriptionForInterface(QAccessibleInterface *iface) { QString desc; if (iface && iface->isValid()) { + bool hasValue = false; desc = iface->text(QAccessible::Name); if (desc.isEmpty()) desc = iface->text(QAccessible::Description); if (desc.isEmpty()) { desc = iface->text(QAccessible::Value); - if (desc.isEmpty()) { - if (QAccessibleValueInterface *valueIface = iface->valueInterface()) { - desc= valueIface->currentValue().toString(); - } + hasValue = !desc.isEmpty(); + } + if (!hasValue && iface->valueInterface()) { + const QString valueStr = textFromValue(iface); + if (!valueStr.isEmpty()) { + if (!desc.isEmpty()) + desc.append(QChar(QChar::Space)); + desc.append(valueStr); } } } - return env->NewString((jchar*) desc.constData(), (jsize) desc.size()); + return desc; } - static jstring descriptionForAccessibleObject(JNIEnv *env, jobject /*thiz*/, jint objectId) + static QString descriptionForAccessibleObject_helper(int objectId) { QAccessibleInterface *iface = interfaceFromId(objectId); - return descriptionForAccessibleObject_helper(env, iface); + return descriptionForInterface(iface); + } + + static jstring descriptionForAccessibleObject(JNIEnv *env, jobject /*thiz*/, jint objectId) + { + QString desc; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return descriptionForAccessibleObject_helper(objectId); + }, &desc); + } + return env->NewString((jchar*) desc.constData(), (jsize) desc.size()); } - static bool populateNode(JNIEnv *env, jobject /*thiz*/, jint objectId, jobject node) + + struct NodeInfo + { + bool valid = false; + QAccessible::State state; + QAccessible::Role role; + QStringList actions; + QString description; + bool hasTextSelection = false; + int selectionStart = 0; + int selectionEnd = 0; + }; + + static NodeInfo populateNode_helper(int objectId) { + NodeInfo info; QAccessibleInterface *iface = interfaceFromId(objectId); - if (!iface || !iface->isValid()) { + if (iface && iface->isValid()) { + info.valid = true; + info.state = iface->state(); + info.role = iface->role(); + info.actions = QAccessibleBridgeUtils::effectiveActionNames(iface); + info.description = descriptionForInterface(iface); + QAccessibleTextInterface *textIface = iface->textInterface(); + if (textIface && (textIface->selectionCount() > 0)) { + info.hasTextSelection = true; + textIface->selection(0, &info.selectionStart, &info.selectionEnd); + } + } + return info; + } + + static jboolean populateNode(JNIEnv *env, jobject /*thiz*/, jint objectId, jobject node) + { + NodeInfo info; + if (m_accessibilityContext) { + runInObjectContext(m_accessibilityContext, [objectId]() { + return populateNode_helper(objectId); + }, &info); + } + if (!info.valid) { __android_log_print(ANDROID_LOG_WARN, m_qtTag, "Accessibility: populateNode for Invalid ID"); return false; } - QAccessible::State state = iface->state(); - const QStringList actions = QAccessibleBridgeUtils::effectiveActionNames(iface); - const bool hasClickableAction = actions.contains(QAccessibleActionInterface::pressAction()) - || actions.contains(QAccessibleActionInterface::toggleAction()); - const bool hasIncreaseAction = actions.contains(QAccessibleActionInterface::increaseAction()); - const bool hasDecreaseAction = actions.contains(QAccessibleActionInterface::decreaseAction()); - // try to fill in the text property, this is what the screen reader reads - jstring jdesc = descriptionForAccessibleObject_helper(env, iface); - - if (QAccessibleTextInterface *textIface = iface->textInterface()) { - if (m_setTextSelectionMethodID && textIface->selectionCount() > 0) { - int startSelection; - int endSelection; - textIface->selection(0, &startSelection, &endSelection); - env->CallVoidMethod(node, m_setTextSelectionMethodID, startSelection, endSelection); - } + const bool hasClickableAction = + info.actions.contains(QAccessibleActionInterface::pressAction()) || + info.actions.contains(QAccessibleActionInterface::toggleAction()); + const bool hasIncreaseAction = + info.actions.contains(QAccessibleActionInterface::increaseAction()); + const bool hasDecreaseAction = + info.actions.contains(QAccessibleActionInterface::decreaseAction()); + + if (info.hasTextSelection && m_setTextSelectionMethodID) { + env->CallVoidMethod(node, m_setTextSelectionMethodID, info.selectionStart, + info.selectionEnd); } - env->CallVoidMethod(node, m_setEnabledMethodID, !state.disabled); - env->CallVoidMethod(node, m_setCheckableMethodID, (bool)state.checkable); - env->CallVoidMethod(node, m_setCheckedMethodID, (bool)state.checked); - env->CallVoidMethod(node, m_setFocusableMethodID, (bool)state.focusable); - env->CallVoidMethod(node, m_setFocusedMethodID, (bool)state.focused); - env->CallVoidMethod(node, m_setVisibleToUserMethodID, !state.invisible); + env->CallVoidMethod(node, m_setCheckableMethodID, (bool)info.state.checkable); + env->CallVoidMethod(node, m_setCheckedMethodID, (bool)info.state.checked); + env->CallVoidMethod(node, m_setEditableMethodID, info.state.editable); + env->CallVoidMethod(node, m_setEnabledMethodID, !info.state.disabled); + env->CallVoidMethod(node, m_setFocusableMethodID, (bool)info.state.focusable); + env->CallVoidMethod(node, m_setFocusedMethodID, (bool)info.state.focused); + if (m_setHeadingMethodID) + env->CallVoidMethod(node, m_setHeadingMethodID, info.role == QAccessible::Heading); + env->CallVoidMethod(node, m_setVisibleToUserMethodID, !info.state.invisible); env->CallVoidMethod(node, m_setScrollableMethodID, hasIncreaseAction || hasDecreaseAction); - env->CallVoidMethod(node, m_setClickableMethodID, hasClickableAction); + env->CallVoidMethod(node, m_setClickableMethodID, hasClickableAction || info.role == QAccessible::Link); // Add ACTION_CLICK if (hasClickableAction) - env->CallVoidMethod(node, m_addActionMethodID, (int)16); // ACTION_CLICK defined in AccessibilityNodeInfo + env->CallVoidMethod(node, m_addActionMethodID, (int)0x00000010); // ACTION_CLICK defined in AccessibilityNodeInfo // Add ACTION_SCROLL_FORWARD if (hasIncreaseAction) - env->CallVoidMethod(node, m_addActionMethodID, (int)4096); // ACTION_SCROLL_FORWARD defined in AccessibilityNodeInfo + env->CallVoidMethod(node, m_addActionMethodID, (int)0x00001000); // ACTION_SCROLL_FORWARD defined in AccessibilityNodeInfo // Add ACTION_SCROLL_BACKWARD if (hasDecreaseAction) - env->CallVoidMethod(node, m_addActionMethodID, (int)8192); // ACTION_SCROLL_BACKWARD defined in AccessibilityNodeInfo - + env->CallVoidMethod(node, m_addActionMethodID, (int)0x00002000); // ACTION_SCROLL_BACKWARD defined in AccessibilityNodeInfo + // try to fill in the text property, this is what the screen reader reads + jstring jdesc = env->NewString((jchar*)info.description.constData(), + (jsize)info.description.size()); //CALL_METHOD(node, "setText", "(Ljava/lang/CharSequence;)V", jdesc) env->CallVoidMethod(node, m_setContentDescriptionMethodID, jdesc); @@ -324,9 +631,13 @@ if (!clazz) { \ GET_AND_CHECK_STATIC_METHOD(m_setCheckedMethodID, nodeInfoClass, "setChecked", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setClickableMethodID, nodeInfoClass, "setClickable", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setContentDescriptionMethodID, nodeInfoClass, "setContentDescription", "(Ljava/lang/CharSequence;)V"); + GET_AND_CHECK_STATIC_METHOD(m_setEditableMethodID, nodeInfoClass, "setEditable", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setEnabledMethodID, nodeInfoClass, "setEnabled", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setFocusableMethodID, nodeInfoClass, "setFocusable", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setFocusedMethodID, nodeInfoClass, "setFocused", "(Z)V"); + if (QtAndroidPrivate::androidSdkVersion() >= 28) { + GET_AND_CHECK_STATIC_METHOD(m_setHeadingMethodID, nodeInfoClass, "setHeading", "(Z)V"); + } GET_AND_CHECK_STATIC_METHOD(m_setScrollableMethodID, nodeInfoClass, "setScrollable", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setVisibleToUserMethodID, nodeInfoClass, "setVisibleToUser", "(Z)V"); GET_AND_CHECK_STATIC_METHOD(m_setTextSelectionMethodID, nodeInfoClass, "setTextSelection", "(II)V"); |