From 24c90450fe3fe097a7bca51edd6a4cffd8fd13aa Mon Sep 17 00:00:00 2001 From: Svetoslav Ganov Date: Wed, 27 Dec 2017 15:17:14 -0800 Subject: Autofill compatibility mode. Autofill helps users fill credentials, addresses, payment methods, emails, etc without manually typing. When focus lands on a fillable element the platform captures a snapshot of the screen content and sends it to an autofill service for analysis and suggestions. The screen snapshot is a structured representation of the screen content. If this content is composed of standard widgets, autofill works out-of-the-box. However, some apps do their own rendering and the content in this case looks like a single view to the platform while it may have semantic structure. For example, a view may render a login page with two input test fields. The platform exposes APIs for apps to report virtual view structure allowing autofill services to handle apps that have virtual content. As opposed to apps using standard widgets, this case requires the app developer to implement the new APIs which may require a fair amount of code and could be seen as a processes that could take some time. The most prominent typs of apps that fall into this category are browsers. Until most apps rendering virtual content and specifically browsers don't implement the virutal APIs, autofill providers need to fall- back to using the accessibliity APIs to provide autofill support for these apps. This requires developers to work against two sets of APIs - autofill and accessibility - which is incovenient and error prone. Also, users need to enable two plugins - autofill and accessibility which is confusing. Additionally, the privacy and perfomance impact of using the accessibility APIs cannot be addressed while autofill providers need to use thes APis. This change adds an autofill compatibility mode that would allow autofill services to work with apps that don't implement the virtual structure autofill APIs. The key idea is to locally enable accessibility for the target package and remap accessibility to autofill APIs and vise versa. This way an autofill provider codes against a single set of APIs, the users enable a single plugin, the privacy/performance implications of using the accessibility APIs are addressed, the target app only takes a performance hit since accessibility is enabled locally which is still more efficient compared to the performance hit it would incur if accessibility is enabled globally. To enable compatibility mode an autofill service declares in its metadata which packages it is interested in and also what is the max version code of the package for which to enable compat mode. Targeted versioning allows targeting only older versions of the package that are known to not support autofill while newer versions that are known to support autofill would work in normal mode. Since compatibility mode should be used only as a fallback we have a white list setting with the packages for which this mode can be requested. This allows applying policy to target only apps that are known to not support autofill. Test: cts-tradefed run cts-dev -m CtsAutoFillServiceTestCases cts-tradefed run cts-dev -m CtsAccessibilityServiceTestCases bug:72811034 Change-Id: I11f1580ced0f8b4300a10b3a5174a1758a5702a0 --- core/java/android/view/ViewRootImpl.java | 87 ++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) (limited to 'core/java/android/view/ViewRootImpl.java') diff --git a/core/java/android/view/ViewRootImpl.java b/core/java/android/view/ViewRootImpl.java index 7bd197e824db..810864eaac3c 100644 --- a/core/java/android/view/ViewRootImpl.java +++ b/core/java/android/view/ViewRootImpl.java @@ -71,6 +71,7 @@ import android.os.Trace; import android.util.AndroidRuntimeException; import android.util.DisplayMetrics; import android.util.Log; +import android.util.LongArray; import android.util.MergedConfiguration; import android.util.Slog; import android.util.SparseArray; @@ -114,6 +115,8 @@ import java.io.PrintWriter; import java.lang.ref.WeakReference; import java.util.ArrayList; import java.util.HashSet; +import java.util.LinkedList; +import java.util.Queue; import java.util.concurrent.CountDownLatch; /** @@ -2569,6 +2572,10 @@ public final class ViewRootImpl implements ViewParent, ~WindowManager.LayoutParams .SOFT_INPUT_IS_FORWARD_NAVIGATION; mHasHadWindowFocus = true; + + // Refocusing a window that has a focused view should fire a + // focus event for the view since the global focused view changed. + fireAccessibilityFocusEventIfHasFocusedNode(); } else { if (mPointerCapture) { handlePointerCaptureChanged(false); @@ -2578,6 +2585,86 @@ public final class ViewRootImpl implements ViewParent, mFirstInputStage.onWindowFocusChanged(hasWindowFocus); } + private void fireAccessibilityFocusEventIfHasFocusedNode() { + if (!AccessibilityManager.getInstance(mContext).isEnabled()) { + return; + } + final View focusedView = mView.findFocus(); + if (focusedView == null) { + return; + } + final AccessibilityNodeProvider provider = focusedView.getAccessibilityNodeProvider(); + if (provider == null) { + focusedView.sendAccessibilityEvent(AccessibilityEvent.TYPE_VIEW_FOCUSED); + } else { + final AccessibilityNodeInfo focusedNode = findFocusedVirtualNode(provider); + if (focusedNode != null) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId( + focusedNode.getSourceNodeId()); + // This is a best effort since clearing and setting the focus via the + // provider APIs could have side effects. We don't have a provider API + // similar to that on View to ask a given event to be fired. + final AccessibilityEvent event = AccessibilityEvent.obtain( + AccessibilityEvent.TYPE_VIEW_FOCUSED); + event.setSource(focusedView, virtualId); + event.setPackageName(focusedNode.getPackageName()); + event.setChecked(focusedNode.isChecked()); + event.setContentDescription(focusedNode.getContentDescription()); + event.setPassword(focusedNode.isPassword()); + event.getText().add(focusedNode.getText()); + event.setEnabled(focusedNode.isEnabled()); + focusedView.getParent().requestSendAccessibilityEvent(focusedView, event); + focusedNode.recycle(); + } + } + } + + private AccessibilityNodeInfo findFocusedVirtualNode(AccessibilityNodeProvider provider) { + AccessibilityNodeInfo focusedNode = provider.findFocus( + AccessibilityNodeInfo.FOCUS_INPUT); + if (focusedNode != null) { + return focusedNode; + } + + if (!mContext.isAutofillCompatibilityEnabled()) { + return null; + } + + // Unfortunately some provider implementations don't properly + // implement AccessibilityNodeProvider#findFocus + AccessibilityNodeInfo current = provider.createAccessibilityNodeInfo( + AccessibilityNodeProvider.HOST_VIEW_ID); + if (current.isFocused()) { + return current; + } + + final Queue fringe = new LinkedList<>(); + fringe.offer(current); + + while (!fringe.isEmpty()) { + current = fringe.poll(); + final LongArray childNodeIds = current.getChildNodeIds(); + if (childNodeIds== null || childNodeIds.size() <= 0) { + continue; + } + final int childCount = childNodeIds.size(); + for (int i = 0; i < childCount; i++) { + final int virtualId = AccessibilityNodeInfo.getVirtualDescendantId( + childNodeIds.get(i)); + final AccessibilityNodeInfo child = provider.createAccessibilityNodeInfo(virtualId); + if (child != null) { + if (child.isFocused()) { + return child; + } + fringe.offer(child); + } + } + current.recycle(); + } + + return null; + } + private void handleOutOfResourcesException(Surface.OutOfResourcesException e) { Log.e(mTag, "OutOfResourcesException initializing HW surface", e); try { -- cgit v1.2.3