Use IME image API impl as fallback in TextViewOnReceiveContentCallback

A bunch of apps implement the keyboard image API (see
https://developer.android.com/guide/topics/text/image-keyboard).
When image support in augmented autofill is released, we'd like it to
work immediately in apps that have previously implemented the keyboard
image API, without having to wait for these apps to move to the new
unified content insertion API. To make this possible, this change adds
logic to call the keyboard image API (InputConnection.commitContent) if
the app implements it and if the app target SDK is <= S. This gives apps
a full Android release to upgrade to the new content insertion API
while augemented autofill will immediately be able to insert images
without any changes in apps that have implemented the keyboard image
API.

Bug: 163400105
Bug: 152068298
Test: Manual and unit tests
  atest FrameworksCoreTests:TextViewOnReceiveContentCallbackTest
  atest CtsWidgetTestCases:TextViewOnReceiveContentCallbackTest
Change-Id: I9280604e7ec7e8d08c1179e6bbf0068647a41040

3 files changed, 321 insertions, 17 deletions

diff --git a/core/java/android/widget/Editor.java b/core/java/android/widget/Editor.java
index eaa738d59577..f2955ac554fc 100644
--- a/core/java/android/widget/Editor.java
+++ b/core/java/android/widget/Editor.java
@@ -206,6 +206,10 @@ public class Editor {
         int TEXT_LINK = 2;
     }
 
+    // Default content insertion handler.
+    private final TextViewOnReceiveContentCallback mDefaultOnReceiveContentCallback =
+            new TextViewOnReceiveContentCallback();
+
     // Each Editor manages its own undo stack.
     private final UndoManager mUndoManager = new UndoManager();
     private UndoOwner mUndoOwner = mUndoManager.getOwner(UNDO_OWNER_TAG, this);
@@ -584,6 +588,11 @@ public class Editor {
         mUndoOwner = mUndoManager.getOwner(UNDO_OWNER_TAG, this);
     }
 
+    @VisibleForTesting
+    public @NonNull TextViewOnReceiveContentCallback getDefaultOnReceiveContentCallback() {
+        return mDefaultOnReceiveContentCallback;
+    }
+
     /**
      * Forgets all undo and redo operations for this Editor.
      */
@@ -709,6 +718,8 @@ public class Editor {
 
         hideCursorAndSpanControllers();
         stopTextActionModeWithPreservingSelection();
+
+        mDefaultOnReceiveContentCallback.clearInputConnectionInfo();
     }
 
     private void discardTextDisplayLists() {
diff --git a/core/java/android/widget/TextView.java b/core/java/android/widget/TextView.java
index 52a3f4145e7e..7bb2b7e92a00 100644
--- a/core/java/android/widget/TextView.java
+++ b/core/java/android/widget/TextView.java
@@ -80,6 +80,7 @@ import android.os.AsyncTask;
 import android.os.Build;
 import android.os.Build.VERSION_CODES;
 import android.os.Bundle;
+import android.os.Handler;
 import android.os.LocaleList;
 import android.os.Parcel;
 import android.os.Parcelable;
@@ -890,13 +891,6 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
     @UnsupportedAppUsage
     private Editor mEditor;
 
-    /**
-     * The default content insertion callback used by {@link TextView}. See
-     * {@link #setOnReceiveContentCallback} for more info.
-     */
-    private static final TextViewOnReceiveContentCallback DEFAULT_ON_RECEIVE_CONTENT_CALLBACK =
-            new TextViewOnReceiveContentCallback();
-
     private static final int DEVICE_PROVISIONED_UNKNOWN = 0;
     private static final int DEVICE_PROVISIONED_NO = 1;
     private static final int DEVICE_PROVISIONED_YES = 2;
@@ -13723,6 +13717,23 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
         }
     }
 
+    /** @hide */
+    @Override
+    public void onInputConnectionOpenedInternal(@NonNull InputConnection ic,
+            @NonNull EditorInfo editorInfo, @Nullable Handler handler) {
+        if (mEditor != null) {
+            mEditor.getDefaultOnReceiveContentCallback().setInputConnectionInfo(ic, editorInfo);
+        }
+    }
+
+    /** @hide */
+    @Override
+    public void onInputConnectionClosedInternal() {
+        if (mEditor != null) {
+            mEditor.getDefaultOnReceiveContentCallback().clearInputConnectionInfo();
+        }
+    }
+
     /**
      * Returns the callback used for handling insertion of content into this view. See
      * {@link #setOnReceiveContentCallback} for more info.
@@ -13773,8 +13784,8 @@ public class TextView extends View implements ViewTreeObserver.OnPreDrawListener
         ClipDescription description = payload.getClip().getDescription();
         if (receiver != null && receiver.supports(this, description)) {
             receiver.onReceiveContent(this, payload);
-        } else {
-            DEFAULT_ON_RECEIVE_CONTENT_CALLBACK.onReceiveContent(this, payload);
+        } else if (mEditor != null) {
+            mEditor.getDefaultOnReceiveContentCallback().onReceiveContent(this, payload);
         }
     }
 
diff --git a/core/java/android/widget/TextViewOnReceiveContentCallback.java b/core/java/android/widget/TextViewOnReceiveContentCallback.java
index 35618cb3d2a5..d7c95b7eae86 100644
--- a/core/java/android/widget/TextViewOnReceiveContentCallback.java
+++ b/core/java/android/widget/TextViewOnReceiveContentCallback.java
@@ -16,24 +16,44 @@
 
 package android.widget;
 
+import static android.content.ContentResolver.SCHEME_CONTENT;
 import static android.view.OnReceiveContentCallback.Payload.FLAG_CONVERT_TO_PLAIN_TEXT;
 import static android.view.OnReceiveContentCallback.Payload.SOURCE_AUTOFILL;
 import static android.view.OnReceiveContentCallback.Payload.SOURCE_DRAG_AND_DROP;
 
+import static java.util.Collections.singleton;
+
 import android.annotation.NonNull;
+import android.annotation.Nullable;
 import android.annotation.SuppressLint;
+import android.compat.Compatibility;
+import android.compat.annotation.ChangeId;
+import android.compat.annotation.EnabledAfter;
 import android.content.ClipData;
+import android.content.ClipDescription;
 import android.content.Context;
+import android.net.Uri;
+import android.os.Build;
+import android.os.Bundle;
 import android.text.Editable;
 import android.text.Selection;
 import android.text.SpannableStringBuilder;
 import android.text.Spanned;
+import android.util.ArraySet;
 import android.util.Log;
 import android.view.OnReceiveContentCallback;
 import android.view.OnReceiveContentCallback.Payload.Flags;
 import android.view.OnReceiveContentCallback.Payload.Source;
+import android.view.View;
+import android.view.inputmethod.EditorInfo;
+import android.view.inputmethod.InputConnection;
+import android.view.inputmethod.InputContentInfo;
+
+import com.android.internal.annotations.VisibleForTesting;
 
-import java.util.Collections;
+import java.lang.ref.WeakReference;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Set;
 
 /**
@@ -46,19 +66,26 @@ import java.util.Set;
 public class TextViewOnReceiveContentCallback implements OnReceiveContentCallback<TextView> {
     private static final String LOG_TAG = "OnReceiveContent";
 
-    private static final Set<String> MIME_TYPES_ALL_TEXT = Collections.singleton("text/*");
+    private static final String MIME_TYPE_ALL_TEXT = "text/*";
+    private static final Set<String> MIME_TYPES_ALL_TEXT = singleton(MIME_TYPE_ALL_TEXT);
+
+    @Nullable private InputConnectionInfo mInputConnectionInfo;
+    @Nullable private ArraySet<String> mCachedSupportedMimeTypes;
 
     @SuppressLint("CallbackMethodName")
     @NonNull
     @Override
     public Set<String> getSupportedMimeTypes(@NonNull TextView view) {
-        return MIME_TYPES_ALL_TEXT;
+        if (!isUsageOfImeCommitContentEnabled(view)) {
+            return MIME_TYPES_ALL_TEXT;
+        }
+        return getSupportedMimeTypesAugmentedWithImeCommitContentMimeTypes();
     }
 
     @Override
     public boolean onReceiveContent(@NonNull TextView view, @NonNull Payload payload) {
         if (Log.isLoggable(LOG_TAG, Log.DEBUG)) {
-            Log.d(LOG_TAG, "onReceive:" + payload);
+            Log.d(LOG_TAG, "onReceive: " + payload);
         }
         ClipData clip = payload.getClip();
         @Source int source = payload.getSource();
@@ -109,13 +136,22 @@ public class TextViewOnReceiveContentCallback implements OnReceiveContentCallbac
         editable.replace(start, end, replacement);
     }
 
-    private static boolean onReceiveForAutofill(@NonNull TextView textView, @NonNull ClipData clip,
+    private boolean onReceiveForAutofill(@NonNull TextView view, @NonNull ClipData clip,
             @Flags int flags) {
-        final CharSequence text = coerceToText(clip, textView.getContext(), flags);
+        if (isUsageOfImeCommitContentEnabled(view)) {
+            clip = handleNonTextViaImeCommitContent(clip);
+            if (clip == null) {
+                if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                    Log.v(LOG_TAG, "onReceive: Handled via IME");
+                }
+                return true;
+            }
+        }
+        final CharSequence text = coerceToText(clip, view.getContext(), flags);
         // First autofill it...
-        textView.setText(text);
+        view.setText(text);
         // ...then move cursor to the end.
-        final Editable editable = (Editable) textView.getText();
+        final Editable editable = (Editable) view.getText();
         Selection.setSelection(editable, editable.length());
         return true;
     }
@@ -146,4 +182,250 @@ public class TextViewOnReceiveContentCallback implements OnReceiveContentCallbac
         }
         return ssb;
     }
+
+    /**
+     * On Android S and above, the platform can provide non-text suggestions (e.g. images) via the
+     * augmented autofill framework (see
+     * <a href="/guide/topics/text/autofill-services">autofill services</a>). In order for an app to
+     * be able to handle these suggestions, it must normally implement the
+     * {@link android.view.OnReceiveContentCallback} API. To make the adoption of this smoother for
+     * apps that have previously implemented the
+     * {@link android.view.inputmethod.InputConnection#commitContent(InputContentInfo, int, Bundle)}
+     * API, we reuse that API as a fallback if {@link android.view.OnReceiveContentCallback} is not
+     * yet implemented by the app. This fallback is only enabled on Android S. This change ID
+     * disables the fallback, such that apps targeting Android T and above must implement the
+     * {@link android.view.OnReceiveContentCallback} API in order to accept non-text suggestions.
+     */
+    @ChangeId
+    @EnabledAfter(targetSdkVersion = Build.VERSION_CODES.S) // Enabled on Android T and higher
+    private static final long AUTOFILL_NON_TEXT_REQUIRES_ON_RECEIVE_CONTENT_CALLBACK = 163400105L;
+
+    /**
+     * Returns true if we can use the IME {@link InputConnection#commitContent} API in order handle
+     * non-text content.
+     */
+    private static boolean isUsageOfImeCommitContentEnabled(@NonNull View view) {
+        if (view.getOnReceiveContentCallback() != null) {
+            if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                Log.v(LOG_TAG, "Fallback to commitContent disabled (custom callback is set)");
+            }
+            return false;
+        }
+        if (Compatibility.isChangeEnabled(AUTOFILL_NON_TEXT_REQUIRES_ON_RECEIVE_CONTENT_CALLBACK)) {
+            if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                Log.v(LOG_TAG, "Fallback to commitContent disabled (target SDK is above S)");
+            }
+            return false;
+        }
+        return true;
+    }
+
+    private static final class InputConnectionInfo {
+        @NonNull private final WeakReference<InputConnection> mInputConnection;
+        @NonNull private final String[] mEditorInfoContentMimeTypes;
+
+        private InputConnectionInfo(@NonNull InputConnection inputConnection,
+                @NonNull String[] editorInfoContentMimeTypes) {
+            mInputConnection = new WeakReference<>(inputConnection);
+            mEditorInfoContentMimeTypes = editorInfoContentMimeTypes;
+        }
+
+        @Override
+        public String toString() {
+            return "InputConnectionInfo{"
+                    + "mimeTypes=" + Arrays.toString(mEditorInfoContentMimeTypes)
+                    + ", ic=" + mInputConnection
+                    + '}';
+        }
+    }
+
+    /**
+     * Invoked by the platform when an {@link InputConnection} is successfully created for the view
+     * that owns this callback instance.
+     */
+    void setInputConnectionInfo(@NonNull InputConnection ic, @NonNull EditorInfo editorInfo) {
+        if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+            Log.v(LOG_TAG, "setInputConnectionInfo: "
+                    + Arrays.toString(editorInfo.contentMimeTypes));
+        }
+        String[] contentMimeTypes = editorInfo.contentMimeTypes;
+        if (contentMimeTypes == null || contentMimeTypes.length == 0) {
+            mInputConnectionInfo = null;
+        } else {
+            mInputConnectionInfo = new InputConnectionInfo(ic, contentMimeTypes);
+        }
+    }
+
+    /**
+     * Invoked by the platform when an {@link InputConnection} is closed for the view that owns this
+     * callback instance.
+     */
+    void clearInputConnectionInfo() {
+        if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+            Log.v(LOG_TAG, "clearInputConnectionInfo: " + mInputConnectionInfo);
+        }
+        mInputConnectionInfo = null;
+    }
+
+    private Set<String> getSupportedMimeTypesAugmentedWithImeCommitContentMimeTypes() {
+        InputConnectionInfo icInfo = mInputConnectionInfo;
+        if (icInfo == null) {
+            if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                Log.v(LOG_TAG, "getSupportedMimeTypes: No usable EditorInfo/InputConnection");
+            }
+            return MIME_TYPES_ALL_TEXT;
+        }
+        String[] editorInfoContentMimeTypes = icInfo.mEditorInfoContentMimeTypes;
+        if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+            Log.v(LOG_TAG, "getSupportedMimeTypes: Augmenting with EditorInfo.contentMimeTypes: "
+                    + Arrays.toString(editorInfoContentMimeTypes));
+        }
+        ArraySet<String> supportedMimeTypes = mCachedSupportedMimeTypes;
+        if (canReuse(supportedMimeTypes, editorInfoContentMimeTypes)) {
+            return supportedMimeTypes;
+        }
+        supportedMimeTypes = new ArraySet<>(editorInfoContentMimeTypes);
+        supportedMimeTypes.add(MIME_TYPE_ALL_TEXT);
+        mCachedSupportedMimeTypes = supportedMimeTypes;
+        return supportedMimeTypes;
+    }
+
+    /**
+     * We want to avoid creating a new set on every invocation of {@link #getSupportedMimeTypes}.
+     * This method will check if the cached set of MIME types matches the data in the given array
+     * from {@link EditorInfo} or if a new set should be created. The custom logic is needed for
+     * comparing the data because the set contains the additional "text/*" MIME type.
+     *
+     * @param cachedMimeTypes Previously cached set of MIME types.
+     * @param newEditorInfoMimeTypes MIME types from {@link EditorInfo}.
+     *
+     * @return Returns true if the data in the given cached set matches the data in the array.
+     *
+     * @hide
+     */
+    @VisibleForTesting
+    public static boolean canReuse(@Nullable ArraySet<String> cachedMimeTypes,
+            @NonNull String[] newEditorInfoMimeTypes) {
+        if (cachedMimeTypes == null) {
+            return false;
+        }
+        if (newEditorInfoMimeTypes.length != cachedMimeTypes.size()
+                && newEditorInfoMimeTypes.length != (cachedMimeTypes.size() - 1)) {
+            return false;
+        }
+        final boolean ignoreAllTextMimeType =
+                newEditorInfoMimeTypes.length == (cachedMimeTypes.size() - 1);
+        for (String mimeType : cachedMimeTypes) {
+            if (ignoreAllTextMimeType && mimeType.equals(MIME_TYPE_ALL_TEXT)) {
+                continue;
+            }
+            boolean present = false;
+            for (String editorInfoContentMimeType : newEditorInfoMimeTypes) {
+                if (editorInfoContentMimeType.equals(mimeType)) {
+                    present = true;
+                    break;
+                }
+            }
+            if (!present) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Tries to insert the content in the clip into the app via the image keyboard API. If all the
+     * items in the clip are successfully inserted, returns null. If one or more of the items in the
+     * clip cannot be inserted, returns a non-null clip that contains the items that were not
+     * inserted.
+     */
+    @Nullable
+    private ClipData handleNonTextViaImeCommitContent(@NonNull ClipData clip) {
+        ClipDescription description = clip.getDescription();
+        if (!containsUri(clip) || containsOnlyText(clip)) {
+            if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                Log.v(LOG_TAG, "onReceive: Clip doesn't contain any non-text URIs: "
+                        + description);
+            }
+            return clip;
+        }
+
+        InputConnectionInfo icInfo = mInputConnectionInfo;
+        InputConnection inputConnection = (icInfo != null) ? icInfo.mInputConnection.get() : null;
+        if (inputConnection == null) {
+            if (Log.isLoggable(LOG_TAG, Log.DEBUG)) {
+                Log.d(LOG_TAG, "onReceive: No usable EditorInfo/InputConnection");
+            }
+            return clip;
+        }
+        String[] editorInfoContentMimeTypes = icInfo.mEditorInfoContentMimeTypes;
+        if (!isClipMimeTypeSupported(editorInfoContentMimeTypes, clip.getDescription())) {
+            if (Log.isLoggable(LOG_TAG, Log.DEBUG)) {
+                Log.d(LOG_TAG,
+                        "onReceive: MIME type is not supported by the app's commitContent impl");
+            }
+            return clip;
+        }
+
+        if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+            Log.v(LOG_TAG, "onReceive: Trying to insert via IME: " + description);
+        }
+        ArrayList<ClipData.Item> remainingItems = new ArrayList<>(0);
+        for (int i = 0; i < clip.getItemCount(); i++) {
+            ClipData.Item item = clip.getItemAt(i);
+            Uri uri = item.getUri();
+            if (uri == null || !SCHEME_CONTENT.equals(uri.getScheme())) {
+                if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                    Log.v(LOG_TAG, "onReceive: No content URI in item: uri=" + uri);
+                }
+                remainingItems.add(item);
+                continue;
+            }
+            if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                Log.v(LOG_TAG, "onReceive: Calling commitContent: uri=" + uri);
+            }
+            InputContentInfo contentInfo = new InputContentInfo(uri, description);
+            if (!inputConnection.commitContent(contentInfo, 0, null)) {
+                if (Log.isLoggable(LOG_TAG, Log.VERBOSE)) {
+                    Log.v(LOG_TAG, "onReceive: Call to commitContent returned false: uri=" + uri);
+                }
+                remainingItems.add(item);
+            }
+        }
+        if (remainingItems.isEmpty()) {
+            return null;
+        }
+        return new ClipData(description, remainingItems);
+    }
+
+    private static boolean isClipMimeTypeSupported(@NonNull String[] supportedMimeTypes,
+            @NonNull ClipDescription description) {
+        for (String imeSupportedMimeType : supportedMimeTypes) {
+            if (description.hasMimeType(imeSupportedMimeType)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static boolean containsUri(@NonNull ClipData clip) {
+        for (int i = 0; i < clip.getItemCount(); i++) {
+            ClipData.Item item = clip.getItemAt(i);
+            if (item.getUri() != null) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static boolean containsOnlyText(@NonNull ClipData clip) {
+        ClipDescription description = clip.getDescription();
+        for (int i = 0; i < description.getMimeTypeCount(); i++) {
+            String mimeType = description.getMimeType(i);
+            if (!mimeType.startsWith("text/")) {
+                return false;
+            }
+        }
+        return true;
+    }
 }