diff options
| author | TreeHugger Robot <treehugger-gerrit@google.com> | 2022-01-13 16:54:42 +0000 |
|---|---|---|
| committer | Android (Google) Code Review <android-gerrit@google.com> | 2022-01-13 16:54:42 +0000 |
| commit | 01d195c67c724816ef7ad844ebe1743fd9e9cd50 (patch) | |
| tree | 7ae275f1b012a921b5f5be91e534b2ac16448da3 /core/java | |
| parent | 16e5a4a46a1b59ebaa2055b3a608429877312b75 (diff) | |
| parent | 94324080afd9e045aefeb01cf550e89da1bfbb2e (diff) | |
Merge "Revert "Revert "Scribe in IMF: startStylusHandwriting & lifecycl...""
Diffstat (limited to 'core/java')
9 files changed, 356 insertions, 7 deletions
diff --git a/core/java/android/inputmethodservice/IInputMethodWrapper.java b/core/java/android/inputmethodservice/IInputMethodWrapper.java index e30594fb9da7..6f15588c0724 100644 --- a/core/java/android/inputmethodservice/IInputMethodWrapper.java +++ b/core/java/android/inputmethodservice/IInputMethodWrapper.java @@ -18,6 +18,7 @@ package android.inputmethodservice; import android.annotation.BinderThread; import android.annotation.MainThread; +import android.annotation.NonNull; import android.annotation.Nullable; import android.compat.annotation.UnsupportedAppUsage; import android.content.Context; @@ -29,6 +30,7 @@ import android.os.RemoteException; import android.os.ResultReceiver; import android.util.Log; import android.view.InputChannel; +import android.view.MotionEvent; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputConnection; @@ -50,6 +52,7 @@ import com.android.internal.view.InlineSuggestionsRequestInfo; import java.io.FileDescriptor; import java.io.PrintWriter; import java.lang.ref.WeakReference; +import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -74,6 +77,8 @@ class IInputMethodWrapper extends IInputMethod.Stub private static final int DO_HIDE_SOFT_INPUT = 70; private static final int DO_CHANGE_INPUTMETHOD_SUBTYPE = 80; private static final int DO_CREATE_INLINE_SUGGESTIONS_REQUEST = 90; + private static final int DO_CAN_START_STYLUS_HANDWRITING = 100; + private static final int DO_START_STYLUS_HANDWRITING = 110; final WeakReference<InputMethodServiceInternal> mTarget; final Context mContext; @@ -169,7 +174,8 @@ class IInputMethodWrapper extends IInputMethod.Stub SomeArgs args = (SomeArgs) msg.obj; try { inputMethod.initializeInternal((IBinder) args.arg1, - (IInputMethodPrivilegedOperations) args.arg2, msg.arg1); + (IInputMethodPrivilegedOperations) args.arg2, msg.arg1, + (boolean) args.arg3); } finally { args.recycle(); } @@ -229,13 +235,25 @@ class IInputMethodWrapper extends IInputMethod.Stub case DO_CHANGE_INPUTMETHOD_SUBTYPE: inputMethod.changeInputMethodSubtype((InputMethodSubtype)msg.obj); return; - case DO_CREATE_INLINE_SUGGESTIONS_REQUEST: + case DO_CREATE_INLINE_SUGGESTIONS_REQUEST: { final SomeArgs args = (SomeArgs) msg.obj; inputMethod.onCreateInlineSuggestionsRequest( (InlineSuggestionsRequestInfo) args.arg1, (IInlineSuggestionsRequestCallback) args.arg2); args.recycle(); return; + } + case DO_CAN_START_STYLUS_HANDWRITING: { + inputMethod.canStartStylusHandwriting(msg.arg1); + return; + } + case DO_START_STYLUS_HANDWRITING: { + final SomeArgs args = (SomeArgs) msg.obj; + inputMethod.startStylusHandwriting((InputChannel) args.arg1, + (List<MotionEvent>) args.arg2); + args.recycle(); + return; + } } Log.w(TAG, "Unhandled message code: " + msg.what); @@ -272,9 +290,10 @@ class IInputMethodWrapper extends IInputMethod.Stub @BinderThread @Override public void initializeInternal(IBinder token, IInputMethodPrivilegedOperations privOps, - int configChanges) { + int configChanges, boolean stylusHwSupported) { mCaller.executeOrSendMessage( - mCaller.obtainMessageIOO(DO_INITIALIZE_INTERNAL, configChanges, token, privOps)); + mCaller.obtainMessageIOOO( + DO_INITIALIZE_INTERNAL, configChanges, token, privOps, stylusHwSupported)); } @BinderThread @@ -383,4 +402,21 @@ class IInputMethodWrapper extends IInputMethod.Stub mCaller.executeOrSendMessage(mCaller.obtainMessageO(DO_CHANGE_INPUTMETHOD_SUBTYPE, subtype)); } + + @BinderThread + @Override + public void canStartStylusHandwriting(int requestId) + throws RemoteException { + mCaller.executeOrSendMessage( + mCaller.obtainMessageI(DO_CAN_START_STYLUS_HANDWRITING, requestId)); + } + + @BinderThread + @Override + public void startStylusHandwriting(@NonNull InputChannel channel, + @Nullable List<MotionEvent> stylusEvents) + throws RemoteException { + mCaller.executeOrSendMessage( + mCaller.obtainMessageOO(DO_START_STYLUS_HANDWRITING, channel, stylusEvents)); + } } diff --git a/core/java/android/inputmethodservice/InkWindow.java b/core/java/android/inputmethodservice/InkWindow.java new file mode 100644 index 000000000000..e11d63562ce3 --- /dev/null +++ b/core/java/android/inputmethodservice/InkWindow.java @@ -0,0 +1,88 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.inputmethodservice; + +import static android.view.WindowManager.LayoutParams; +import static android.view.WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN; +import static android.view.WindowManager.LayoutParams.FLAG_LAYOUT_NO_LIMITS; +import static android.view.WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE; +import static android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE; + +import android.annotation.NonNull; +import android.content.Context; +import android.os.IBinder; +import android.util.Slog; +import android.view.View; +import android.view.WindowManager; + +import com.android.internal.policy.PhoneWindow; + +/** + * Window of type {@code LayoutParams.TYPE_INPUT_METHOD_DIALOG} for drawing + * Handwriting Ink on screen. + * @hide + */ +final class InkWindow extends PhoneWindow { + + private final WindowManager mWindowManager; + + public InkWindow(@NonNull Context context) { + super(context); + + setType(LayoutParams.TYPE_INPUT_METHOD); + final LayoutParams attrs = getAttributes(); + attrs.layoutInDisplayCutoutMode = LayoutParams.LAYOUT_IN_DISPLAY_CUTOUT_MODE_ALWAYS; + attrs.setFitInsetsTypes(0); + setAttributes(attrs); + // Ink window is not touchable with finger. + addFlags(FLAG_LAYOUT_IN_SCREEN | FLAG_LAYOUT_NO_LIMITS | FLAG_NOT_TOUCHABLE + | FLAG_NOT_FOCUSABLE); + setBackgroundDrawableResource(android.R.color.transparent); + setLayout(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT); + mWindowManager = context.getSystemService(WindowManager.class); + } + + /** + * Method to show InkWindow on screen. + * Emulates internal behavior similar to Dialog.show(). + */ + void show() { + if (getDecorView() == null) { + Slog.i(InputMethodService.TAG, "DecorView is not set for InkWindow. show() failed."); + return; + } + getDecorView().setVisibility(View.VISIBLE); + mWindowManager.addView(getDecorView(), getAttributes()); + } + + /** + * Method to hide InkWindow from screen. + * Emulates internal behavior similar to Dialog.hide(). + * @param remove set {@code true} to remove InkWindow surface completely. + */ + void hide(boolean remove) { + if (getDecorView() != null) { + getDecorView().setVisibility(remove ? View.GONE : View.INVISIBLE); + } + } + + void setToken(@NonNull IBinder token) { + WindowManager.LayoutParams lp = getAttributes(); + lp.token = token; + setAttributes(lp); + } +} diff --git a/core/java/android/inputmethodservice/InputMethodService.java b/core/java/android/inputmethodservice/InputMethodService.java index afcb6fccb137..09d50850788b 100644 --- a/core/java/android/inputmethodservice/InputMethodService.java +++ b/core/java/android/inputmethodservice/InputMethodService.java @@ -96,6 +96,7 @@ import android.util.PrintWriterPrinter; import android.util.Printer; import android.util.proto.ProtoOutputStream; import android.view.Gravity; +import android.view.InputChannel; import android.view.KeyCharacterMap; import android.view.KeyEvent; import android.view.LayoutInflater; @@ -121,6 +122,7 @@ import android.view.inputmethod.InputConnection; import android.view.inputmethod.InputContentInfo; import android.view.inputmethod.InputMethod; import android.view.inputmethod.InputMethodEditorTraceProto.InputMethodServiceTraceProto; +import android.view.inputmethod.InputMethodInfo; import android.view.inputmethod.InputMethodManager; import android.view.inputmethod.InputMethodSubtype; import android.widget.FrameLayout; @@ -144,6 +146,7 @@ import java.io.PrintWriter; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -560,11 +563,15 @@ public class InputMethodService extends AbstractInputMethodService { private boolean mAutomotiveHideNavBarForKeyboard; private boolean mIsAutomotive; + private boolean mHandwritingStarted; private Handler mHandler; private boolean mImeSurfaceScheduledForRemoval; private ImsConfigurationTracker mConfigTracker = new ImsConfigurationTracker(); private boolean mDestroyed; + /** Stylus handwriting Ink window. */ + private InkWindow mInkWindow; + /** * An opaque {@link Binder} token of window requesting {@link InputMethodImpl#showSoftInput} * The original app window token is passed from client app window. @@ -639,7 +646,8 @@ public class InputMethodService extends AbstractInputMethodService { @MainThread @Override public final void initializeInternal(@NonNull IBinder token, - IInputMethodPrivilegedOperations privilegedOperations, int configChanges) { + IInputMethodPrivilegedOperations privilegedOperations, int configChanges, + boolean stylusHwSupported) { if (mDestroyed) { Log.i(TAG, "The InputMethodService has already onDestroyed()." + "Ignore the initialization."); @@ -649,6 +657,9 @@ public class InputMethodService extends AbstractInputMethodService { mConfigTracker.onInitialize(configChanges); mPrivOps.set(privilegedOperations); InputMethodPrivilegedOperationsRegistry.put(token, mPrivOps); + if (stylusHwSupported) { + mInkWindow = new InkWindow(mWindow.getContext()); + } attachToken(token); Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } @@ -681,6 +692,9 @@ public class InputMethodService extends AbstractInputMethodService { attachToWindowToken(token); mToken = token; mWindow.setToken(token); + if (mInkWindow != null) { + mInkWindow.setToken(token); + } } /** @@ -864,6 +878,49 @@ public class InputMethodService extends AbstractInputMethodService { /** * {@inheritDoc} + * @hide + */ + @Override + public void canStartStylusHandwriting(int requestId) { + if (DEBUG) Log.v(TAG, "canStartStylusHandwriting()"); + if (mHandwritingStarted) { + Log.d(TAG, "There is an ongoing Handwriting session. ignoring."); + return; + } + if (!mInputStarted) { + Log.d(TAG, "Input should have started before starting Stylus handwriting."); + return; + } + if (onStartStylusHandwriting()) { + mPrivOps.onStylusHandwritingReady(requestId); + } else { + Log.i(TAG, "IME is not ready. Can't start Stylus Handwriting"); + } + } + + /** + * {@inheritDoc} + * @hide + */ + @MainThread + @Override + public void startStylusHandwriting( + @NonNull InputChannel channel, @Nullable List<MotionEvent> stylusEvents) { + if (DEBUG) Log.v(TAG, "startStylusHandwriting()"); + if (mHandwritingStarted) { + return; + } + + mHandwritingStarted = true; + mShowInputRequested = false; + + mInkWindow.show(); + // TODO: deliver previous @param stylusEvents + // TODO: create spy receiver for @param channel + } + + /** + * {@inheritDoc} */ @MainThread @Override @@ -2233,6 +2290,77 @@ public class InputMethodService extends AbstractInputMethodService { } /** + * Called when an app requests stylus handwriting + * {@link InputMethodManager#startStylusHandwriting(View)}. + * + * This will always be preceded by {@link #onStartInput(EditorInfo, boolean)} for the + * {@link EditorInfo} and {@link InputConnection} for which stylus handwriting is being + * requested. + * + * If the IME supports handwriting for the current input, it should return {@code true}, + * ensure its inking views are attached to the {@link #getStylusHandwritingWindow()}, and handle + * stylus input received on the ink window via {@link #getCurrentInputConnection()}. + * @return {@code true} if IME can honor the request, {@code false} if IME cannot at this time. + */ + public boolean onStartStylusHandwriting() { + // Intentionally empty + return false; + } + + /** + * Called when the current stylus handwriting session was finished (either by the system or + * via {@link #finishStylusHandwriting()}. + * + * When this is called, the ink window has been made invisible, and the IME no longer + * intercepts handwriting-related {@code MotionEvent}s. + */ + public void onFinishStylusHandwriting() { + // Intentionally empty + } + + /** + * Returns the stylus handwriting inking window. + * IMEs supporting stylus input are expected to attach their inking views to this + * window (e.g. with {@link Window#setContentView(View)} )). Handwriting-related + * {@link MotionEvent}s are dispatched to the attached view hierarchy. + * + * Note: This returns {@code null} if IME doesn't support stylus handwriting + * i.e. if {@link InputMethodInfo#supportsStylusHandwriting()} is false. + * This method should be called after {@link #onStartStylusHandwriting()}. + * @see #onStartStylusHandwriting() + */ + @Nullable + public final Window getStylusHandwritingWindow() { + return mInkWindow; + } + + /** + * Finish the current stylus handwriting session. + * + * This dismisses the {@link #getStylusHandwritingWindow ink window} and stops intercepting + * stylus {@code MotionEvent}s. + * + * Note for IME developers: Call this method at any time to finish current handwriting session. + * Generally, this should be invoked after a short timeout, giving the user enough time + * to start the next stylus stroke, if any. + * + * Handwriting session will be finished by framework on next {@link #onFinishInput()}. + */ + public final void finishStylusHandwriting() { + if (DEBUG) Log.v(TAG, "finishStylusHandwriting()"); + if (mInkWindow == null) { + return; + } + if (!mHandwritingStarted) { + return; + } + + mHandwritingStarted = false; + mInkWindow.hide(false /* remove */); + onFinishStylusHandwriting(); + } + + /** * The system has decided that it may be time to show your input method. * This is called due to a corresponding call to your * {@link InputMethod#showSoftInput InputMethod.showSoftInput()} @@ -2501,6 +2629,9 @@ public class InputMethodService extends AbstractInputMethodService { mInputStarted = false; mStartedInputConnection = null; mCurCompletions = null; + if (mInkWindow != null) { + finishStylusHandwriting(); + } } void doStartInput(InputConnection ic, EditorInfo attribute, boolean restarting) { diff --git a/core/java/android/view/inputmethod/InputMethod.java b/core/java/android/view/inputmethod/InputMethod.java index 5b2068ff16cd..fda72d5ba966 100644 --- a/core/java/android/view/inputmethod/InputMethod.java +++ b/core/java/android/view/inputmethod/InputMethod.java @@ -26,12 +26,16 @@ import android.os.IBinder; import android.os.RemoteException; import android.os.ResultReceiver; import android.util.Log; +import android.view.InputChannel; +import android.view.MotionEvent; import android.view.View; import com.android.internal.inputmethod.IInputMethodPrivilegedOperations; import com.android.internal.view.IInlineSuggestionsRequestCallback; import com.android.internal.view.InlineSuggestionsRequestInfo; +import java.util.List; + /** * The InputMethod interface represents an input method which can generate key * events and text, such as digital, email addresses, CJK characters, other @@ -100,11 +104,13 @@ public interface InputMethod { * operations that are allowed only to the * current IME. * @param configChanges {@link InputMethodInfo#getConfigChanges()} declared by IME. + * @param stylusHwSupported {@link InputMethodInfo#supportsStylusHandwriting()} declared by IME. * @hide */ @MainThread default void initializeInternal(IBinder token, - IInputMethodPrivilegedOperations privilegedOperations, int configChanges) { + IInputMethodPrivilegedOperations privilegedOperations, int configChanges, + boolean stylusHwSupported) { attachToken(token); } @@ -384,4 +390,23 @@ public interface InputMethod { */ public void setCurrentHideInputToken(IBinder hideInputToken); + /** + * Checks if IME is ready to start stylus handwriting session. + * If yes, {@link #startStylusHandwriting(InputChannel, List)} is called. + * @param requestId + * @hide + */ + default void canStartStylusHandwriting(int requestId) { + // intentionally empty + } + + /** + * Start stylus handwriting session. + * @hide + */ + default void startStylusHandwriting( + @NonNull InputChannel channel, @Nullable List<MotionEvent> events) { + // intentionally empty + } + } diff --git a/core/java/android/view/inputmethod/InputMethodManager.java b/core/java/android/view/inputmethod/InputMethodManager.java index 3583cd4c6d8b..6fc246eb2514 100644 --- a/core/java/android/view/inputmethod/InputMethodManager.java +++ b/core/java/android/view/inputmethod/InputMethodManager.java @@ -1761,6 +1761,50 @@ public final class InputMethodManager { } /** + * Start stylus handwriting session. + * + * If supported by the current input method, a stylus handwriting session is started on the + * given View, capturing all stylus input and converting it to InputConnection commands. + * + * If handwriting mode is started successfully by the IME, any currently dispatched stylus + * pointers will be {@code android.view.MotionEvent#FLAG_CANCELED} cancelled. + * + * If Stylus handwriting mode is not supported or cannot be fulfilled for any reason by IME, + * request will be ignored and Stylus touch will continue as normal touch input. + * + * @param view the View for which stylus handwriting is requested. It and + * {@link View#hasWindowFocus its window} must be {@link View#hasFocus focused}. + */ + public void startStylusHandwriting(@NonNull View view) { + // Re-dispatch if there is a context mismatch. + final InputMethodManager fallbackImm = getFallbackInputMethodManagerIfNecessary(view); + if (fallbackImm != null) { + fallbackImm.startStylusHandwriting(view); + } + + checkFocus(); + synchronized (mH) { + if (view == null || !hasServedByInputMethodLocked(view)) { + Log.w(TAG, + "Ignoring startStylusHandwriting() as view=" + view + " is not served."); + return; + } + if (view.getViewRootImpl() != mCurRootView) { + Log.w(TAG, "Ignoring startStylusHandwriting: View's window does not have focus."); + return; + } + + try { + mService.startStylusHandwriting(mClient); + // TODO(b/210039666): do we need any extra work for supporting non-native + // UI toolkits? + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } + } + + /** * This method toggles the input method window display. * If the input window is already displayed, it gets hidden. * If not the input window will be displayed. diff --git a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl index 9d0f209d8b2d..08bc8c7fa339 100644 --- a/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl +++ b/core/java/com/android/internal/inputmethod/IInputMethodPrivilegedOperations.aidl @@ -42,4 +42,5 @@ oneway interface IInputMethodPrivilegedOperations { void shouldOfferSwitchingToNextInputMethod(in AndroidFuture future /* T=Boolean */); void notifyUserActionAsync(); void applyImeVisibilityAsync(IBinder showOrHideInputToken, boolean setVisible); + void onStylusHandwritingReady(int requestId); } diff --git a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java index d4cc376385b8..7ebcc88b593b 100644 --- a/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java +++ b/core/java/com/android/internal/inputmethod/InputMethodPrivilegedOperations.java @@ -394,4 +394,20 @@ public final class InputMethodPrivilegedOperations { throw e.rethrowFromSystemServer(); } } + + /** + * Calls {@link IInputMethodPrivilegedOperations#onStylusHandwritingReady()} + */ + @AnyThread + public void onStylusHandwritingReady(int requestId) { + final IInputMethodPrivilegedOperations ops = mOps.getAndWarnIfNull(); + if (ops == null) { + return; + } + try { + ops.onStylusHandwritingReady(requestId); + } catch (RemoteException e) { + throw e.rethrowFromSystemServer(); + } + } } diff --git a/core/java/com/android/internal/view/IInputMethod.aidl b/core/java/com/android/internal/view/IInputMethod.aidl index 139660a29ede..402fa64036a0 100644 --- a/core/java/com/android/internal/view/IInputMethod.aidl +++ b/core/java/com/android/internal/view/IInputMethod.aidl @@ -19,6 +19,7 @@ package com.android.internal.view; import android.os.IBinder; import android.os.ResultReceiver; import android.view.InputChannel; +import android.view.MotionEvent; import android.view.inputmethod.EditorInfo; import android.view.inputmethod.InputBinding; import android.view.inputmethod.InputMethodSubtype; @@ -36,7 +37,7 @@ import com.android.internal.view.InlineSuggestionsRequestInfo; */ oneway interface IInputMethod { void initializeInternal(IBinder token, IInputMethodPrivilegedOperations privOps, - int configChanges); + int configChanges, boolean stylusHwSupported); void onCreateInlineSuggestionsRequest(in InlineSuggestionsRequestInfo requestInfo, in IInlineSuggestionsRequestCallback cb); @@ -59,4 +60,8 @@ oneway interface IInputMethod { void hideSoftInput(in IBinder hideInputToken, int flags, in ResultReceiver resultReceiver); void changeInputMethodSubtype(in InputMethodSubtype subtype); + + void canStartStylusHandwriting(int requestId); + + void startStylusHandwriting(in InputChannel channel, in List<MotionEvent> events); } diff --git a/core/java/com/android/internal/view/IInputMethodManager.aidl b/core/java/com/android/internal/view/IInputMethodManager.aidl index 2dc7c42c95a9..0df3e870b80c 100644 --- a/core/java/com/android/internal/view/IInputMethodManager.aidl +++ b/core/java/com/android/internal/view/IInputMethodManager.aidl @@ -81,4 +81,7 @@ interface IInputMethodManager { void startImeTrace(); // Stops an ime trace. void stopImeTrace(); + + /** Start Stylus handwriting session **/ + void startStylusHandwriting(in IInputMethodClient client); } |
