Send iOS AutoFill text to the live view's remote field once

images/chromium-headful/client/src/components/video.vue

@@ -1531,7 +1531,7 @@
         // inputmode="none" raises the keyboard.
         this.overlayHeld = false
         setInputMode(this._overlay, keyboard.inputmode)
-        this.keepText = false
+        this.keepText = DEFAULT_KEYBOARD.keepText
         this.resetInputSentinel()
         this.onOverlayFocus()
         return
@@ -1653,13 +1653,23 @@
       this._overlay.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length)
     }
 
+    // AutoFill replaced the overlay's whole value, sentinel included
+    restoreInputSentinel() {
+      const value = this._overlay.value
+      if (value.startsWith(INPUT_SENTINEL)) return
+      this._overlay.value = INPUT_SENTINEL + value
+      this._overlay.setSelectionRange(this._overlay.value.length, this._overlay.value.length)
+    }
+
     // Soft keyboards that report keyCode 229 (Android) deliver text only through
     // input events, which the Guacamole keyboard does not handle.
     onOverlayInput(e: Event) {
       if (!this.is_touch_device) return
       if (this.keepText) {
         const kept = this.inputEnabled && this.typedText.edit(this._overlay.value)
-        if (!kept && !this.textInput.composing) this.resetInputSentinel()
+        if (this.textInput.composing) return
+        if (kept) this.restoreInputSentinel()
+        else this.resetInputSentinel()
         return
       }
       if (this.inputEnabled) {
@@ -1682,7 +1692,8 @@
     onCompositionEnd(e: CompositionEvent) {
       // the overlay's edits already sent a composition it keeps
       this.textInput.compositionEnd(e.data || '', this.inputEnabled && !this.keepText)
-      if (!this.keepText || !this.inputEnabled || !this.typedText.edit(this._overlay.value)) this.resetInputSentinel()
+      if (this.keepText && this.inputEnabled && this.typedText.edit(this._overlay.value)) this.restoreInputSentinel()
+      else this.resetInputSentinel()
     }
 
     focusOverlay() {

images/chromium-headful/client/src/utils/field-keyboard.ts

@@ -3,6 +3,8 @@
 // where type="email" or "number" would not), one-time-code AutoFill, and
 // autocorrect for plain text only. iOS reads everything but inputmode when the
 // overlay gains focus, so a change to an open keyboard only switches layouts.
+// Fields with AutoFill keep their text in the overlay too: iOS fills a field
+// by replacing its text, which can take more than one edit.
 
 import { RemoteField } from './remote-focus'
 
@@ -16,7 +18,7 @@ export interface KeyboardAttributes {
   autocapitalize: string
   spellcheck: boolean
   enterkeyhint: string | null
-  // the overlay keeps the typed text, which autocorrect needs to see
+  // the overlay keeps the typed text, which autocorrect and AutoFill replace
   keepText: boolean
 }
 
@@ -55,7 +57,7 @@ export const DEFAULT_KEYBOARD: KeyboardAttributes = {
   autocapitalize: 'off',
   spellcheck: false,
   enterkeyhint: null,
-  keepText: false,
+  keepText: true,
 }
 
 function tokens(field: RemoteField) {
@@ -101,15 +103,16 @@ export function keyboardFor(field: RemoteField | null): KeyboardAttributes {
         ...exact,
         inputmode: pageMode || (field.type === 'number' || field.type === 'tel' ? 'numeric' : null),
         autocomplete: 'one-time-code',
+        keepText: true,
       }
     case 'email':
-      return { ...exact, inputmode: 'email', autocomplete: 'email' }
+      return { ...exact, inputmode: 'email', autocomplete: 'email', keepText: true }
     case 'username':
       return { ...exact, inputmode: pageMode, autocomplete: 'off' }
     case 'url':
       return { ...exact, inputmode: 'url', autocomplete: 'off' }
     case 'tel':
-      return { ...exact, inputmode: 'tel', autocomplete: 'tel' }
+      return { ...exact, inputmode: 'tel', autocomplete: 'tel', keepText: true }
     case 'number':
       return { ...exact, inputmode: pageMode || 'decimal', autocomplete: 'off' }
   }
@@ -128,7 +131,7 @@ export function keyboardFor(field: RemoteField | null): KeyboardAttributes {
     autocapitalize,
     spellcheck: field.spellcheck !== 'false',
     enterkeyhint,
-    keepText: autocorrect === 'on',
+    keepText: autocorrect === 'on' || !!contact,
   }
 }
 

images/chromium-headful/client/src/utils/typed-text.ts

@@ -4,7 +4,8 @@
 // instead of sending keys, and Android keyboards compose a word and rewrite
 // it. Each change to the overlay is diffed against what was sent and goes to
 // the remote as backspaces and text, so the text before the remote caret
-// keeps matching the overlay.
+// keeps matching the overlay. AutoFill replaces the field's text the same
+// way, so text it fills in more than one edit is sent once.
 
 import { XK_BACKSPACE, textDiff } from './text-input'
 
@@ -45,17 +46,20 @@ export class TypedText {
 
   // The overlay's value after an edit. Returns false when the overlay should
   // be reset: its sentinel was deleted, which stands for text typed before
-  // the overlay last started over, or it holds too much.
+  // the overlay last started over, or it holds too much. AutoFill can replace
+  // the whole value, sentinel included, which keeps the text before the
+  // overlay, so the caller puts the sentinel back instead.
   edit(value: string) {
     const kept = value.startsWith(this.sentinel)
+    const replaced = !kept && value !== ''
     const next = kept ? value.slice(this.sentinel.length) : value
     const { deletes, insert } = textDiff(this.text, next)
     this.editing = true
     // the sentinel stands for one character before the overlay's text
-    for (let i = 0; i < deletes + (kept ? 0 : 1); i++) this.out.backspace()
+    for (let i = 0; i < deletes + (kept || replaced ? 0 : 1); i++) this.out.backspace()
     if (insert) this.out.type(insert)
     this.editing = false
     this.text = next
-    return kept && next.length <= MAX_TEXT
+    return (kept || replaced) && next.length <= MAX_TEXT
   }
 }

images/chromium-headful/client/tests/field-keyboard.test.ts

@@ -58,11 +58,30 @@ describe('field keyboard', () => {
         autocorrect: 'off',
         autocapitalize: 'off',
         spellcheck: false,
-        keepText: false,
       })
     }
   })
 
+  test('fields with AutoFill keep their text, so a fill that takes several edits is sent once', () => {
+    for (const field of [
+      { type: 'email' },
+      { type: 'tel' },
+      { type: 'text', autocomplete: 'one-time-code' },
+      { type: 'text', autocomplete: 'given-name', autocorrect: 'off' },
+    ]) {
+      expect(keyboardFor(field).keepText).toBe(true)
+    }
+    for (const field of [
+      { type: 'password' },
+      { type: 'text', autocomplete: 'username' },
+      { type: 'url' },
+      { type: 'number' },
+      { type: 'text', autocorrect: 'off' },
+    ]) {
+      expect(keyboardFor(field).keepText).toBe(false)
+    }
+  })
+
   test('chooses the keyboard layout', () => {
     expect(keyboardFor({ type: 'email' }).inputmode).toBe('email')
     expect(keyboardFor({ type: 'tel' }).inputmode).toBe('tel')
@@ -88,6 +107,8 @@ describe('field keyboard', () => {
 
   test('without a field the keyboard is the default one', () => {
     expect(keyboardFor(null)).toBe(DEFAULT_KEYBOARD)
+    // it offers codes from Messages, so it keeps its text too
+    expect(DEFAULT_KEYBOARD.keepText).toBe(true)
   })
 })
 

images/chromium-headful/client/tests/typed-text.test.ts

@@ -77,6 +77,29 @@ describe('typed text', () => {
     expect(t.remote()).toBe('a')
   })
 
+  test('AutoFill replacing the sentinel with its text sends the text once', () => {
+    const t = setup()
+    expect(t.typed.edit('me@example.com')).toBe(true)
+    // iOS fills again after the caller puts the sentinel back
+    expect(t.typed.edit('me@example.com')).toBe(true)
+    expect(t.typed.edit(SENTINEL + 'me@example.com')).toBe(true)
+    expect(t.remote()).toBe('me@example.com')
+  })
+
+  test('AutoFill completing typed text replaces it', () => {
+    const t = setup()
+    t.edit('m', 'me')
+    t.typed.edit('me@example.com')
+    expect(t.remote()).toBe('me@example.com')
+  })
+
+  test('AutoFill that inserts a space before replacing the text sends the text once', () => {
+    const t = setup()
+    t.edit(' ')
+    t.typed.edit('me@example.com')
+    expect(t.remote()).toBe('me@example.com')
+  })
+
   test('backspace removes what was typed', () => {
     const t = setup()
     t.edit('abc', 'ab')

images/chromium-headful/client/tests/video-focus.test.ts

@@ -8,6 +8,7 @@ import { FocusKeyboard } from '../src/utils/focus-keyboard'
 import { RemoteField } from '../src/utils/remote-focus'
 import { SoftKeyboard } from '../src/utils/soft-keyboard'
 import { TapKeyboard } from '../src/utils/tap-keyboard'
+import { TextInput, XK_BACKSPACE, XK_SHIFT_L } from '../src/utils/text-input'
 import { TypedText } from '../src/utils/typed-text'
 
 // Load the real component methods without mounting its media player or child components.
@@ -493,7 +494,7 @@ describe('raising the keyboard for a remote field', () => {
       spellcheck: 'false',
       inputmode: 'email',
     })
-    expect(t.ctx.keepText).toBe(false)
+    expect(t.ctx.keepText).toBe(true)
   })
 
   test('refocuses an open keyboard inside the tap when more than its layout changes', () => {
@@ -522,9 +523,10 @@ describe('raising the keyboard for a remote field', () => {
     expect(t.el.attrs.get('inputmode')).toBe('none')
     t.report(notes)
     expect(t.keyboard()).toBe(true)
-    // iOS keeps the attributes it was focused with
+    // iOS keeps the attributes it was focused with: the default keyboard's,
+    // which offer codes from Messages
     expect(t.el.attrs.get('autocorrect')).toBe('off')
-    expect(t.ctx.keepText).toBe(false)
+    expect(t.ctx.keepText).toBe(true)
   })
 })
 
@@ -559,6 +561,68 @@ describe('overlay text for autocorrect', () => {
   })
 })
 
+describe('AutoFill', () => {
+  // the remote field's text, typed through the real key path
+  const emailSetup = (open = (t: ReturnType<typeof phone>) => t.tap({ x: 50, y: 140 })) => {
+    const t = phone()
+    let remote = ''
+    const textInput = new TextInput(
+      (event, key) => {
+        if (event !== 'keydown' || key === XK_SHIFT_L) return
+        remote = key === XK_BACKSPACE ? remote.slice(0, -1) : remote + String.fromCodePoint(key & 0x00ffffff)
+      },
+      () => true,
+    )
+    t.ctx.textInput = textInput
+    t.ctx.typedText = new TypedText(' ', {
+      type: (s) => textInput.type(s),
+      backspace: () => textInput.key(XK_BACKSPACE),
+    })
+    open(t)
+    // iOS changes the overlay's value, then fires input
+    const fill = (inputType: string, value: string, data: string | null) => {
+      t.el.value = value
+      t.ctx.onOverlayInput({ inputType, data })
+    }
+    return { t, fill, remote: () => remote }
+  }
+  const EMAIL = 'me@example.com'
+
+  test('a fill that replaces the sentinel types the email once', () => {
+    const { t, fill, remote } = emailSetup()
+    fill('insertReplacementText', EMAIL, EMAIL)
+    expect(remote()).toBe(EMAIL)
+    expect(t.el.value).toBe(' ' + EMAIL)
+  })
+
+  test('a fill iOS repeats after the overlay changes under it types the email once', () => {
+    const { fill, remote } = emailSetup()
+    for (let i = 0; i < 3; i++) fill('insertReplacementText', EMAIL, EMAIL)
+    expect(remote()).toBe(EMAIL)
+  })
+
+  test('a fill that inserts a space, then replaces the text, types the email once', () => {
+    const { fill, remote } = emailSetup()
+    fill('insertText', '  ', ' ')
+    fill('insertText', EMAIL, EMAIL)
+    expect(remote()).toBe(EMAIL)
+  })
+
+  test('a code from Messages on the keyboard opened without a field is typed once', () => {
+    const { t, fill, remote } = emailSetup((t) => t.ctx.focusForTyping(null))
+    expect(t.el.attrs.get('autocomplete')).toBe('one-time-code')
+    for (let i = 0; i < 2; i++) fill('insertReplacementText', '123456', '123456')
+    expect(remote()).toBe('123456')
+  })
+
+  test('a fill after the first typed letters completes them', () => {
+    const { fill, remote } = emailSetup()
+    fill('insertText', ' m', 'm')
+    fill('insertReplacementText', ' ' + EMAIL, EMAIL)
+    expect(remote()).toBe(EMAIL)
+  })
+})
+
 describe('taps with the focus stream', () => {
   const tapContext = (live: { live: boolean; inViewport: boolean; field: unknown } | null) => {
     const calls: string[] = []