From a647f8a4c7934815813c6e48e239f298b34759d2 Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:56:28 +0000 Subject: [PATCH 1/2] Send iOS AutoFill text to the live view's remote field once --- .../client/src/components/video.vue | 15 ++++- .../client/src/utils/field-keyboard.ts | 11 ++-- .../client/src/utils/typed-text.ts | 12 ++-- .../client/tests/field-keyboard.test.ts | 21 ++++++- .../client/tests/typed-text.test.ts | 23 ++++++++ .../client/tests/video-focus.test.ts | 58 ++++++++++++++++++- 6 files changed, 128 insertions(+), 12 deletions(-) diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue index 461d0704..9321b526 100644 --- a/images/chromium-headful/client/src/components/video.vue +++ b/images/chromium-headful/client/src/components/video.vue @@ -1653,13 +1653,23 @@ this._overlay.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length) } + // AutoFill replaced the overlay's whole value, sentinel included + restoreInputSentinel() { + const value = this._overlay.value + if (value.startsWith(INPUT_SENTINEL)) return + this._overlay.value = INPUT_SENTINEL + value + this._overlay.setSelectionRange(this._overlay.value.length, this._overlay.value.length) + } + // Soft keyboards that report keyCode 229 (Android) deliver text only through // input events, which the Guacamole keyboard does not handle. onOverlayInput(e: Event) { if (!this.is_touch_device) return if (this.keepText) { const kept = this.inputEnabled && this.typedText.edit(this._overlay.value) - if (!kept && !this.textInput.composing) this.resetInputSentinel() + if (this.textInput.composing) return + if (kept) this.restoreInputSentinel() + else this.resetInputSentinel() return } if (this.inputEnabled) { @@ -1682,7 +1692,8 @@ onCompositionEnd(e: CompositionEvent) { // the overlay's edits already sent a composition it keeps this.textInput.compositionEnd(e.data || '', this.inputEnabled && !this.keepText) - if (!this.keepText || !this.inputEnabled || !this.typedText.edit(this._overlay.value)) this.resetInputSentinel() + if (this.keepText && this.inputEnabled && this.typedText.edit(this._overlay.value)) this.restoreInputSentinel() + else this.resetInputSentinel() } focusOverlay() { diff --git a/images/chromium-headful/client/src/utils/field-keyboard.ts b/images/chromium-headful/client/src/utils/field-keyboard.ts index 3574034b..1378de0e 100644 --- a/images/chromium-headful/client/src/utils/field-keyboard.ts +++ b/images/chromium-headful/client/src/utils/field-keyboard.ts @@ -3,6 +3,8 @@ // where type="email" or "number" would not), one-time-code AutoFill, and // autocorrect for plain text only. iOS reads everything but inputmode when the // overlay gains focus, so a change to an open keyboard only switches layouts. +// Fields with AutoFill keep their text in the overlay too: iOS fills a field +// by replacing its text, which can take more than one edit. import { RemoteField } from './remote-focus' @@ -16,7 +18,7 @@ export interface KeyboardAttributes { autocapitalize: string spellcheck: boolean enterkeyhint: string | null - // the overlay keeps the typed text, which autocorrect needs to see + // the overlay keeps the typed text, which autocorrect and AutoFill replace keepText: boolean } @@ -101,15 +103,16 @@ export function keyboardFor(field: RemoteField | null): KeyboardAttributes { ...exact, inputmode: pageMode || (field.type === 'number' || field.type === 'tel' ? 'numeric' : null), autocomplete: 'one-time-code', + keepText: true, } case 'email': - return { ...exact, inputmode: 'email', autocomplete: 'email' } + return { ...exact, inputmode: 'email', autocomplete: 'email', keepText: true } case 'username': return { ...exact, inputmode: pageMode, autocomplete: 'off' } case 'url': return { ...exact, inputmode: 'url', autocomplete: 'off' } case 'tel': - return { ...exact, inputmode: 'tel', autocomplete: 'tel' } + return { ...exact, inputmode: 'tel', autocomplete: 'tel', keepText: true } case 'number': return { ...exact, inputmode: pageMode || 'decimal', autocomplete: 'off' } } @@ -128,7 +131,7 @@ export function keyboardFor(field: RemoteField | null): KeyboardAttributes { autocapitalize, spellcheck: field.spellcheck !== 'false', enterkeyhint, - keepText: autocorrect === 'on', + keepText: autocorrect === 'on' || !!contact, } } diff --git a/images/chromium-headful/client/src/utils/typed-text.ts b/images/chromium-headful/client/src/utils/typed-text.ts index 0c1f0ff4..e5acb834 100644 --- a/images/chromium-headful/client/src/utils/typed-text.ts +++ b/images/chromium-headful/client/src/utils/typed-text.ts @@ -4,7 +4,8 @@ // instead of sending keys, and Android keyboards compose a word and rewrite // it. Each change to the overlay is diffed against what was sent and goes to // the remote as backspaces and text, so the text before the remote caret -// keeps matching the overlay. +// keeps matching the overlay. AutoFill replaces the field's text the same +// way, so text it fills in more than one edit is sent once. import { XK_BACKSPACE, textDiff } from './text-input' @@ -45,17 +46,20 @@ export class TypedText { // The overlay's value after an edit. Returns false when the overlay should // be reset: its sentinel was deleted, which stands for text typed before - // the overlay last started over, or it holds too much. + // the overlay last started over, or it holds too much. AutoFill can replace + // the whole value, sentinel included, which keeps the text before the + // overlay, so the caller puts the sentinel back instead. edit(value: string) { const kept = value.startsWith(this.sentinel) + const replaced = !kept && value !== '' const next = kept ? value.slice(this.sentinel.length) : value const { deletes, insert } = textDiff(this.text, next) this.editing = true // the sentinel stands for one character before the overlay's text - for (let i = 0; i < deletes + (kept ? 0 : 1); i++) this.out.backspace() + for (let i = 0; i < deletes + (kept || replaced ? 0 : 1); i++) this.out.backspace() if (insert) this.out.type(insert) this.editing = false this.text = next - return kept && next.length <= MAX_TEXT + return (kept || replaced) && next.length <= MAX_TEXT } } diff --git a/images/chromium-headful/client/tests/field-keyboard.test.ts b/images/chromium-headful/client/tests/field-keyboard.test.ts index 1d554225..372a77b0 100644 --- a/images/chromium-headful/client/tests/field-keyboard.test.ts +++ b/images/chromium-headful/client/tests/field-keyboard.test.ts @@ -58,11 +58,30 @@ describe('field keyboard', () => { autocorrect: 'off', autocapitalize: 'off', spellcheck: false, - keepText: false, }) } }) + test('fields with AutoFill keep their text, so a fill that takes several edits is sent once', () => { + for (const field of [ + { type: 'email' }, + { type: 'tel' }, + { type: 'text', autocomplete: 'one-time-code' }, + { type: 'text', autocomplete: 'given-name', autocorrect: 'off' }, + ]) { + expect(keyboardFor(field).keepText).toBe(true) + } + for (const field of [ + { type: 'password' }, + { type: 'text', autocomplete: 'username' }, + { type: 'url' }, + { type: 'number' }, + { type: 'text', autocorrect: 'off' }, + ]) { + expect(keyboardFor(field).keepText).toBe(false) + } + }) + test('chooses the keyboard layout', () => { expect(keyboardFor({ type: 'email' }).inputmode).toBe('email') expect(keyboardFor({ type: 'tel' }).inputmode).toBe('tel') diff --git a/images/chromium-headful/client/tests/typed-text.test.ts b/images/chromium-headful/client/tests/typed-text.test.ts index 72ec88a1..e0da3d78 100644 --- a/images/chromium-headful/client/tests/typed-text.test.ts +++ b/images/chromium-headful/client/tests/typed-text.test.ts @@ -77,6 +77,29 @@ describe('typed text', () => { expect(t.remote()).toBe('a') }) + test('AutoFill replacing the sentinel with its text sends the text once', () => { + const t = setup() + expect(t.typed.edit('me@example.com')).toBe(true) + // iOS fills again after the caller puts the sentinel back + expect(t.typed.edit('me@example.com')).toBe(true) + expect(t.typed.edit(SENTINEL + 'me@example.com')).toBe(true) + expect(t.remote()).toBe('me@example.com') + }) + + test('AutoFill completing typed text replaces it', () => { + const t = setup() + t.edit('m', 'me') + t.typed.edit('me@example.com') + expect(t.remote()).toBe('me@example.com') + }) + + test('AutoFill that inserts a space before replacing the text sends the text once', () => { + const t = setup() + t.edit(' ') + t.typed.edit('me@example.com') + expect(t.remote()).toBe('me@example.com') + }) + test('backspace removes what was typed', () => { const t = setup() t.edit('abc', 'ab') diff --git a/images/chromium-headful/client/tests/video-focus.test.ts b/images/chromium-headful/client/tests/video-focus.test.ts index 376a515a..7f96819f 100644 --- a/images/chromium-headful/client/tests/video-focus.test.ts +++ b/images/chromium-headful/client/tests/video-focus.test.ts @@ -8,6 +8,7 @@ import { FocusKeyboard } from '../src/utils/focus-keyboard' import { RemoteField } from '../src/utils/remote-focus' import { SoftKeyboard } from '../src/utils/soft-keyboard' import { TapKeyboard } from '../src/utils/tap-keyboard' +import { TextInput, XK_BACKSPACE, XK_SHIFT_L } from '../src/utils/text-input' import { TypedText } from '../src/utils/typed-text' // Load the real component methods without mounting its media player or child components. @@ -493,7 +494,7 @@ describe('raising the keyboard for a remote field', () => { spellcheck: 'false', inputmode: 'email', }) - expect(t.ctx.keepText).toBe(false) + expect(t.ctx.keepText).toBe(true) }) test('refocuses an open keyboard inside the tap when more than its layout changes', () => { @@ -559,6 +560,61 @@ describe('overlay text for autocorrect', () => { }) }) +describe('AutoFill into an email field', () => { + // the remote field's text, typed through the real key path + const emailSetup = () => { + const t = phone() + let remote = '' + const textInput = new TextInput( + (event, key) => { + if (event !== 'keydown' || key === XK_SHIFT_L) return + remote = key === XK_BACKSPACE ? remote.slice(0, -1) : remote + String.fromCodePoint(key & 0x00ffffff) + }, + () => true, + ) + t.ctx.textInput = textInput + t.ctx.typedText = new TypedText(' ', { + type: (s) => textInput.type(s), + backspace: () => textInput.key(XK_BACKSPACE), + }) + t.tap({ x: 50, y: 140 }) + // iOS changes the overlay's value, then fires input + const fill = (inputType: string, value: string, data: string | null) => { + t.el.value = value + t.ctx.onOverlayInput({ inputType, data }) + } + return { t, fill, remote: () => remote } + } + const EMAIL = 'me@example.com' + + test('a fill that replaces the sentinel types the email once', () => { + const { t, fill, remote } = emailSetup() + fill('insertReplacementText', EMAIL, EMAIL) + expect(remote()).toBe(EMAIL) + expect(t.el.value).toBe(' ' + EMAIL) + }) + + test('a fill iOS repeats after the overlay changes under it types the email once', () => { + const { fill, remote } = emailSetup() + for (let i = 0; i < 3; i++) fill('insertReplacementText', EMAIL, EMAIL) + expect(remote()).toBe(EMAIL) + }) + + test('a fill that inserts a space, then replaces the text, types the email once', () => { + const { fill, remote } = emailSetup() + fill('insertText', ' ', ' ') + fill('insertText', EMAIL, EMAIL) + expect(remote()).toBe(EMAIL) + }) + + test('a fill after the first typed letters completes them', () => { + const { fill, remote } = emailSetup() + fill('insertText', ' m', 'm') + fill('insertReplacementText', ' ' + EMAIL, EMAIL) + expect(remote()).toBe(EMAIL) + }) +}) + describe('taps with the focus stream', () => { const tapContext = (live: { live: boolean; inViewport: boolean; field: unknown } | null) => { const calls: string[] = [] From 39e0c239bbb8c58685896248ba453cba396cec20 Mon Sep 17 00:00:00 2001 From: masnwilliams <43387599+masnwilliams@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:35:57 +0000 Subject: [PATCH 2/2] Keep the default keyboard's text, so a code from Messages is typed once --- .../client/src/components/video.vue | 2 +- .../client/src/utils/field-keyboard.ts | 2 +- .../client/tests/field-keyboard.test.ts | 2 ++ .../client/tests/video-focus.test.ts | 18 +++++++++++++----- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/images/chromium-headful/client/src/components/video.vue b/images/chromium-headful/client/src/components/video.vue index 9321b526..ff26ee83 100644 --- a/images/chromium-headful/client/src/components/video.vue +++ b/images/chromium-headful/client/src/components/video.vue @@ -1531,7 +1531,7 @@ // inputmode="none" raises the keyboard. this.overlayHeld = false setInputMode(this._overlay, keyboard.inputmode) - this.keepText = false + this.keepText = DEFAULT_KEYBOARD.keepText this.resetInputSentinel() this.onOverlayFocus() return diff --git a/images/chromium-headful/client/src/utils/field-keyboard.ts b/images/chromium-headful/client/src/utils/field-keyboard.ts index 1378de0e..6472fd0e 100644 --- a/images/chromium-headful/client/src/utils/field-keyboard.ts +++ b/images/chromium-headful/client/src/utils/field-keyboard.ts @@ -57,7 +57,7 @@ export const DEFAULT_KEYBOARD: KeyboardAttributes = { autocapitalize: 'off', spellcheck: false, enterkeyhint: null, - keepText: false, + keepText: true, } function tokens(field: RemoteField) { diff --git a/images/chromium-headful/client/tests/field-keyboard.test.ts b/images/chromium-headful/client/tests/field-keyboard.test.ts index 372a77b0..f875e61f 100644 --- a/images/chromium-headful/client/tests/field-keyboard.test.ts +++ b/images/chromium-headful/client/tests/field-keyboard.test.ts @@ -107,6 +107,8 @@ describe('field keyboard', () => { test('without a field the keyboard is the default one', () => { expect(keyboardFor(null)).toBe(DEFAULT_KEYBOARD) + // it offers codes from Messages, so it keeps its text too + expect(DEFAULT_KEYBOARD.keepText).toBe(true) }) }) diff --git a/images/chromium-headful/client/tests/video-focus.test.ts b/images/chromium-headful/client/tests/video-focus.test.ts index 7f96819f..cfe1c81b 100644 --- a/images/chromium-headful/client/tests/video-focus.test.ts +++ b/images/chromium-headful/client/tests/video-focus.test.ts @@ -523,9 +523,10 @@ describe('raising the keyboard for a remote field', () => { expect(t.el.attrs.get('inputmode')).toBe('none') t.report(notes) expect(t.keyboard()).toBe(true) - // iOS keeps the attributes it was focused with + // iOS keeps the attributes it was focused with: the default keyboard's, + // which offer codes from Messages expect(t.el.attrs.get('autocorrect')).toBe('off') - expect(t.ctx.keepText).toBe(false) + expect(t.ctx.keepText).toBe(true) }) }) @@ -560,9 +561,9 @@ describe('overlay text for autocorrect', () => { }) }) -describe('AutoFill into an email field', () => { +describe('AutoFill', () => { // the remote field's text, typed through the real key path - const emailSetup = () => { + const emailSetup = (open = (t: ReturnType) => t.tap({ x: 50, y: 140 })) => { const t = phone() let remote = '' const textInput = new TextInput( @@ -577,7 +578,7 @@ describe('AutoFill into an email field', () => { type: (s) => textInput.type(s), backspace: () => textInput.key(XK_BACKSPACE), }) - t.tap({ x: 50, y: 140 }) + open(t) // iOS changes the overlay's value, then fires input const fill = (inputType: string, value: string, data: string | null) => { t.el.value = value @@ -607,6 +608,13 @@ describe('AutoFill into an email field', () => { expect(remote()).toBe(EMAIL) }) + test('a code from Messages on the keyboard opened without a field is typed once', () => { + const { t, fill, remote } = emailSetup((t) => t.ctx.focusForTyping(null)) + expect(t.el.attrs.get('autocomplete')).toBe('one-time-code') + for (let i = 0; i < 2; i++) fill('insertReplacementText', '123456', '123456') + expect(remote()).toBe('123456') + }) + test('a fill after the first typed letters completes them', () => { const { fill, remote } = emailSetup() fill('insertText', ' m', 'm')