diff --git a/changelog.d/1405.fixed.md b/changelog.d/1405.fixed.md new file mode 100644 index 00000000..6724926d --- /dev/null +++ b/changelog.d/1405.fixed.md @@ -0,0 +1 @@ +Voice mode no longer reads a reply's framing aloud: the host header, a reply's delegation trace (`Chief → Product`) and the untrusted-fence markers a stored remote line carries are stripped along the same text chain a row preview reads, so `speechSynthesis` says the words and nothing else. diff --git a/core/client.js b/core/client.js index a1a21ab7..a1dca244 100644 --- a/core/client.js +++ b/core/client.js @@ -2532,6 +2532,9 @@ function dropKnownTrace(line2, names) { return /^[\s,,.。::]*$/.test(rest) ? "" : rest; } var botNames = (roster) => [...roster?.bots ?? [], ...roster?.remoteBots ?? [], ...roster?.agents ?? []].map((entry) => entry?.name).filter((name) => typeof name === "string" && name !== ""); +function speechOf(text, names = []) { + return readableLine(text, names).replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*_`#>|~-]+/g, " ").replace(/\s+/g, " ").trim(); +} var formatTokens = (count) => count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count); function usageOf(list, ids) { let input = 0; @@ -14144,7 +14147,6 @@ function ExchangeDialog({ pair, roster, actions }) { // src/client/voice.js var import_react33 = require("react"); -var plainSpeech = (text) => stripHeader(text).replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*_`#>|~-]+/g, " ").replace(/\s+/g, " ").trim(); var VOICE_LABELS = { listening: "Listening", thinking: "Thinking", @@ -14333,7 +14335,7 @@ function VoiceMode({ sessionId, roster, actions, useSessions, useSessionStatus } const [count, text] = [lastReply.slice(0, lastReply.indexOf("|")), lastReply.slice(lastReply.indexOf("|") + 1)]; if (count === waiting.current || !text.trim()) return; waiting.current = null; - const spoken = plainSpeech(text); + const spoken = speechOf(text, botNames(roster)); setSaid(spoken); setHeard(""); if (!window.speechSynthesis || !spoken) { diff --git a/core/src/client/text.js b/core/src/client/text.js index f65d91fc..9db11330 100644 --- a/core/src/client/text.js +++ b/core/src/client/text.js @@ -332,6 +332,18 @@ function dropKnownTrace(line, names) { export const botNames = roster => [...(roster?.bots ?? []), ...(roster?.remoteBots ?? []), ...(roster?.agents ?? [])] .map(entry => entry?.name).filter(name => typeof name === 'string' && name !== '') +// What voice mode reads aloud: a reply's words, read the one way a row reads a stored line — header, +// fence and a known-name delegation trace dropped (#1395, #1408) — and then with the Markdown a speech +// synthesiser would otherwise say out loud (a code fence, a link's target, emphasis marks). +export function speechOf(text, names = []) { + return readableLine(text, names) + .replace(/```[\s\S]*?```/g, ' ') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/[*_`#>|~-]+/g, ' ') + .replace(/\s+/g, ' ') + .trim() +} + const formatTokens = count => (count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count)) // Same buckets as the shell's usage pill: every prompt-side billing bucket plus output. diff --git a/core/src/client/voice.js b/core/src/client/voice.js index feb1ad0f..869154d9 100644 --- a/core/src/client/voice.js +++ b/core/src/client/voice.js @@ -1,5 +1,5 @@ import { createElement as h, useEffect, useRef, useState } from 'react' -import { stripHeader } from './text.js' +import { botNames, speechOf } from './text.js' import { MicIcon, CloseIcon } from './icons.js' import { BotMark, RoomAvatar } from './mark.js' import { speechApi, useDictation } from './composer.js' @@ -12,7 +12,6 @@ import { openCapture, speechCatalog, transcribeWave, turnDetector } from './spee // again. With DSH's Voice Input plugin on, its recognizer hears the user: one // recording ends after a pause and is transcribed whole. Otherwise the browser's // own speech recognition does. The browser's speech synthesis reads the answer. -const plainSpeech = text => stripHeader(text).replace(/```[\s\S]*?```/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1').replace(/[*_`#>|~-]+/g, ' ').replace(/\s+/g, ' ').trim() const VOICE_LABELS = { listening: 'Listening', thinking: 'Thinking', speaking: 'Speaking', paused: 'Paused', setup: 'Voice input needs its speech model', unsupported: 'Voice chat is not available in this browser', @@ -184,7 +183,7 @@ export function VoiceMode({ sessionId, roster, actions, useSessions, useSessionS const [count, text] = [lastReply.slice(0, lastReply.indexOf('|')), lastReply.slice(lastReply.indexOf('|') + 1)] if (count === waiting.current || !text.trim()) return waiting.current = null - const spoken = plainSpeech(text) + const spoken = speechOf(text, botNames(roster)) setSaid(spoken) setHeard('') if (!window.speechSynthesis || !spoken) { setPhase('listening'); return } diff --git a/core/test/voice-speech.test.js b/core/test/voice-speech.test.js new file mode 100644 index 00000000..6e4e4f13 --- /dev/null +++ b/core/test/voice-speech.test.js @@ -0,0 +1,82 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' +import { fenceUntrusted } from '../src/host/delivery.js' + +// Voice mode reads a Bot's reply aloud, and a stored turn is framing plus words: the host's header, +// a reply's delegation trace and an untrusted fence are routing, not content. The speech text is read +// the one way a row reads a stored line — `readableLine(text, names)` in text.js — because +// `speechSynthesis` repeats what it is handed: a fence mark reaches the user's ears as +// ` `${header}\n${fenceUntrusted(name, body)}` + +const text = await build({ + entryPoints: [fileURLToPath(new URL('../src/client/text.js', import.meta.url))], + bundle: true, format: 'esm', platform: 'node', write: false, +}) +const { speechOf } = await import(`data:text/javascript;base64,${Buffer.from(text.outputFiles[0].text).toString('base64')}`) + +// The names a conversation's roster carries: a trace hop is a real Bot name, and only a hop that is +// one is routing (#1408). +const NAMES = ['Chief', 'Product', 'Dev Lead', 'Dev-1'] + +test('a fenced stored reply is spoken as its words, never as its fence mark', () => { + assert.equal(typeof speechOf, 'function', 'text.js must export the read-side chain the voice speaks through') + const line = stored('[Message from User]', 'User', 'B踢后测4') + const spoken = speechOf(line, NAMES) + assert.equal(spoken, 'B踢后测4') + assert.equal(spoken.includes(' { + for (const line of ['B踢后测4', 'hello there', 'It costs $5.']) assert.equal(speechOf(line, NAMES), line) +}) + +test('the rest of what the voice says is unchanged: markdown is words', () => { + assert.equal(speechOf('**bold** and `code`', NAMES), 'bold and code') + assert.equal(speechOf('see [the issue](https://example.com/1)', NAMES), 'see the issue') + assert.equal(speechOf('```\nconst a = 1\n```', NAMES), '') + assert.equal(speechOf(' spaced\t\tout ', NAMES), 'spaced out') +}) + +test('voice.js speaks through text.js, not through a chain of its own', () => { + const voice = readFileSync(new URL('../src/client/voice.js', import.meta.url), 'utf8') + assert.match(voice, /import \{[^}]*\bbotNames\b[^}]*\bspeechOf\b[^}]*\} from '\.\/text\.js'/, 'the speech text and the conversation names come from the read side') + assert.match(voice, /const spoken = speechOf\(text, botNames\(roster\)\)/, 'one chain: the voice hands the read side its names') + assert.equal(/plainSpeech/.test(voice), false, 'the voice keeps no second reading of its own') + assert.equal(/stripHeader\(text\)/.test(voice), false, 'the header, the trace and the fence are read once, not by the voice') +}) diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index 000ec6a3..2e7ba870 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -2379,7 +2379,7 @@ diff --git a/src/client/text.js b/src/client/text.js // parts) are not events in the chat, nor is a request handed back after its line was // already shown: deferred, resumed after a restart, or retried after a tool call // written as text. -@@ -200,18 +200,72 @@ export function commRunOf(order, peers, turn, hiddenTurns = []) { +@@ -200,18 +200,84 @@ export function commRunOf(order, peers, turn, hiddenTurns = []) { } // Latest visible line for a sidebar row; Bot-to-Bot exchange turns stay out of it. @@ -2451,6 +2451,18 @@ diff --git a/src/client/text.js b/src/client/text.js +// A chat whose roster is not loaded still reads its text, just never as a trace. +export const botNames = roster => [...(roster?.bots ?? []), ...(roster?.remoteBots ?? []), ...(roster?.agents ?? [])] + .map(entry => entry?.name).filter(name => typeof name === 'string' && name !== '') ++ ++// What voice mode reads aloud: a reply's words, read the one way a row reads a stored line — header, ++// fence and a known-name delegation trace dropped (#1395, #1408) — and then with the Markdown a speech ++// synthesiser would otherwise say out loud (a code fence, a link's target, emphasis marks). ++export function speechOf(text, names = []) { ++ return readableLine(text, names) ++ .replace(/```[\s\S]*?```/g, ' ') ++ .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') ++ .replace(/[*_`#>|~-]+/g, ' ') ++ .replace(/\s+/g, ' ') ++ .trim() ++} + const formatTokens = count => (count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count)) @@ -2494,6 +2506,21 @@ diff --git a/src/client/text.js b/src/client/text.js diff --git a/src/client/voice.js b/src/client/voice.js --- a/src/client/voice.js +++ b/src/client/voice.js +@@ -1,5 +1,5 @@ + import { createElement as h, useEffect, useRef, useState } from 'react' +-import { stripHeader } from './text.js' ++import { botNames, speechOf } from './text.js' + import { MicIcon, CloseIcon } from './icons.js' + import { BotMark, RoomAvatar } from './mark.js' + import { speechApi, useDictation } from './composer.js' +@@ -12,7 +12,6 @@ import { openCapture, speechCatalog, transcribeWave, turnDetector } from './spee + // again. With DSH's Voice Input plugin on, its recognizer hears the user: one + // recording ends after a pause and is transcribed whole. Otherwise the browser's + // own speech recognition does. The browser's speech synthesis reads the answer. +-const plainSpeech = text => stripHeader(text).replace(/```[\s\S]*?```/g, ' ').replace(/\[([^\]]*)\]\([^)]*\)/g, '$1').replace(/[*_`#>|~-]+/g, ' ').replace(/\s+/g, ' ').trim() + const VOICE_LABELS = { + listening: 'Listening', thinking: 'Thinking', speaking: 'Speaking', paused: 'Paused', + setup: 'Voice input needs its speech model', unsupported: 'Voice chat is not available in this browser', @@ -27,7 +27,7 @@ function useMicLevel(enabled) { let audio = null let frame = 0 @@ -2503,6 +2530,15 @@ diff --git a/src/client/voice.js b/src/client/voice.js if (stopped) { media.getTracks().forEach(track => track.stop()); return } stream = media audio = new AudioContext() +@@ -184,7 +184,7 @@ export function VoiceMode({ sessionId, roster, actions, useSessions, useSessionS + const [count, text] = [lastReply.slice(0, lastReply.indexOf('|')), lastReply.slice(lastReply.indexOf('|') + 1)] + if (count === waiting.current || !text.trim()) return + waiting.current = null +- const spoken = plainSpeech(text) ++ const spoken = speechOf(text, botNames(roster)) + setSaid(spoken) + setHeard('') + if (!window.speechSynthesis || !spoken) { setPhase('listening'); return } diff --git a/src/host/agents.js b/src/host/agents.js --- a/src/host/agents.js +++ b/src/host/agents.js diff --git a/patches/ds-bot/0008-client-bundle.patch b/patches/ds-bot/0008-client-bundle.patch index 2f1b2ddc..b469a2e3 100644 --- a/patches/ds-bot/0008-client-bundle.patch +++ b/patches/ds-bot/0008-client-bundle.patch @@ -1743,7 +1743,7 @@ diff --git a/client.js b/client.js var historyLoading = (snapshot) => snapshot?.openState === "cold" || snapshot?.openState === "loading"; function toolArgs(root) { if (typeof root.args?.text === "function") { -@@ -1597,17 +1597,43 @@ function commRunOf(order, peers, turn, hiddenTurns = []) { +@@ -1597,17 +1597,46 @@ function commRunOf(order, peers, turn, hiddenTurns = []) { } return { count, peerIds }; } @@ -1787,6 +1787,9 @@ diff --git a/client.js b/client.js + return /^[\s,,.。::]*$/.test(rest) ? "" : rest; +} +var botNames = (roster) => [...roster?.bots ?? [], ...roster?.remoteBots ?? [], ...roster?.agents ?? []].map((entry) => entry?.name).filter((name) => typeof name === "string" && name !== ""); ++function speechOf(text, names = []) { ++ return readableLine(text, names).replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*_`#>|~-]+/g, " ").replace(/\s+/g, " ").trim(); ++} var formatTokens = (count) => count >= 1e6 ? `${(count / 1e6).toFixed(1)}M` : count >= 1e3 ? `${(count / 1e3).toFixed(1)}K` : String(count); function usageOf(list, ids) { let input = 0; @@ -16126,7 +16129,7 @@ diff --git a/client.js b/client.js let stop = false; let timer; let seen = null; -@@ -8612,48 +8612,42 @@ function ExchangeDialog({ pair, roster, actions }) { +@@ -8612,48 +8612,41 @@ function ExchangeDialog({ pair, roster, actions }) { const right = roster.byId[pair[1]]; let lastDay = 0; const paneLeft = usePaneLeft(); @@ -16186,7 +16189,7 @@ diff --git a/client.js b/client.js // src/client/voice.js -var import_react22 = require("react"); +var import_react33 = require("react"); - var plainSpeech = (text) => stripHeader(text).replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*_`#>|~-]+/g, " ").replace(/\s+/g, " ").trim(); +-var plainSpeech = (text) => stripHeader(text).replace(/```[\s\S]*?```/g, " ").replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").replace(/[*_`#>|~-]+/g, " ").replace(/\s+/g, " ").trim(); var VOICE_LABELS = { listening: "Listening", @@ -8665,14 +8665,14 @@ var VOICE_LABELS = { @@ -16272,7 +16275,7 @@ diff --git a/client.js b/client.js if (!speech || !catalog || !listen) return void 0; const own = turn.current; let capture = null; -@@ -8838,7 +8838,7 @@ function VoiceMode({ sessionId, roster, actions, useSessions, useSessionStatus } +@@ -8838,12 +8838,12 @@ function VoiceMode({ sessionId, roster, actions, useSessions, useSessionStatus } setServiceLevel(0); }; }, [speech, catalog, listen, cycle]); @@ -16281,6 +16284,12 @@ diff --git a/client.js b/client.js if (phase !== "thinking" || running || waiting.current === null) return; const [count, text] = [lastReply.slice(0, lastReply.indexOf("|")), lastReply.slice(lastReply.indexOf("|") + 1)]; if (count === waiting.current || !text.trim()) return; + waiting.current = null; +- const spoken = plainSpeech(text); ++ const spoken = speechOf(text, botNames(roster)); + setSaid(spoken); + setHeard(""); + if (!window.speechSynthesis || !spoken) { @@ -8858,7 +8858,7 @@ function VoiceMode({ sessionId, roster, actions, useSessions, useSessionStatus } window.speechSynthesis.cancel(); window.speechSynthesis.speak(utterance); diff --git a/patches/ds-bot/files/test/voice-speech.test.js b/patches/ds-bot/files/test/voice-speech.test.js new file mode 100644 index 00000000..6e4e4f13 --- /dev/null +++ b/patches/ds-bot/files/test/voice-speech.test.js @@ -0,0 +1,82 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' +import { fenceUntrusted } from '../src/host/delivery.js' + +// Voice mode reads a Bot's reply aloud, and a stored turn is framing plus words: the host's header, +// a reply's delegation trace and an untrusted fence are routing, not content. The speech text is read +// the one way a row reads a stored line — `readableLine(text, names)` in text.js — because +// `speechSynthesis` repeats what it is handed: a fence mark reaches the user's ears as +// ` `${header}\n${fenceUntrusted(name, body)}` + +const text = await build({ + entryPoints: [fileURLToPath(new URL('../src/client/text.js', import.meta.url))], + bundle: true, format: 'esm', platform: 'node', write: false, +}) +const { speechOf } = await import(`data:text/javascript;base64,${Buffer.from(text.outputFiles[0].text).toString('base64')}`) + +// The names a conversation's roster carries: a trace hop is a real Bot name, and only a hop that is +// one is routing (#1408). +const NAMES = ['Chief', 'Product', 'Dev Lead', 'Dev-1'] + +test('a fenced stored reply is spoken as its words, never as its fence mark', () => { + assert.equal(typeof speechOf, 'function', 'text.js must export the read-side chain the voice speaks through') + const line = stored('[Message from User]', 'User', 'B踢后测4') + const spoken = speechOf(line, NAMES) + assert.equal(spoken, 'B踢后测4') + assert.equal(spoken.includes(' { + for (const line of ['B踢后测4', 'hello there', 'It costs $5.']) assert.equal(speechOf(line, NAMES), line) +}) + +test('the rest of what the voice says is unchanged: markdown is words', () => { + assert.equal(speechOf('**bold** and `code`', NAMES), 'bold and code') + assert.equal(speechOf('see [the issue](https://example.com/1)', NAMES), 'see the issue') + assert.equal(speechOf('```\nconst a = 1\n```', NAMES), '') + assert.equal(speechOf(' spaced\t\tout ', NAMES), 'spaced out') +}) + +test('voice.js speaks through text.js, not through a chain of its own', () => { + const voice = readFileSync(new URL('../src/client/voice.js', import.meta.url), 'utf8') + assert.match(voice, /import \{[^}]*\bbotNames\b[^}]*\bspeechOf\b[^}]*\} from '\.\/text\.js'/, 'the speech text and the conversation names come from the read side') + assert.match(voice, /const spoken = speechOf\(text, botNames\(roster\)\)/, 'one chain: the voice hands the read side its names') + assert.equal(/plainSpeech/.test(voice), false, 'the voice keeps no second reading of its own') + assert.equal(/stripHeader\(text\)/.test(voice), false, 'the header, the trace and the fence are read once, not by the voice') +})