diff --git a/apps/chrome-extension/e2e/browser.ts b/apps/chrome-extension/e2e/browser.ts new file mode 100644 index 00000000000..5308ac41509 --- /dev/null +++ b/apps/chrome-extension/e2e/browser.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { chromium } from "@playwright/test"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +export const extensionPath = path.resolve(__dirname, "../dist"); + +// Chrome derives an unpacked extension's id from its absolute path: on +// Windows from the UTF-16 path with an upper-case drive letter. +export const unpackedExtensionId = createHash("sha256") + .update( + process.platform === "win32" + ? Buffer.from( + extensionPath.replace(/^[a-z]:/, (drive) => drive.toUpperCase()), + "utf16le", + ) + : extensionPath, + ) + .digest("hex") + .slice(0, 32) + .replace(/[0-9a-f]/g, (digit) => + String.fromCharCode(97 + Number.parseInt(digit, 16)), + ); + +// CAP_E2E_BROWSER points the suites at an installed browser (Chrome, Edge, +// Brave) instead of Playwright's Chromium. Branded Chrome ignores +// --load-extension, so installed browsers load the extension over CDP. +const installedBrowser = process.env.CAP_E2E_BROWSER; + +// Allows the extension's camera and mic in a new profile, as if the user had +// accepted the permission prompt. Chrome refuses CDP permission grants for +// extension origins, so the setting is written before the browser starts. +const allowExtensionMediaInProfile = async (userDataDir: string) => { + const allow = { setting: 1 }; + const exceptions = { + [`chrome-extension://${unpackedExtensionId}/,*`]: allow, + [`chrome-extension://${unpackedExtensionId},*`]: allow, + }; + await mkdir(path.join(userDataDir, "Default"), { recursive: true }); + await writeFile( + path.join(userDataDir, "Default", "Preferences"), + JSON.stringify({ + profile: { + content_settings: { + exceptions: { + media_stream_camera: exceptions, + media_stream_mic: exceptions, + }, + }, + }, + }), + ); +}; + +export const launchExtensionBrowser = async ({ + profilePrefix, + headless, + args, + allowExtensionMedia = false, +}: { + profilePrefix: string; + headless: boolean; + args: string[]; + allowExtensionMedia?: boolean; +}) => { + const userDataDir = await mkdtemp(path.join(tmpdir(), profilePrefix)); + if (allowExtensionMedia) await allowExtensionMediaInProfile(userDataDir); + const context = await chromium.launchPersistentContext( + userDataDir, + installedBrowser + ? { + executablePath: installedBrowser, + headless, + ignoreDefaultArgs: ["--disable-extensions"], + args: ["--enable-unsafe-extension-debugging", ...args], + } + : { + channel: "chromium", + headless, + args: [ + `--disable-extensions-except=${extensionPath}`, + `--load-extension=${extensionPath}`, + ...args, + ], + }, + ); + if (installedBrowser) { + const browser = context.browser(); + if (!browser) throw new Error("Installed browser did not expose CDP"); + const session = await browser.newBrowserCDPSession(); + await session.send( + "Extensions.loadUnpacked" as never, + { + path: extensionPath, + } as never, + ); + await session.detach(); + } + const cleanup = async () => { + await context.close(); + await rm(userDataDir, { recursive: true, force: true }); + }; + return { context, cleanup }; +}; diff --git a/apps/chrome-extension/e2e/overlay-ui.spec.ts b/apps/chrome-extension/e2e/overlay-ui.spec.ts index 307609c637a..24dd119ff78 100644 --- a/apps/chrome-extension/e2e/overlay-ui.spec.ts +++ b/apps/chrome-extension/e2e/overlay-ui.spec.ts @@ -1,20 +1,16 @@ -import { mkdtemp, rm } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse, } from "node:http"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; import { type BrowserContext, type CDPSession, - chromium, expect, type Page, test, } from "@playwright/test"; +import { launchExtensionBrowser } from "./browser"; type ChromeGlobal = typeof globalThis & { chrome: { @@ -56,8 +52,6 @@ type ChromeGlobal = typeof globalThis & { }; }; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -103,8 +97,19 @@ const createMockCapServer = async ( sendJson(response, 204, {}); return; } - if (request.method === "GET" && url.pathname === "/capture.html") { - response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }); + if ( + request.method === "GET" && + (url.pathname === "/capture.html" || url.pathname === "/strict.html") + ) { + response.writeHead(200, { + "Content-Type": "text/html; charset=utf-8", + ...(url.pathname === "/strict.html" + ? { + "Content-Security-Policy": + "default-src 'none'; frame-src 'none'; img-src 'none'; style-src 'none'; script-src 'none'", + } + : {}), + }); response.end( `Cap Close UI Target

Close UI page

`, ); @@ -199,13 +204,10 @@ const createMockCapServer = async ( }; const launchExtensionContext = async () => { - const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-close-e2e-")); - const context = await chromium.launchPersistentContext(userDataDir, { - channel: "chromium", + return launchExtensionBrowser({ + profilePrefix: "cap-close-e2e-", headless: true, args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, "--allow-http-screen-capture", "--auto-select-tab-capture-source-by-title=Cap Close UI Target", "--auto-select-desktop-capture-source=Cap Close UI Target", @@ -215,11 +217,6 @@ const launchExtensionContext = async () => { "--use-fake-ui-for-media-stream", ], }); - const cleanup = async () => { - await context.close(); - await rm(userDataDir, { recursive: true, force: true }); - }; - return { context, cleanup }; }; const getServiceWorker = async (context: BrowserContext) => { @@ -235,7 +232,11 @@ const getServiceWorker = async (context: BrowserContext) => { const configureExtension = async ( worker: Awaited>, apiBaseUrl: string, - options: { countdownEnabled?: boolean; webcamEnabled?: boolean } = {}, + options: { + countdownEnabled?: boolean; + webcamEnabled?: boolean; + microphoneEnabled?: boolean; + } = {}, ) => { await worker.evaluate( async ({ authKey, bootstrapKey, settingsKey, apiBaseUrl, options }) => { @@ -271,7 +272,10 @@ const configureExtension = async ( shape: "round", mirror: false, }, - microphone: { enabled: false, deviceId: null }, + microphone: { + enabled: options.microphoneEnabled ?? false, + deviceId: null, + }, systemAudio: { enabled: false }, sounds: { enabled: false }, countdown: { @@ -601,14 +605,17 @@ test("an abandoned recording start clears without leaving controls behind", asyn .toBe(true); const devtools = await extension.context.newCDPSession(targetPage); + // A start is abandoned when the worker dies mid-start: session storage + // keeps "creating" and the next worker life has to clear it. await writeStaleCreatingState(worker); - await expect.poll(() => readSessionRecordingPhase(worker)).toBe("creating"); + await devtools.send("ServiceWorker.enable"); + await devtools.send("ServiceWorker.stopAllWorkers"); await targetPage.waitForTimeout(500); expect( await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-recording-badge", ), ).toBeNull(); @@ -621,7 +628,10 @@ test("an abandoned recording start clears without leaving controls behind", asyn type: "get-recording-status", })) as { status?: { phase?: string } }; expect(statusResponse.status?.phase).toBe("idle"); - await expect.poll(() => readSessionRecordingPhase(worker)).toBe("idle"); + const restartedWorker = await getServiceWorker(extension.context); + await expect + .poll(() => readSessionRecordingPhase(restartedWorker)) + .toBe("idle"); await expect .poll(() => frameWithUrl(targetPage, "popup.html") === null, { timeout: 10_000, @@ -631,7 +641,7 @@ test("an abandoned recording start clears without leaving controls behind", asyn await getClosedShadowElementBox( devtools, "class", - "cap-extension-recording-rail", + "cap-extension-recording-badge", ), ).toBeNull(); } finally { @@ -888,92 +898,77 @@ test("recording controls stay stable and the camera resizes directly", async () .toBe(true); const devtools = await extension.context.newCDPSession(targetPage); + const getBadgeBox = () => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-badge", + ); + const getDockBox = () => + getClosedShadowElementBox( + devtools, + "class", + "cap-extension-vertical-dock", + ); + const readPhase = async () => { + const response = (await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + })) as { status?: { phase?: string } }; + return response.status?.phase; + }; await expect - .poll( - async () => - (await getClosedShadowElementBox( - devtools, - "class", - "cap-extension-recording-rail", - )) !== null, - { timeout: 10_000 }, - ) + .poll(async () => (await getBadgeBox()) !== null, { timeout: 10_000 }) .toBe(true); - const initialBox = await getClosedShadowElementBox( - devtools, - "class", - "cap-extension-recording-rail", - ); - if (!initialBox) throw new Error("recording bar missing"); - expect(initialBox.x).toBeLessThanOrEqual(20); - expect(initialBox.width).toBeGreaterThan(100); - expect(initialBox.width).toBeLessThanOrEqual(128); - expect(initialBox.height).toBeLessThanOrEqual(44); - const handleBox = await getClosedShadowElementBox( - devtools, - "data-drag-handle", - ); - if (!handleBox) throw new Error("recording bar drag handle missing"); + const initialBox = await getBadgeBox(); + if (!initialBox) throw new Error("recording badge missing"); + expect(initialBox.x).toBeLessThanOrEqual(32); + expect(initialBox.height).toBeLessThanOrEqual(64); const timeBox = await getClosedShadowElementBox( devtools, - "data-recording-time", + "class", + "cap-extension-recording-badge-time", ); - if (!timeBox || timeBox.width < 30) { + if (!timeBox || timeBox.width < 20) { throw new Error("recording time is not visible at rest"); } - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("0"); + expect(await getDockBox()).toBeNull(); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); + await expect.poll(async () => (await getDockBox()) !== null).toBe(true); + const settledPause: { box: ElementBox | null } = { box: null }; await expect - .poll(() => - getClosedShadowComputedStyle( + .poll(async () => { + const previous = settledPause.box; + const next = await getClosedShadowElementBox( devtools, - "data-recording-actions", - "opacity", - ), - ) - .toBe("1"); - const actionsBox = await getClosedShadowElementBox( - devtools, - "data-recording-actions", - ); - if (!actionsBox) throw new Error("recording action capsule missing"); - const actionGap = actionsBox.x - (initialBox.x + initialBox.width); - expect(actionGap).toBeGreaterThanOrEqual(4); - expect(actionGap).toBeLessThanOrEqual(8); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "border-top-left-radius", - ), - ).toBe("14px"); - const pauseBox = await getClosedShadowElementBox( - devtools, - "data-recording-pause", - ); + "class", + "is-pause", + ); + settledPause.box = next; + return ( + previous !== null && + next !== null && + previous.x === next.x && + previous.y === next.y + ); + }) + .toBe(true); + const pauseBox = settledPause.box; if (!pauseBox) throw new Error("pause action missing on hover"); + expect( + await getClosedShadowElementBox(devtools, "class", "is-mic"), + ).toBeNull(); await targetPage.mouse.move( pauseBox.x + pauseBox.width / 2, pauseBox.y + pauseBox.height / 2, + { steps: 4 }, ); await targetPage.waitForTimeout(300); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); + expect(await getDockBox()).not.toBeNull(); await targetPage.screenshot({ path: "test-results/recording-bar-expanded.png", }); @@ -981,57 +976,41 @@ test("recording controls stay stable and the camera resizes directly", async () pauseBox.x + pauseBox.width / 2, pauseBox.y + pauseBox.height / 2, ); - await expect - .poll(async () => { - const response = (await sendServiceWorkerMessage(messengerPage, { - target: "service-worker", - type: "get-recording-status", - })) as { status?: { phase?: string } }; - return response.status?.phase; - }) - .toBe("paused"); - expect( - await getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ).toBe("1"); + await expect.poll(readPhase).toBe("paused"); + expect(await getDockBox()).not.toBeNull(); + await targetPage.waitForTimeout(500); + const resumeBox = await getClosedShadowElementBox( + devtools, + "class", + "is-paused", + ); + if (!resumeBox) throw new Error("resume action missing while paused"); await targetPage.mouse.click( - pauseBox.x + pauseBox.width / 2, - pauseBox.y + pauseBox.height / 2, + resumeBox.x + resumeBox.width / 2, + resumeBox.y + resumeBox.height / 2, ); - await expect - .poll(async () => { - const response = (await sendServiceWorkerMessage(messengerPage, { - target: "service-worker", - type: "get-recording-status", - })) as { status?: { phase?: string } }; - return response.status?.phase; - }) - .toBe("recording"); + await expect.poll(readPhase).toBe("recording"); + + await targetPage.mouse.move(760, 40); + await expect.poll(async () => (await getDockBox()) === null).toBe(true); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2, - handleBox.y + handleBox.height / 2, + initialBox.x + initialBox.width / 2, + initialBox.y + initialBox.height / 2, ); await targetPage.mouse.down(); await targetPage.mouse.move( - handleBox.x + handleBox.width / 2 + 120, - handleBox.y + handleBox.height / 2 - 80, + initialBox.x + initialBox.width / 2 + 420, + initialBox.y + initialBox.height / 2 - 80, { steps: 8 }, ); await targetPage.mouse.up(); await expect .poll(async () => { - const movedBox = await getClosedShadowElementBox( - devtools, - "class", - "cap-extension-recording-rail", - ); + const movedBox = await getBadgeBox(); return ( movedBox !== null && - movedBox.x - initialBox.x > 80 && + movedBox.x - initialBox.x > 380 && initialBox.y - movedBox.y > 50 ); }) @@ -1063,17 +1042,10 @@ test("recording controls stay stable and the camera resizes directly", async () ), ) .toBe(true); + expect(await readPhase()).toBe("recording"); await targetPage.mouse.move(760, 40); - await expect - .poll(() => - getClosedShadowComputedStyle( - devtools, - "data-recording-actions", - "opacity", - ), - ) - .toBe("0"); + await expect.poll(async () => (await getDockBox()) === null).toBe(true); await targetPage.waitForTimeout(500); await targetPage.screenshot({ path: "test-results/recording-bar-visible.png", @@ -1149,3 +1121,300 @@ test("recording controls stay stable and the camera resizes directly", async () await mockServer.close(); } }); + +test("the dock mutes the recording microphone without changing saved settings", async () => { + test.setTimeout(90_000); + const mockServer = await createMockCapServer(); + const extension = await launchExtensionContext(); + + try { + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin, { + webcamEnabled: false, + microphoneEnabled: true, + }); + + const messengerPage = await extension.context.newPage(); + await messengerPage.goto( + `chrome-extension://${new URL(worker.url()).host}/popup.html`, + ); + const targetPage = await extension.context.newPage(); + await targetPage.goto(`${mockServer.origin}/capture.html`); + await targetPage.bringToFront(); + + const startResponse = (await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "start-recording", + mode: "fullscreen", + })) as { ok: boolean }; + expect(startResponse.ok).toBe(true); + + const devtools = await extension.context.newCDPSession(targetPage); + const readMicrophone = async () => { + const response = (await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + })) as { status?: { microphone?: string } }; + return response.status?.microphone; + }; + const readSavedMicrophoneEnabled = () => + worker.evaluate( + (settingsKey) => + new Promise((resolve) => { + const chromeApi = (globalThis as ChromeGlobal).chrome; + chromeApi.storage.local.get(settingsKey, (items) => { + const settings = items[settingsKey] as + | { microphone?: { enabled?: boolean } } + | undefined; + resolve(settings?.microphone?.enabled); + }); + }), + SETTINGS_KEY, + ); + const settledMicBox = async () => { + const holder: { box: ElementBox | null } = { box: null }; + await expect + .poll(async () => { + const previous = holder.box; + const next = await getClosedShadowElementBox( + devtools, + "class", + "is-mic", + ); + holder.box = next; + return ( + previous !== null && + next !== null && + previous.x === next.x && + previous.y === next.y + ); + }) + .toBe(true); + if (!holder.box) throw new Error("microphone action missing on hover"); + return holder.box; + }; + const clickMic = async () => { + const badge = await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-badge", + ); + if (!badge) throw new Error("recording badge missing"); + await targetPage.mouse.move( + badge.x + badge.width / 2, + badge.y + badge.height / 2, + ); + const micBox = await settledMicBox(); + await targetPage.mouse.move( + micBox.x + micBox.width / 2, + micBox.y + micBox.height / 2, + { steps: 4 }, + ); + await targetPage.mouse.click( + micBox.x + micBox.width / 2, + micBox.y + micBox.height / 2, + ); + }; + + await expect.poll(readMicrophone, { timeout: 15_000 }).toBe("live"); + await expect + .poll( + async () => + (await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-recording-badge", + )) !== null, + { timeout: 10_000 }, + ) + .toBe(true); + + await clickMic(); + await expect.poll(readMicrophone).toBe("muted"); + await expect + .poll( + async () => + (await getClosedShadowElementBox(devtools, "class", "is-muted")) !== + null, + ) + .toBe(true); + expect(await readSavedMicrophoneEnabled()).toBe(true); + + await clickMic(); + await expect.poll(readMicrophone).toBe("live"); + expect(await readSavedMicrophoneEnabled()).toBe(true); + + const stopResponse = (await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + })) as { ok: boolean }; + expect(stopResponse.ok).toBe(true); + } finally { + await extension.cleanup(); + await mockServer.close(); + } +}); + +test("choosing Camera mode picks a camera so Start stays available", async () => { + const mockServer = await createMockCapServer(); + const extension = await launchExtensionContext(); + + try { + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin, { + webcamEnabled: false, + }); + const messengerPage = await extension.context.newPage(); + await messengerPage.goto( + `chrome-extension://${new URL(worker.url()).host}/popup.html`, + ); + const targetPage = await extension.context.newPage(); + await targetPage.goto(`${mockServer.origin}/capture.html`); + await targetPage.bringToFront(); + await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + await expect + .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { + timeout: 15_000, + }) + .toBe(true); + const panel = frameWithUrl(targetPage, "popup.html"); + if (!panel) throw new Error("recorder panel missing"); + + await panel + .getByRole("combobox") + .filter({ hasText: /Full Screen/ }) + .click(); + await panel.getByRole("option", { name: "Camera only" }).click(); + + await expect( + panel.getByRole("button", { name: "Start recording" }), + ).toBeEnabled(); + const savedCamera = await worker.evaluate( + (settingsKey) => + new Promise((resolve) => { + const chromeApi = (globalThis as ChromeGlobal).chrome; + chromeApi.storage.local.get(settingsKey, (items) => { + const settings = items[settingsKey] as + | { webcam?: { deviceId?: string | null } } + | undefined; + resolve(settings?.webcam?.deviceId); + }); + }), + SETTINGS_KEY, + ); + expect(savedCamera).toBeTruthy(); + } finally { + await extension.cleanup(); + await mockServer.close(); + } +}); + +test("the recorder panel still loads after other tabs push its token out of the registry", async () => { + const mockServer = await createMockCapServer(); + const extension = await launchExtensionContext(); + + try { + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin, { + webcamEnabled: false, + }); + const messengerPage = await extension.context.newPage(); + await messengerPage.goto( + `chrome-extension://${new URL(worker.url()).host}/popup.html`, + ); + const targetPage = await extension.context.newPage(); + await targetPage.goto(`${mockServer.origin}/capture.html`); + await targetPage.bringToFront(); + const togglePanel = () => + sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + const startButton = () => + frameWithUrl(targetPage, "popup.html") + ?.getByRole("button", { name: "Start recording" }) + .isVisible() + .catch(() => false) ?? false; + + await togglePanel(); + await expect.poll(startButton, { timeout: 15_000 }).toBe(true); + await togglePanel(); + await expect + .poll(() => frameWithUrl(targetPage, "popup.html") === null) + .toBe(true); + + // Hundreds of newer registrations, as from many tabs loading the + // overlay, evict this page's tokens. + await worker.evaluate( + () => + new Promise((resolve) => { + const now = Date.now(); + const tokens = Object.fromEntries( + Array.from({ length: 300 }, (_, index) => [ + `other-tab-${index}`, + now - 300 + index, + ]), + ); + (globalThis as ChromeGlobal).chrome.storage.session.set( + { "cap-extension-overlay-tokens": tokens }, + resolve, + ); + }), + ); + + await togglePanel(); + await expect.poll(startButton, { timeout: 15_000 }).toBe(true); + } finally { + await extension.cleanup(); + await mockServer.close(); + } +}); + +test("the recorder panel and camera preview load on a page with a strict CSP", async () => { + const mockServer = await createMockCapServer(); + const extension = await launchExtensionContext(); + + try { + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin); + const messengerPage = await extension.context.newPage(); + await messengerPage.goto( + `chrome-extension://${new URL(worker.url()).host}/popup.html`, + ); + const targetPage = await extension.context.newPage(); + await targetPage.goto(`${mockServer.origin}/strict.html`); + await targetPage.bringToFront(); + await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "open-recorder-panel", + }); + await expect + .poll(() => frameWithUrl(targetPage, "popup.html") !== null, { + timeout: 15_000, + }) + .toBe(true); + const panel = frameWithUrl(targetPage, "popup.html"); + if (!panel) throw new Error("recorder panel missing"); + await expect( + panel.getByRole("button", { name: "Start recording" }), + ).toBeVisible(); + await expect + .poll(() => frameWithUrl(targetPage, "camera-preview.html") !== null, { + timeout: 15_000, + }) + .toBe(true); + const devtools = await extension.context.newCDPSession(targetPage); + const panelBox = await getClosedShadowElementBox( + devtools, + "class", + "cap-extension-panel", + ); + expect(panelBox?.width ?? 0).toBeGreaterThan(200); + } finally { + await extension.cleanup(); + await mockServer.close(); + } +}); diff --git a/apps/chrome-extension/e2e/recording-upload.spec.ts b/apps/chrome-extension/e2e/recording-upload.spec.ts index fcf73c51481..becf9c65c79 100644 --- a/apps/chrome-extension/e2e/recording-upload.spec.ts +++ b/apps/chrome-extension/e2e/recording-upload.spec.ts @@ -1,4 +1,5 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, @@ -6,15 +7,9 @@ import { } from "node:http"; import { tmpdir } from "node:os"; import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { - type BrowserContext, - chromium, - expect, - type Page, - test, -} from "@playwright/test"; +import { type BrowserContext, expect, type Page, test } from "@playwright/test"; import type { RecordingStatus } from "../src/shared/types"; +import { launchExtensionBrowser, unpackedExtensionId } from "./browser"; type ChromeRuntimeResponse = | { @@ -32,8 +27,10 @@ type MockState = { initiateBodies: unknown[]; presignBodies: unknown[]; uploadBytes: number[]; + uploadParts: Map; uploadHeaders: Record[]; videoId: string; + bootstrapDelayMs: number; }; type ChromeGlobal = typeof globalThis & { @@ -54,8 +51,6 @@ type ChromeGlobal = typeof globalThis & { }; }; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -150,6 +145,35 @@ const animatedCapturePage = () => ` `; +// Flashes white and beeps together once a second, both driven by the audio +// clock, so the uploaded file shows whether audio and video stay aligned. +const audibleCapturePage = () => ` + + + Cap E2E Capture Target + + + + +`; + const createMockCapServer = async () => { const state: MockState = { completeBodies: [], @@ -157,6 +181,8 @@ const createMockCapServer = async () => { initiateBodies: [], presignBodies: [], uploadBytes: [], + uploadParts: new Map(), + bootstrapDelayMs: 0, uploadHeaders: [], videoId: `e2e-${Date.now()}`, }; @@ -175,10 +201,18 @@ const createMockCapServer = async () => { return; } + if (request.method === "GET" && url.pathname === "/capture-audio.html") { + sendHtml(response, audibleCapturePage()); + return; + } + if ( request.method === "GET" && url.pathname === "/api/extension/bootstrap" ) { + await new Promise((resolve) => + setTimeout(resolve, state.bootstrapDelayMs), + ); sendJson(response, 200, { user: { id: "user-e2e", @@ -246,6 +280,10 @@ const createMockCapServer = async () => { if (request.method === "PUT" && url.pathname.startsWith("/mock-s3/")) { const body = await readRequestBody(request); state.uploadBytes.push(body.byteLength); + state.uploadParts.set( + Number(url.pathname.replace("/mock-s3/part-", "")), + body, + ); state.uploadHeaders.push(request.headers); response.writeHead(200, { "Access-Control-Allow-Origin": "*", @@ -327,33 +365,55 @@ const createMockCapServer = async () => { } resolve(); }); + // The share page the extension opens after an upload holds a + // keep-alive connection that would otherwise block close. + server.closeAllConnections(); }), }; }; -const launchExtensionContext = async () => { - const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-extension-e2e-")); - const context = await chromium.launchPersistentContext(userDataDir, { - channel: "chromium", - headless: true, +// Chromium's fake capture devices and fake permission UI both make tab capture +// fail with "Requested device not found", and headless Chromium captures tab +// audio as silence, so tab recordings launch headed without them. +const canLaunchHeaded = + process.platform !== "linux" || Boolean(process.env.DISPLAY); + +// CAP_E2E_SYSTEM_DEVICES uses the machine's own camera and microphone (on CI +// rigs, OS-level synthetic devices such as a v4l2loopback camera and a +// PulseAudio virtual mic) instead of Chromium's built-in fake devices. +const useSystemDevices = Boolean(process.env.CAP_E2E_SYSTEM_DEVICES); + +const launchExtensionContext = async ({ + tabCapture = false, + fakeAudioFile, +}: { + tabCapture?: boolean; + fakeAudioFile?: string; +} = {}) => { + const extension = await launchExtensionBrowser({ + profilePrefix: "cap-extension-e2e-", + headless: !tabCapture, + // Without the fake permission UI, grant the camera and mic up front. + allowExtensionMedia: tabCapture, args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, + `--allowlisted-extension-id=${unpackedExtensionId}`, "--allow-http-screen-capture", "--auto-select-desktop-capture-source=Cap E2E Capture Target", "--auto-select-tab-capture-source-by-title=Cap E2E Capture Target", "--enable-usermedia-screen-capturing", "--autoplay-policy=no-user-gesture-required", - "--use-fake-device-for-media-stream", - "--use-fake-ui-for-media-stream", + ...(!tabCapture + ? [ + "--use-fake-ui-for-media-stream", + ...(useSystemDevices ? [] : ["--use-fake-device-for-media-stream"]), + ...(fakeAudioFile + ? [`--use-file-for-fake-audio-capture=${fakeAudioFile}`] + : []), + ] + : []), ], }); - const cleanup = async () => { - await context.close(); - await rm(userDataDir, { recursive: true, force: true }); - }; - - return { context, cleanup }; + return extension; }; const getServiceWorker = async (context: BrowserContext) => { @@ -379,15 +439,40 @@ const openExtensionMessengerPage = async ( return page; }; +type RecordingOptions = { + mode: "fullscreen" | "tab" | "camera"; + microphone: boolean; + webcam?: boolean; + systemAudio: boolean; + capturePath: string; + durationMs: number; + cachedPlan?: { isPro: boolean; maxRecordingSeconds: number | null }; + microphoneWarning?: boolean; +}; + +const DEFAULT_RECORDING_OPTIONS: RecordingOptions = { + mode: RECORDING_MODE, + microphone: false, + systemAudio: false, + capturePath: "/capture.html", + durationMs: RECORDING_MS, +}; + const configureExtension = async ( worker: Awaited>, apiBaseUrl: string, + options: RecordingOptions = DEFAULT_RECORDING_OPTIONS, ) => { await worker.evaluate( async ({ authKey, bootstrapKey, recordingMode, + microphone, + webcam, + systemAudio, + cachedPlan, + microphoneWarning, settingsKey, apiBaseUrl, }) => { @@ -419,10 +504,7 @@ const configureExtension = async ( id: "org-e2e", name: "Extension E2E", }, - plan: { - isPro: true, - maxRecordingSeconds: 600, - }, + plan: cachedPlan, }, cachedAt: Date.now(), }, @@ -434,19 +516,19 @@ const configureExtension = async ( microphone: null, }, webcam: { - enabled: false, - deviceId: null, + enabled: webcam, + deviceId: webcam ? "__cap_default_camera__" : null, position: "bottom-left", size: 230, shape: "round", mirror: false, }, microphone: { - enabled: false, + enabled: microphone, deviceId: null, }, systemAudio: { - enabled: false, + enabled: systemAudio, }, sounds: { enabled: false, @@ -456,7 +538,7 @@ const configureExtension = async ( seconds: 3, }, microphoneWarning: { - enabled: false, + enabled: microphoneWarning, }, }, }, @@ -475,7 +557,15 @@ const configureExtension = async ( apiBaseUrl, authKey: AUTH_KEY, bootstrapKey: BOOTSTRAP_CACHE_KEY, - recordingMode: RECORDING_MODE, + recordingMode: options.mode, + microphone: options.microphone, + webcam: options.webcam ?? false, + systemAudio: options.systemAudio, + microphoneWarning: options.microphoneWarning ?? false, + cachedPlan: options.cachedPlan ?? { + isPro: true, + maxRecordingSeconds: 600, + }, settingsKey: SETTINGS_KEY, }, ); @@ -536,21 +626,223 @@ const expectSuccessfulUpload = async (page: Page, state: MockState) => { ).toBeGreaterThanOrEqual(1); }; +type StreamDurations = { video: number; audio: number | null }; + +const onsets = (values: { time: number; level: number }[], threshold: number) => + values + .filter( + (value, index) => + value.level > threshold && + (index === 0 || values[index - 1].level <= threshold), + ) + .map((value) => value.time); + +// Pairs each white flash with the nearest beep and returns audio minus video +// time in seconds for every pair. +const measureFlashBeepOffsets = ( + file: string, + samples: Float32Array, + sampleRate: number, +) => { + const lines = execFileSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + file, + "-an", + "-vf", + "signalstats,metadata=print:key=lavfi.signalstats.YAVG:file=-", + "-f", + "null", + "-", + ], + { maxBuffer: 16 * 1024 * 1024 }, + ) + .toString() + .split("\n"); + const frames: { time: number; level: number }[] = []; + let frameTime: number | null = null; + for (const line of lines) { + const time = /pts_time:([\d.]+)/.exec(line); + if (time) frameTime = Number(time[1]); + const level = /YAVG=([\d.]+)/.exec(line); + if (level && frameTime !== null) { + frames.push({ time: frameTime, level: Number(level[1]) }); + } + } + const windowSize = sampleRate / 100; + const windows: { time: number; level: number }[] = []; + for ( + let start = 0; + start + windowSize <= samples.length; + start += windowSize + ) { + let peak = 0; + for (let index = start; index < start + windowSize; index += 1) { + peak = Math.max(peak, Math.abs(samples[index])); + } + windows.push({ time: start / sampleRate, level: peak }); + } + const beeps = onsets(windows, 0.1); + return onsets(frames, 128).flatMap((flash) => { + const nearest = beeps.reduce( + (best, beep) => + best === null || Math.abs(beep - flash) < Math.abs(best - flash) + ? beep + : best, + null, + ); + return nearest !== null && Math.abs(nearest - flash) < 0.5 + ? [nearest - flash] + : []; + }); +}; + +// Frame size, mean brightness and how many frames differ from the one before, +// so a black or frozen camera fails even when the recording has frames. +const measureVideo = (file: string) => { + const [width, height] = execFileSync("ffprobe", [ + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "stream=width,height", + "-of", + "csv=p=0", + file, + ]) + .toString() + .trim() + .split(",") + .map(Number); + const lines = execFileSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + file, + "-an", + "-vf", + "signalstats,metadata=print:file=-", + "-f", + "null", + "-", + ], + { maxBuffer: 16 * 1024 * 1024 }, + ) + .toString() + .split("\n"); + const luma: number[] = []; + let movingFrames = 0; + for (const line of lines) { + const average = /YAVG=([\d.]+)/.exec(line); + if (average) luma.push(Number(average[1])); + const difference = /YDIF=([\d.]+)/.exec(line); + if (difference && Number(difference[1]) > 0.5) movingFrames += 1; + } + return { + width, + height, + frames: luma.length, + meanLuma: luma.reduce((total, value) => total + value, 0) / luma.length, + movingFrames, + }; +}; + +const probeUploadedRecording = async (state: MockState) => { + const parts = [...state.uploadParts.entries()] + .sort(([a], [b]) => a - b) + .map(([, body]) => body); + const directory = await mkdtemp(path.join(tmpdir(), "cap-e2e-media-")); + const file = path.join(directory, "recording.webm"); + await writeFile(file, Buffer.concat(parts)); + try { + const duration = (type: "v" | "a") => { + const output = execFileSync("ffprobe", [ + "-v", + "error", + "-select_streams", + type, + "-show_entries", + "packet=pts_time", + "-of", + "csv=p=0", + file, + ]) + .toString() + .trim() + .split("\n") + .map(Number) + .filter(Number.isFinite); + return output.length > 0 ? Math.max(...output) : null; + }; + const pcm = execFileSync( + "ffmpeg", + [ + "-v", + "error", + "-i", + file, + "-vn", + "-ac", + "1", + "-ar", + "8000", + "-f", + "f32le", + "-", + ], + { maxBuffer: 64 * 1024 * 1024 }, + ); + const samples = new Float32Array( + pcm.buffer, + pcm.byteOffset, + Math.floor(pcm.byteLength / 4), + ); + const tail = samples.subarray(Math.max(0, samples.length - 8000 * 1.5)); + const tailPeak = tail.reduce( + (peak, value) => Math.max(peak, Math.abs(value)), + 0, + ); + const durations: StreamDurations = { + video: duration("v") ?? 0, + audio: duration("a"), + }; + return { + durations, + tailPeak, + audioPeak: samples.reduce( + (peak, value) => Math.max(peak, Math.abs(value)), + 0, + ), + video: measureVideo(file), + avOffsets: measureFlashBeepOffsets(file, samples, 8000), + }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +}; + const startRecording = async ( context: BrowserContext, worker: Awaited>, apiBaseUrl: string, + options: RecordingOptions = DEFAULT_RECORDING_OPTIONS, ) => { - await configureExtension(worker, apiBaseUrl); + await configureExtension(worker, apiBaseUrl, options); const messengerPage = await openExtensionMessengerPage(context, worker); const capturePage = await context.newPage(); - await capturePage.goto(`${apiBaseUrl}/capture.html`); + await capturePage.goto(`${apiBaseUrl}${options.capturePath}`); await capturePage.bringToFront(); const startResponse = await sendServiceWorkerMessage(messengerPage, { target: "service-worker", type: "start-recording", - mode: RECORDING_MODE, + mode: options.mode, }); if (!startResponse.ok) { throw new Error(startResponse.error); @@ -567,7 +859,7 @@ const startRecording = async ( }) .toBe("recording"); - await capturePage.waitForTimeout(RECORDING_MS); + await capturePage.waitForTimeout(options.durationMs); return { capturePage, messengerPage, @@ -608,6 +900,382 @@ test.describe("extension recording upload", () => { await expectSuccessfulUpload(messengerPage, mockServer.state); }); + for (const options of [ + { + name: "fullscreen with microphone", + mode: "fullscreen", + microphone: true, + systemAudio: false, + capturePath: "/capture.html", + }, + { + name: "tab with tab audio", + mode: "tab", + microphone: false, + systemAudio: true, + capturePath: "/capture-audio.html", + }, + ] as const) { + test(`keeps audio for the whole ${options.name} recording`, async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + if (options.mode === "tab") { + test.skip(!canLaunchHeaded, "Tab audio capture needs a headed browser"); + await extension.cleanup(); + extension = await launchExtensionContext({ tabCapture: true }); + } + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + { ...options, durationMs: 6_000 }, + ); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + + const { durations, tailPeak, avOffsets } = await probeUploadedRecording( + mockServer.state, + ); + if (process.env.CAP_E2E_REPORT_AV) { + console.log( + JSON.stringify({ + name: options.name, + durations, + tailPeak, + avOffsets, + }), + ); + } + expect(durations.video).toBeGreaterThan(5); + expect(durations.audio).not.toBeNull(); + expect(Math.abs((durations.audio ?? 0) - durations.video)).toBeLessThan( + 0.5, + ); + expect(tailPeak).toBeGreaterThan(0.01); + if (options.mode === "tab") { + expect(avOffsets.length).toBeGreaterThanOrEqual(4); + const sorted = [...avOffsets].sort((a, b) => a - b); + const median = sorted[Math.floor(sorted.length / 2)]; + expect(Math.abs(median)).toBeLessThan(0.15); + expect(sorted[sorted.length - 1] - sorted[0]).toBeLessThan(0.1); + } + }); + } + + for (const input of [ + { + name: "quiet room tone starts without a warning", + source: "anoisesrc=color=pink:amplitude=0.0005", + warns: false, + }, + { + name: "a silent microphone still warns before starting", + source: "anullsrc=channel_layout=mono:sample_rate=48000", + warns: true, + }, + ]) { + test(`microphone check: ${input.name}`, async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + test.skip( + useSystemDevices, + "Feeds the mic through Chromium's fake device", + ); + const audioDirectory = await mkdtemp(path.join(tmpdir(), "cap-e2e-mic-")); + try { + const audioFile = path.join(audioDirectory, "mic.wav"); + execFileSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + input.source, + "-t", + "10", + "-ar", + "48000", + "-ac", + "1", + audioFile, + ]); + await extension.cleanup(); + extension = await launchExtensionContext({ fakeAudioFile: audioFile }); + const worker = await getServiceWorker(extension.context); + await configureExtension(worker, mockServer.origin, { + ...DEFAULT_RECORDING_OPTIONS, + microphone: true, + microphoneWarning: true, + }); + const messengerPage = await openExtensionMessengerPage( + extension.context, + worker, + ); + const capturePage = await extension.context.newPage(); + await capturePage.goto(`${mockServer.origin}/capture.html`); + await capturePage.bringToFront(); + void sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "start-recording", + mode: DEFAULT_RECORDING_OPTIONS.mode, + }).catch(() => undefined); + + const readPhase = async () => { + const response = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + return response.ok ? response.status?.phase : response.error; + }; + if (input.warns) { + await capturePage.waitForTimeout(4_000); + expect(await readPhase()).toBe("idle"); + expect(mockServer.state.initiateBodies).toHaveLength(0); + } else { + await expect.poll(readPhase, { timeout: 10_000 }).toBe("recording"); + await capturePage.waitForTimeout(1_000); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + } + } finally { + await rm(audioDirectory, { recursive: true, force: true }); + } + }); + } + + test("re-checks a cached free plan so an upgraded account is not cut off", async () => { + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + // The popup's background plan refresh is still in flight when the + // recording starts, as when Start is pressed right after opening it. + mockServer.state.bootstrapDelayMs = 1_500; + const worker = await getServiceWorker(extension.context); + const { messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + { + ...DEFAULT_RECORDING_OPTIONS, + durationMs: 4_000, + cachedPlan: { isPro: false, maxRecordingSeconds: 2 }, + }, + ); + const status = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + expect(status).toMatchObject({ ok: true, status: { phase: "recording" } }); + + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + }); + + // Random interleavings of everything a user or the browser can do around a + // recording. Run with CAP_E2E_FUZZ= (and CAP_E2E_FUZZ_SEED to + // replay a failure). After every step the service worker must answer, no + // start may hang, and at the end a clean recording must still upload. + test("survives random recording interleavings", async () => { + const iterations = Number(process.env.CAP_E2E_FUZZ ?? 0); + test.skip(iterations <= 0, "Set CAP_E2E_FUZZ to run the fuzz test"); + test.setTimeout(30 * 60_000); + if (!extension || !mockServer) + throw new Error("Test harness did not start"); + const context = extension.context; + const server = mockServer; + let seed = Number(process.env.CAP_E2E_FUZZ_SEED ?? Date.now() % 1_000_000); + const random = () => { + seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648; + return seed / 2_147_483_648; + }; + const initialSeed = seed; + const pick = (values: readonly T[]) => + values[Math.floor(random() * values.length)]; + + await configureExtension(await getServiceWorker(context), server.origin); + const messengerPage = await openExtensionMessengerPage( + context, + await getServiceWorker(context), + ); + const capturePage = await context.newPage(); + await capturePage.goto(`${server.origin}/capture.html`); + await capturePage.bringToFront(); + let otherPage: Page | null = null; + + const withTimeout = (promise: Promise, ms: number, label: string) => + Promise.race([ + promise, + new Promise((_, reject) => + setTimeout(() => reject(new Error(`${label} timed out`)), ms), + ), + ]); + const send = (message: Record) => + sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + ...message, + }); + const liveWorker = async () => { + const existing = context + .serviceWorkers() + .find((worker) => worker.url().includes("assets/service-worker.js")); + if (existing) return existing; + const waiting = getServiceWorker(context); + await send({ type: "get-recording-status" }).catch(() => undefined); + return waiting; + }; + const readPhase = async () => { + const response = await withTimeout( + send({ type: "get-recording-status" }), + 5_000, + "get-recording-status", + ); + return response.ok ? (response.status?.phase ?? "unknown") : "failed"; + }; + + const pendingStarts: { startedAt: number; settled: boolean }[] = []; + const log: string[] = []; + const actions: Record Promise> = { + start: async () => { + const entry = { startedAt: Date.now(), settled: false }; + pendingStarts.push(entry); + void send({ type: "start-recording", mode: "fullscreen" }) + .catch(() => undefined) + .finally(() => { + entry.settled = true; + }); + }, + stop: () => send({ type: "stop-recording" }), + pause: () => send({ type: "pause-recording" }), + resume: () => send({ type: "resume-recording" }), + openPanel: () => send({ type: "open-recorder-panel" }), + closeUi: () => send({ type: "close-extension-ui" }), + reloadPage: () => capturePage.reload(), + switchTab: async () => { + if (otherPage && random() < 0.5) { + await capturePage.bringToFront(); + return; + } + otherPage ??= await context.newPage(); + await otherPage.goto(`${server.origin}/capture-audio.html`); + await otherPage.bringToFront(); + }, + closeOtherTab: async () => { + await otherPage?.close(); + otherPage = null; + await capturePage.bringToFront(); + }, + killWorker: async () => { + const session = await context.newCDPSession(capturePage); + await session.send("ServiceWorker.enable"); + await session.send("ServiceWorker.stopAllWorkers"); + await session.detach(); + }, + closeOffscreen: async () => { + const worker = await liveWorker(); + await worker.evaluate(async () => { + const chromeApi = ( + globalThis as unknown as { + chrome: { offscreen: { closeDocument: () => Promise } }; + } + ).chrome; + await chromeApi.offscreen.closeDocument().catch(() => undefined); + }); + }, + toggleMicrophone: async () => { + const response = await send({ type: "bootstrap" }); + const settings = ( + response as { settings?: { microphone?: { enabled: boolean } } } + ).settings; + if (!settings?.microphone) return; + await send({ + type: "settings-updated", + settings: { + ...settings, + microphone: { + ...settings.microphone, + enabled: !settings.microphone.enabled, + }, + }, + }); + }, + }; + const weighted = [ + ...Array(4).fill("start"), + ...Array(3).fill("stop"), + "pause", + "resume", + "openPanel", + "closeUi", + "reloadPage", + "switchTab", + "closeOtherTab", + "killWorker", + "closeOffscreen", + "toggleMicrophone", + ] as const; + + try { + for (let step = 0; step < iterations; step += 1) { + const action = pick(weighted); + const phaseBefore = await readPhase(); + log.push(`${step}:${action}@${phaseBefore}`); + await withTimeout( + actions[action]().catch(() => undefined), + 15_000, + action, + ); + await capturePage.waitForTimeout(Math.floor(random() * 1_500)); + const phase = await readPhase(); + expect(phase).not.toBe("failed"); + const stuck = pendingStarts.find( + (entry) => !entry.settled && Date.now() - entry.startedAt > 20_000, + ); + expect(stuck, "a recording start never settled").toBeUndefined(); + } + + await send({ type: "stop-recording" }).catch(() => undefined); + await expect + .poll(readPhase, { timeout: 30_000 }) + .not.toMatch(/^(creating|recording|paused|uploading)$/); + + server.state.completeBodies = []; + server.state.progressBodies = []; + server.state.initiateBodies = []; + server.state.presignBodies = []; + server.state.uploadBytes = []; + server.state.uploadParts = new Map(); + server.state.uploadHeaders = []; + server.state.videoId = `e2e-${Date.now()}-after-fuzz`; + await capturePage.bringToFront(); + const started = await send({ + type: "start-recording", + mode: "fullscreen", + }); + expect(started).toMatchObject({ ok: true }); + await expect.poll(readPhase).toBe("recording"); + await capturePage.waitForTimeout(2_000); + expect(await send({ type: "stop-recording" })).toMatchObject({ + ok: true, + }); + await expectSuccessfulUpload(messengerPage, server.state); + } catch (error) { + console.log(`fuzz seed ${initialSeed}: ${log.join(" ")}`); + throw error; + } + }); + test("can complete two consecutive recording uploads without stale state", async () => { if (!extension || !mockServer) throw new Error("Test harness did not start"); @@ -638,6 +1306,7 @@ test.describe("extension recording upload", () => { mockServer.state.initiateBodies = []; mockServer.state.presignBodies = []; mockServer.state.uploadBytes = []; + mockServer.state.uploadParts = new Map(); mockServer.state.uploadHeaders = []; mockServer.state.videoId = `e2e-${Date.now()}-second`; @@ -660,3 +1329,221 @@ test.describe("extension recording upload", () => { ); }); }); + +// Reports whether the in-page camera preview is showing moving frames. +const readCameraPreview = async (page: Page) => { + const frame = page + .frames() + .find((candidate) => candidate.url().includes("/camera-preview.html")); + if (!frame) return "no preview frame"; + return frame + .evaluate( + () => + new Promise((resolve) => { + const video = document.querySelector("video"); + if (!video) return resolve("no video element"); + if (video.videoWidth === 0) return resolve("no frames yet"); + const startTime = video.currentTime; + const timer = window.setTimeout(() => { + resolve(video.currentTime > startTime ? "live" : "frozen"); + }, 1_000); + video.requestVideoFrameCallback(() => { + window.clearTimeout(timer); + resolve("live"); + }); + }), + ) + .catch((error: unknown) => + error instanceof Error ? error.message : String(error), + ); +}; + +// Starts and stops a recording with the webcam and microphone on, over and +// over in one browser, as someone recording several clips back to back would. +// Every start must show a live camera preview, report the mic as live and +// upload a recording whose audio carries the mic signal. With +// CAP_E2E_SYSTEM_DEVICES the camera and mic are the OS devices; set +// CAP_E2E_DEVICE_LOOPS to hammer it harder. +test.describe("camera and microphone start", () => { + const loops = Number(process.env.CAP_E2E_DEVICE_LOOPS ?? 3); + + // Display capture of a virtual X screen fails, so with OS devices the + // screen half of the matrix records a tab instead. + const screenMode = useSystemDevices ? "tab" : "fullscreen"; + + for (const mode of [screenMode, "camera"] as const) { + test(`starts the webcam and microphone on every ${mode} recording`, async () => { + test.setTimeout(60_000 + loops * 40_000); + test.skip( + mode === "tab" && !canLaunchHeaded, + "Tab capture needs a headed browser", + ); + // Tab capture cannot use the fake permission UI, and installed + // browsers ignore the media grant seeded into a fresh profile. + test.skip( + mode === "tab" && Boolean(process.env.CAP_E2E_BROWSER), + "Installed browsers need the permission page to grant the mic", + ); + const extension = await launchExtensionContext({ + tabCapture: mode === "tab", + }); + try { + for (let index = 0; index < loops; index += 1) { + const mockServer = await createMockCapServer(); + try { + // Chrome stops an idle service worker between recordings, so + // wake it with an extension page and take a fresh handle. + const wakePage = await extension.context.newPage(); + await wakePage.goto( + `chrome-extension://${unpackedExtensionId}/popup.html`, + ); + const worker = await getServiceWorker(extension.context); + await wakePage.close(); + const startedAt = Date.now(); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + { + ...DEFAULT_RECORDING_OPTIONS, + mode, + microphone: true, + webcam: true, + durationMs: 0, + }, + ); + const recordingAfterMs = Date.now() - startedAt; + + const status = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "get-recording-status", + }); + expect(status).toMatchObject({ + ok: true, + status: { phase: "recording", microphone: "live" }, + }); + await expect + .poll(() => readCameraPreview(capturePage), { timeout: 10_000 }) + .toBe("live"); + const previewAfterMs = Date.now() - startedAt; + + await capturePage.waitForTimeout(3_000); + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + + const { durations, audioPeak, video } = + await probeUploadedRecording(mockServer.state); + if (process.env.CAP_E2E_REPORT_DEVICES) { + console.log( + JSON.stringify({ + mode, + loop: index, + recordingAfterMs, + previewAfterMs, + durations, + audioPeak, + video, + }), + ); + } + expect(durations.video).toBeGreaterThan(2); + expect(durations.audio).not.toBeNull(); + expect(audioPeak).toBeGreaterThan(0.01); + expect(video.movingFrames).toBeGreaterThan(video.frames / 4); + if (mode === "camera") { + // Black video sits at 16 in limited-range luma. + expect(video.meanLuma).toBeGreaterThan(20); + // Camera mode records the camera's own frames. A rig that knows + // its camera's size (CAP_E2E_CAMERA_SIZE=1280x720) checks it. + const [width, height] = (process.env.CAP_E2E_CAMERA_SIZE ?? "") + .split("x") + .map(Number); + if (width && height) { + expect(video).toMatchObject({ width, height }); + } + } + + await capturePage.close(); + await messengerPage.close(); + } finally { + await mockServer.close(); + } + } + } finally { + await extension.cleanup(); + } + }); + } + + test("keeps the camera preview in the recorded tab when a background tab opens", async () => { + // Camera mode keeps the fake permission UI, which tab capture cannot use. + const mode = useSystemDevices ? "camera" : "fullscreen"; + const extension = await launchExtensionContext(); + const mockServer = await createMockCapServer(); + try { + const worker = await getServiceWorker(extension.context); + const { capturePage, messengerPage } = await startRecording( + extension.context, + worker, + mockServer.origin, + { + ...DEFAULT_RECORDING_OPTIONS, + mode, + microphone: true, + webcam: true, + durationMs: 0, + }, + ); + await expect + .poll(() => readCameraPreview(capturePage), { timeout: 10_000 }) + .toBe("live"); + + // Links opened in the background mid-recording load the overlay + // while the recording is live. + const urls = [0, 1, 2].map( + (index) => `${mockServer.origin}/background-tab-${index}`, + ); + await worker.evaluate(async (urls) => { + const chromeApi = ( + globalThis as unknown as { + chrome: { + tabs: { create(options: object): Promise }; + }; + } + ).chrome; + for (const url of urls) { + await chromeApi.tabs.create({ url, active: false }); + } + }, urls); + await expect + .poll( + () => + extension.context + .pages() + .filter((page) => urls.includes(page.url())).length, + ) + .toBe(urls.length); + // Give every background overlay time to load and connect. + await capturePage.waitForTimeout(4_000); + expect(await readCameraPreview(capturePage)).toBe("live"); + for (const page of extension.context.pages()) { + if (!urls.includes(page.url())) continue; + expect(await readCameraPreview(page)).toBe("no preview frame"); + } + + const stopResponse = await sendServiceWorkerMessage(messengerPage, { + target: "service-worker", + type: "stop-recording", + }); + expect(stopResponse).toMatchObject({ ok: true }); + await expectSuccessfulUpload(messengerPage, mockServer.state); + } finally { + await extension.cleanup(); + await mockServer.close(); + } + }); +}); diff --git a/apps/chrome-extension/e2e/webcam-start.spec.ts b/apps/chrome-extension/e2e/webcam-start.spec.ts index dc1344b00f4..eafae2b2343 100644 --- a/apps/chrome-extension/e2e/webcam-start.spec.ts +++ b/apps/chrome-extension/e2e/webcam-start.spec.ts @@ -1,19 +1,10 @@ -import { mkdtemp, rm } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse, } from "node:http"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import { - type BrowserContext, - chromium, - expect, - type Page, - test, -} from "@playwright/test"; +import { type BrowserContext, expect, type Page, test } from "@playwright/test"; +import { launchExtensionBrowser } from "./browser"; type ChromeGlobal = typeof globalThis & { chrome: { @@ -33,8 +24,6 @@ type ChromeGlobal = typeof globalThis & { }; }; -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const extensionPath = path.resolve(__dirname, "../dist"); const SETTINGS_KEY = "cap-extension-settings"; const AUTH_KEY = "cap-extension-auth"; const BOOTSTRAP_CACHE_KEY = "cap-extension-bootstrap-cache"; @@ -163,14 +152,11 @@ const createMockCapServer = async () => { }; }; -const launchExtensionContext = async () => { - const userDataDir = await mkdtemp(path.join(tmpdir(), "cap-repro-e2e-")); - const context = await chromium.launchPersistentContext(userDataDir, { - channel: "chromium", +const launchExtensionContext = () => + launchExtensionBrowser({ + profilePrefix: "cap-repro-e2e-", headless: true, args: [ - `--disable-extensions-except=${extensionPath}`, - `--load-extension=${extensionPath}`, "--allow-http-screen-capture", "--auto-select-tab-capture-source-by-title=Cap Repro Capture Target", "--auto-select-desktop-capture-source=Cap Repro Capture Target", @@ -180,12 +166,6 @@ const launchExtensionContext = async () => { "--use-fake-ui-for-media-stream", ], }); - const cleanup = async () => { - await context.close(); - await rm(userDataDir, { recursive: true, force: true }); - }; - return { context, cleanup }; -}; const getServiceWorker = async (context: BrowserContext) => { const existing = context diff --git a/apps/chrome-extension/src/background/service-worker.test.ts b/apps/chrome-extension/src/background/service-worker.test.ts index fc25cca41b7..b29f474272b 100644 --- a/apps/chrome-extension/src/background/service-worker.test.ts +++ b/apps/chrome-extension/src/background/service-worker.test.ts @@ -175,3 +175,31 @@ describe("recording status synchronization", () => { }); }); }); + +describe("offscreen messaging", () => { + it("retries a request that reached the extension before the offscreen listener", async () => { + const response = new Promise((resolve) => { + onMessage( + { target: "service-worker", type: "pause-recording" }, + {}, + resolve, + ); + }); + answerContexts(true); + await vi.waitFor(() => expect(pollReplies).toHaveLength(1)); + const runtime = chrome.runtime as { lastError?: { message: string } }; + runtime.lastError = { + message: "The message port closed before a response was received.", + }; + pollReplies[0](undefined as unknown as OffscreenResponse); + runtime.lastError = undefined; + + await vi.waitFor(() => expect(contextReplies).toHaveLength(1)); + answerContexts(true); + await vi.waitFor(() => expect(pollReplies).toHaveLength(2)); + const pausedStatus = { ...liveStatus, phase: "paused" as const }; + pollReplies[1]({ ok: true, status: pausedStatus }); + + expect(await response).toMatchObject({ ok: true, status: pausedStatus }); + }); +}); diff --git a/apps/chrome-extension/src/background/service-worker.ts b/apps/chrome-extension/src/background/service-worker.ts index d3d3e65f4ff..1215d083140 100644 --- a/apps/chrome-extension/src/background/service-worker.ts +++ b/apps/chrome-extension/src/background/service-worker.ts @@ -245,12 +245,17 @@ const wait = (durationMs: number) => globalThis.setTimeout(resolve, durationMs); }); +// A runtime message reaches every extension page. When the offscreen document +// is still loading, the panel or welcome page can receive it first and let it +// go unanswered, so a closed port means the same as a missing receiver: the +// offscreen listener never kept the channel open. const isTransientOffscreenMessageError = (error: unknown) => { if (!(error instanceof Error)) return false; const message = error.message.toLowerCase(); return ( message.includes("receiving end does not exist") || - message.includes("could not establish connection") + message.includes("could not establish connection") || + message.includes("message port closed before a response was received") ); }; @@ -820,6 +825,7 @@ const broadcastOverlayHide = async () => { const broadcastOverlayCountdown = async ( seconds: number, durationMs: number, + recordedTabId?: number, ) => { const tabs = await getTabs(); await Promise.all( @@ -827,6 +833,9 @@ const broadcastOverlayCountdown = async ( if (!canInjectIntoTab(tab) || tab.id === undefined) { return undefined; } + // The recorded tab may still be lazy-loading its overlay when the + // countdown starts, so it gets the injecting, retrying send that keeps + // the count visible there. return sendOverlay( tab.id, { @@ -834,7 +843,7 @@ const broadcastOverlayCountdown = async ( seconds, durationMs, }, - false, + tab.id === recordedTabId, ).catch(() => undefined); }), ); @@ -1047,6 +1056,36 @@ const refreshBootstrapInBackground = ( }); }; +const START_PLAN_REFRESH_TIMEOUT_MS = 3000; + +// The recorder enforces the free-plan limit from the bootstrap it is started +// with, and a cached free plan can predate an upgrade, so a free-plan start +// re-checks the plan. A slow or failed request falls back to the cache. +const refreshFreePlanForStart = async ( + settings: ExtensionSettings, + auth: ExtensionAuth, + cached: BootstrapData, +) => { + if (cached.plan.isPro) return cached; + let timer: ReturnType | undefined; + try { + const fresh = await Promise.race([ + fetchBootstrap(settings, auth), + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), START_PLAN_REFRESH_TIMEOUT_MS); + }), + ]); + if (!fresh) return cached; + bootstrapCache = fresh; + await saveCachedBootstrap(fresh).catch(() => undefined); + return fresh; + } catch { + return cached; + } finally { + clearTimeout(timer); + } +}; + const loadSignedInState = async () => { const [settings, auth, pendingAuth, authError] = await Promise.all([ loadSettings(), @@ -1182,6 +1221,7 @@ const resolveMicWarning = async ( const performRecordingStart = async (mode: RecordingMode) => { const { settings, auth, bootstrap } = await requireSignedInState(); + const startBootstrap = refreshFreePlanForStart(settings, auth, bootstrap); externalCaptureAutoPipPending = false; const recordingSettings = settings.capture.recordingMode === mode @@ -1234,7 +1274,7 @@ const performRecordingStart = async (mode: RecordingMode) => { mode, settings: recordingSettings, auth, - bootstrap, + bootstrap: await startBootstrap, tabId, tabStreamId, }); @@ -1431,6 +1471,18 @@ const beginAuthFlow = async (settings: ExtensionSettings) => { const handlePreviewReady = async (tabId?: number) => { if (tabId === undefined) return; if (pendingPreviewTabId !== null && pendingPreviewTabId !== tabId) return; + // Hand-offs to another tab always go through showOverlayInTab, which marks + // the tab pending. A preview that opened on its own somewhere else (a + // background tab whose overlay loaded when the recording started) must not + // take the camera away from the tab it was placed in. + if ( + pendingPreviewTabId === null && + activePreviewTabId !== null && + activePreviewTabId !== tabId + ) { + await hidePreviewTab(tabId); + return; + } activePreviewTabId = tabId; pendingPreviewTabId = null; await hidePreviewTabsExcept(tabId); @@ -1653,6 +1705,9 @@ const handleRequest = async ( type: "toggle-microphone-mute", muted: message.muted, }); + if (response.ok && response.status) { + setRecordingStatusAndBroadcast(response.status); + } return response; } @@ -1772,7 +1827,11 @@ const handleRequest = async ( } if (message.type === "show-countdown") { - await broadcastOverlayCountdown(message.seconds, message.durationMs); + await broadcastOverlayCountdown( + message.seconds, + message.durationMs, + message.tabId, + ); return { ok: true }; } diff --git a/apps/chrome-extension/src/content/overlay.css b/apps/chrome-extension/src/content/overlay.css index 2202cb47c88..fd13f7c1136 100644 --- a/apps/chrome-extension/src/content/overlay.css +++ b/apps/chrome-extension/src/content/overlay.css @@ -266,37 +266,6 @@ border-radius: 0 0 6px; } -.cap-extension-camera-dismiss-btn { - position: absolute; - top: 10px; - right: 10px; - z-index: 12; - display: flex; - align-items: center; - justify-content: center; - width: 26px; - height: 26px; - border: none; - border-radius: 50%; - background: rgba(18, 18, 23, 0.65); - color: #ffffff; - cursor: pointer; - opacity: 0; - pointer-events: none; - transition: all 140ms ease; - backdrop-filter: blur(8px); -} - -.cap-extension-camera-window:hover .cap-extension-camera-dismiss-btn { - opacity: 1; - pointer-events: auto; -} - -.cap-extension-camera-dismiss-btn:hover { - background: rgba(239, 68, 68, 0.9); - transform: scale(1.1); -} - .cap-extension-camera-iframe { position: absolute; inset: 0; diff --git a/apps/chrome-extension/src/content/overlay.tsx b/apps/chrome-extension/src/content/overlay.tsx index f7f2946cef4..d3d09a6c7fb 100644 --- a/apps/chrome-extension/src/content/overlay.tsx +++ b/apps/chrome-extension/src/content/overlay.tsx @@ -1,4 +1,12 @@ -import { X } from "lucide-react"; +import { + Circle, + FlipHorizontal, + Maximize2, + PictureInPicture, + RectangleHorizontal, + Square, + X, +} from "lucide-react"; import { type PointerEvent as ReactPointerEvent, useCallback, @@ -48,6 +56,10 @@ import { replayStartupMessages, setStartupMessages } from "./startup-messages"; const ROOT_ID = "cap-extension-recorder-overlay"; const WINDOW_PADDING = 20; const BAR_HEIGHT = 52; +// The iframe tokens are readable from the host page DOM (they sit in the +// iframe src), so token checks alone cannot authenticate window messages. +// Frames we embed always run on the extension origin; require it too. +const EXTENSION_ORIGIN = new URL(chrome.runtime.getURL("")).origin; const createSecureToken = () => { if (typeof crypto.randomUUID === "function") return crypto.randomUUID(); const bytes = new Uint8Array(16); @@ -61,6 +73,12 @@ const PREVIEW_TOKEN = createSecureToken(); const PREVIEW_URL = chrome.runtime.getURL("camera-preview.html"); const PREVIEW_SRC = `${PREVIEW_URL}#${encodeURIComponent(PREVIEW_TOKEN)}`; const PREVIEW_ERROR_DELAY_MS = 1200; +const PANEL_TOKEN = createSecureToken(); +const PANEL_URL = chrome.runtime.getURL("popup.html"); +const PANEL_SRC = `${PANEL_URL}#${encodeURIComponent(PANEL_TOKEN)}`; +const PANEL_WIDTH = 300; +const PANEL_DEFAULT_HEIGHT = 460; +const PANEL_MARGIN = 16; const CAMERA_MIN_SIZE = 120; const CAMERA_MAX_SIZE = 420; const CAMERA_RESIZE_CORNERS = ["nw", "ne", "sw", "se"] as const; @@ -73,12 +91,14 @@ const FRAME_PERSIST_INTERVAL_MS = 5000; // so the service worker only honours requests from frames whose URL-hash // token was registered by this content script. Registration must complete // before an iframe is rendered, otherwise its first camera connect races the -// registry write. +// registry write. The registry keeps only the most recently registered +// tokens, so frames register again each time they mount instead of trusting +// an old registration that other tabs may have pushed out. let overlayTokensRegistration: Promise | null = null; -const ensureOverlayTokensRegistered = () => { +const registerOverlayTokens = () => { overlayTokensRegistration ??= Promise.all( - [PREVIEW_TOKEN].map((token) => + [PREVIEW_TOKEN, PANEL_TOKEN].map((token) => sendServiceWorkerMessage({ target: "service-worker", type: "register-overlay-token", @@ -86,16 +106,10 @@ const ensureOverlayTokensRegistered = () => { }), ), ) - .then((responses) => { - const registered = responses.every((response) => response.ok); - if (!registered) { - overlayTokensRegistration = null; - } - return registered; - }) - .catch(() => { + .then((responses) => responses.every((response) => response.ok)) + .catch(() => false) + .finally(() => { overlayTokensRegistration = null; - return false; }); return overlayTokensRegistration; }; @@ -314,6 +328,132 @@ const connectCameraPreview = async ( }; }; +type PanelFrameMessage = + | { + source: "cap-extension-panel"; + token: string; + type: "size"; + height: number; + } + | { + source: "cap-extension-panel"; + token: string; + type: "dismiss"; + }; + +const isPanelFrameMessage = (value: unknown): value is PanelFrameMessage => { + if (!value || typeof value !== "object") return false; + const candidate = value as Partial; + if ( + candidate.source !== "cap-extension-panel" || + candidate.token !== PANEL_TOKEN + ) { + return false; + } + if (candidate.type === "dismiss") return true; + return ( + candidate.type === "size" && + typeof candidate.height === "number" && + Number.isFinite(candidate.height) + ); +}; + +function RecorderPanelOverlay({ + open, + onClose, +}: { + open: boolean; + onClose: () => void; +}) { + const [pageVisible, setPageVisible] = useState( + () => document.visibilityState === "visible", + ); + const [contentHeight, setContentHeight] = useState(PANEL_DEFAULT_HEIGHT); + const [viewportHeight, setViewportHeight] = useState( + () => window.innerHeight, + ); + const [tokenReady, setTokenReady] = useState(false); + const shown = open && pageVisible; + + // Hidden tabs never mount the panel, so only register when it is shown. + useEffect(() => { + if (!shown) { + setTokenReady(false); + return; + } + let disposed = false; + void registerOverlayTokens().then((registered) => { + if (!disposed && registered) setTokenReady(true); + }); + return () => { + disposed = true; + }; + }, [shown]); + + useEffect(() => { + const handleVisibility = () => + setPageVisible(document.visibilityState === "visible"); + document.addEventListener("visibilitychange", handleVisibility); + return () => + document.removeEventListener("visibilitychange", handleVisibility); + }, []); + + useEffect(() => { + const handleFrameMessage = (event: MessageEvent) => { + if (event.origin !== EXTENSION_ORIGIN) return; + if (!isPanelFrameMessage(event.data)) return; + if (event.data.type === "size") { + setContentHeight(Math.max(320, Math.ceil(event.data.height))); + return; + } + onClose(); + }; + + window.addEventListener("message", handleFrameMessage); + return () => window.removeEventListener("message", handleFrameMessage); + }, [onClose]); + + useEffect(() => { + if (!open) return; + const handleResize = () => setViewportHeight(window.innerHeight); + handleResize(); + window.addEventListener("resize", handleResize); + return () => window.removeEventListener("resize", handleResize); + }, [open]); + + // Only the tab on screen renders the iframe; hidden tabs keep just the + // shared flag so the panel reappears instantly when they come forward. + // The iframe also waits for its token registration so the panel page can + // verify it was embedded by this extension. + if (!open || !pageVisible || !tokenReady) return null; + + const height = Math.min(contentHeight, viewportHeight - PANEL_MARGIN * 2); + + return ( + <> +