diff --git a/src/commands/test.run.spec.ts b/src/commands/test.run.spec.ts index 1db649d9..d9bfd000 100644 --- a/src/commands/test.run.spec.ts +++ b/src/commands/test.run.spec.ts @@ -3666,6 +3666,187 @@ describe('runTestRunAll — batch fresh run', () => { }; } + it.each(['SIGINT', 'SIGTERM'] as const)( + 'names unverified conflicts in text when no runs were accepted on %s', + async signal => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdout: string[] = []; + const stderr: string[] = []; + const methods: string[] = []; + const error: unknown = await runTestRunAll( + { + profile: 'default', + output: 'text', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + shutdown, + fetchImpl: makeFetch((url, init) => { + methods.push(init.method ?? 'GET'); + if (init.method === 'POST') { + return { + body: { + ...BATCH_FRESH_RESP, + accepted: [], + conflicts: [ + { testId: 'test_busy', currentRunId: 'run_busy' }, + { testId: 'test_busy_2', currentRunId: 'run_busy_2' }, + { testId: 'test_unknown', reason: 'in_flight' }, + ], + }, + }; + } + if (url.includes('/runs/')) { + shutdown.interrupt(signal); + throw shutdown.signal.reason; + } + return { body: { environments: [] } }; + }), + stdout: line => stdout.push(line), + stderr: line => stderr.push(line), + }, + ).catch((caught: unknown) => caught); + expect(error).toBe(shutdown.signal.reason); + expect(shutdown.isArmed).toBe(false); + expect(stdout.join('\n')).toBe( + 'run_busy unverified (not attached)\nrun_busy_2 unverified (not attached)', + ); + expect(stderr.join('\n')).not.toContain('[auto-resume]'); + expect(methods.filter(method => method !== 'GET')).toEqual(['POST']); + }, + ); + + it.each(['SIGINT', 'SIGTERM'] as const)( + 'keeps unsent filtered test IDs when dispatch stops before metadata interruption on %s', + async signal => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdout: string[] = []; + const methods: string[] = []; + const tests = Array.from({ length: 51 }, (_, i) => ({ + id: `test_${i}`, + name: `selected ${i}`, + projectId: 'project_be', + type: i >= 2 && i < 50 ? 'frontend' : 'backend', + })); + let dispatches = 0; + const error: unknown = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + nameFilter: 'selected', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + shutdown, + fetchImpl: makeFetch((url, init) => { + methods.push(init.method ?? 'GET'); + if (url.includes('/tests?')) return { body: { items: tests, nextToken: null } }; + if (init.method === 'POST') { + dispatches++; + if (dispatches > 1) return errorBody('PRECONDITION_FAILED'); + return { + body: { + ...BATCH_FRESH_RESP, + accepted: [ + { testId: 'test_0', runId: 'run_new', enqueuedAt: '2026-06-09T10:00:00Z' }, + ], + conflicts: [{ testId: 'test_1', currentRunId: 'run_busy' }], + skippedFrontend: tests.slice(2, 50).map(test => test.id), + }, + }; + } + if (url.includes('/runs/run_busy')) { + shutdown.interrupt(signal); + throw shutdown.signal.reason; + } + return { body: { environments: [] } }; + }), + stdout: line => stdout.push(line), + stderr: () => {}, + }, + ).catch((caught: unknown) => caught); + expect(error).toBe(shutdown.signal.reason); + expect(shutdown.isArmed).toBe(false); + expect(JSON.parse(stdout.join('\n'))).toMatchObject({ + accepted: [{ testId: 'test_0', runId: 'run_new', status: 'running' }], + conflicts: [{ testId: 'test_1', currentRunId: 'run_busy' }], + notRunTestIds: ['test_50'], + skippedFrontend: tests.slice(2, 50).map(test => test.id), + }); + expect(methods.filter(method => method !== 'GET')).toEqual(['POST', 'POST']); + }, + ); + + it.each(['SIGINT', 'SIGTERM'] as const)( + 'preserves batch receipts and disarms the conflict metadata scope on %s', + async signal => { + const { credentialsPath } = makeCreds(); + const shutdown = new ShutdownController(); + const stdout: string[] = []; + const stderr: string[] = []; + let armedAtRead = false; + const response: BatchRunFreshResponse = { + ...BATCH_FRESH_RESP, + conflicts: [{ testId: 'test_busy', currentRunId: 'run_busy', reason: 'in_flight' }], + skippedFrontend: ['test_frontend'], + skippedIntegration: [{ testId: 'test_integration' }], + }; + const error: unknown = await runTestRunAll( + { + profile: 'default', + output: 'json', + debug: false, + projectId: 'project_be', + wait: true, + timeoutSeconds: 60, + maxConcurrency: 5, + }, + { + credentialsPath, + shutdown, + fetchImpl: makeFetch((url, init) => { + if (init.method === 'POST') return { body: response }; + if (url.includes('/runs/run_busy')) { + armedAtRead = shutdown.isArmed; + shutdown.interrupt(signal); + throw shutdown.signal.reason; + } + return { body: { environments: [] } }; + }), + stdout: line => stdout.push(line), + stderr: line => stderr.push(line), + }, + ).catch((caught: unknown) => caught); + expect(error).toBe(shutdown.signal.reason); + expect(armedAtRead).toBe(true); + expect(shutdown.isArmed).toBe(false); + expect(JSON.parse(stdout.join('\n'))).toEqual({ + accepted: response.accepted.map(e => ({ + testId: e.testId, + runId: e.runId, + status: 'running', + })), + conflicts: response.conflicts, + deferred: [], + skippedFrontend: response.skippedFrontend, + skippedIntegration: response.skippedIntegration, + }); + expect(stderr.join('\n')).toContain('testsprite test wait run_fresh_01 run_fresh_02'); + }, + ); + it('run --all --wait: a NOT_FOUND member poll → exit 4 with error.code (was folded to exit 1 before this PR)', async () => { const { credentialsPath } = makeCreds(); let caughtError: unknown; diff --git a/src/commands/test.ts b/src/commands/test.ts index aceee925..e116cb8a 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -10986,6 +10986,7 @@ export async function runTestRunAll( // resolvable in-flight run stay hard (nothing to wait on). // An in-flight run may belong to another deployment/environment. CI can // only use runs accepted by this invocation to judge its own change. + const metadataConflicts = conflicts; // A `no_producer` / `producer_in_flight` conflict's `currentRunId` is the // result the server seeded BLOCKED before dispatch. It is reported with the // server's reason and exit 6, as a single `test run` reports it, not polled. @@ -11025,6 +11026,8 @@ export async function runTestRunAll( } return listing; }; + // Keep the metadata stage graceful even between tracked HTTP requests. + const disarmResolve = shutdownOf(deps).arm(); const resumedOrRefused = await Promise.all( resumableConflicts.map( async (c): Promise<{ entry: BatchRunFreshAccepted } | { refused: RunConflict }> => { @@ -11091,7 +11094,43 @@ export async function runTestRunAll( return { entry: { testId: c.testId, runId: inFlightRunId, enqueuedAt: notBefore } }; }, ), - ); + ) + .catch((err: unknown) => { + if (err instanceof InterruptError) { + const partialResults: CliBatchRunFreshResult[] = accepted.map(e => ({ + testId: e.testId, + runId: e.runId, + status: 'running', + })); + out.print( + { + accepted: partialResults, + conflicts: metadataConflicts, + deferred, + skippedFrontend, + skippedIntegration, + ...(notRunTestIds.length > 0 ? { notRunTestIds } : {}), + }, + () => + partialResults.length > 0 + ? partialResults.map(r => `${r.runId} ${r.status}`).join('\n') + : resumableConflicts + .map(c => `${c.currentRunId} unverified (not attached)`) + .join('\n'), + ); + if (accepted.length > 0) { + err.runWaitContext = true; + stderrFn( + interruptDetachMessage( + err, + accepted.map(e => e.runId), + ), + ); + } + } + throw err; + }) + .finally(disarmResolve); const resumableEntries: BatchRunFreshAccepted[] = []; for (const outcome of resumedOrRefused) { if ('entry' in outcome) { diff --git a/test/e2e/batch-resume-signal.e2e.test.ts b/test/e2e/batch-resume-signal.e2e.test.ts new file mode 100644 index 00000000..cbfb544c --- /dev/null +++ b/test/e2e/batch-resume-signal.e2e.test.ts @@ -0,0 +1,326 @@ +import { spawn } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { describe, expect, it } from 'vitest'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const binary = join(root, 'dist/index.js'); +const batch = { + accepted: [{ testId: 'test_new', runId: 'run_new', enqueuedAt: '2026-06-09T10:00:00Z' }], + conflicts: [{ testId: 'test_busy', currentRunId: 'run_busy', reason: 'in_flight' }], + deferred: [], + skippedFrontend: ['test_frontend'], + skippedIntegration: [{ testId: 'test_integration' }], +}; + +function passedRun(runId: string) { + return { + runId, + testId: runId === 'run_new' ? 'test_new' : 'test_busy', + projectId: 'project_signal', + userId: 'user_signal', + status: 'passed', + source: 'cli', + createdAt: '2026-06-09T10:00:00Z', + startedAt: '2026-06-09T10:00:01Z', + finishedAt: '2026-06-09T10:00:02Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + createdFrom: 'cli', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }; +} + +type Stage = 'request' | 'body' | 'idle' | 'complete'; + +async function runBatch( + stage: Stage, + signal: 'SIGINT' | 'SIGTERM', + output: 'json' | 'text', + scenario: 'mixed' | 'conflicts-only' | 'stopped-dispatch' = 'mixed', +) { + const directory = mkdtempSync(join(tmpdir(), 'batch-resume-signal-')); + const preload = join(directory, 'preload.mjs'); + // The idle seam pauses getRun AFTER its real HTTP request has settled. It + // makes the otherwise tiny gap deterministic, with no tracked request left + // for the disarmed signal handler to drain. Signals and handlers stay real. + writeFileSync( + preload, + ` + import os from 'node:os'; + import { syncBuiltinESMExports } from 'node:module'; + import { HttpClient } from ${JSON.stringify(pathToFileURL(join(root, 'dist/lib/http.js')).href)}; + import { globalShutdown } from ${JSON.stringify(pathToFileURL(join(root, 'dist/lib/interrupt.js')).href)}; + os.homedir = () => ${JSON.stringify(directory)}; + syncBuiltinESMExports(); + const fetch = globalThis.fetch; + globalThis.fetch = (input, init) => { + const url = new URL(typeof input === 'string' || input instanceof URL ? input : input.url); + if (url.origin !== process.env.TESTSPRITE_API_URL) throw new Error('Unexpected origin'); + return fetch(input, init); + }; + if (${JSON.stringify(stage)} === 'idle') { + const getRun = HttpClient.prototype.getRun; + HttpClient.prototype.getRun = async function(id, options) { + const result = await getRun.call(this, id, options); + if (id === 'run_busy' && options?.waitSeconds === undefined) { + await new Promise((resolve, reject) => { + const keepAlive = setInterval(() => {}, 1000); + globalShutdown.signal.addEventListener('abort', () => { clearInterval(keepAlive); reject(globalShutdown.signal.reason); }, { once: true }); + process.send({ idle: true, critical: globalShutdown.hasCriticalOperations }); + }); + } + return result; + }; + } + `, + ); + const requests: Array<{ method: string; path: string }> = []; + const filteredIds = [ + 'test_new', + 'test_busy', + ...Array.from({ length: 49 }, (_, i) => `test_${i + 2}`), + ]; + let dispatches = 0; + let ready!: () => void; + const reached = new Promise(resolveReady => { + ready = resolveReady; + }); + const server = createServer((req, res) => { + const url = new URL(req.url!, 'http://localhost'); + const path = url.pathname.replace('/api/cli/v1', ''); + requests.push({ method: req.method!, path: req.url! }); + let body: unknown; + if (path === '/projects/project_signal/env') body = { environments: [] }; + else if (path === '/tests') { + body = { + items: filteredIds.map((id, i) => ({ + id, + name: `selected ${id}`, + projectId: 'project_signal', + type: i >= 2 && i < 50 ? 'frontend' : 'backend', + })), + nextToken: null, + }; + } else if (path === '/tests/batch/run' && req.method === 'POST') { + dispatches++; + if (scenario === 'stopped-dispatch' && dispatches > 1) { + res.writeHead(412, { 'content-type': 'application/json' }); + res.end( + JSON.stringify({ + error: { + code: 'PRECONDITION_FAILED', + message: 'Dispatch stopped', + requestId: 'fixture', + }, + }), + ); + return; + } + body = { + ...batch, + ...(scenario === 'conflicts-only' ? { accepted: [] } : {}), + ...(scenario === 'stopped-dispatch' + ? { skippedFrontend: filteredIds.slice(2, 50), skippedIntegration: [] } + : {}), + }; + } else if (path.startsWith('/runs/')) { + if ( + path === '/runs/run_busy' && + !url.searchParams.has('waitSeconds') && + (stage === 'request' || stage === 'body') + ) { + if (stage === 'body') { + res.writeHead(200, { 'content-type': 'application/json' }); + res.write('{"runId":"run_busy"'); + } + ready(); + return; + } + body = passedRun(path.slice('/runs/'.length)); + } else { + res.writeHead(500); + res.end('{}'); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify(body)); + }); + await new Promise(resolveListen => server.listen(0, '127.0.0.1', resolveListen)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('No loopback port'); + const child = spawn( + process.execPath, + [ + '--import', + preload, + binary, + 'test', + 'run', + '--all', + '--project', + 'project_signal', + '--wait', + ...(scenario === 'stopped-dispatch' ? ['--filter', 'selected'] : []), + '--timeout', + '30', + '--max-concurrency', + '1', + '--output', + output, + ], + { + env: { + ...process.env, + TESTSPRITE_API_URL: `http://127.0.0.1:${address.port}`, + TESTSPRITE_API_KEY: 'sk-user-e2e-batch-signal', + TESTSPRITE_NO_TELEMETRY: '1', + TESTSPRITE_NO_UPDATE_NOTIFIER: '1', + TESTSPRITE_NO_SKILL_WARNING: '1', + }, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }, + ); + let stdout = ''; + let stderr = ''; + let idleCritical: unknown; + child.stdout!.on('data', (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr!.on('data', (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on('message', (message: { idle?: boolean; critical?: boolean }) => { + if (message.idle) { + idleCritical = message.critical; + ready(); + } + }); + const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>( + (resolveClose, reject) => { + child.on('error', reject); + child.on('close', (code, exitSignal) => resolveClose({ code, signal: exitSignal })); + }, + ); + let timer: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`Batch fixture timed out: ${stderr}`)), 10_000); + }); + try { + if (stage !== 'complete') { + await Promise.race([ + reached, + closed.then(result => { + throw new Error(`Exited before metadata signal: ${JSON.stringify(result)} ${stderr}`); + }), + deadline, + ]); + child.kill(signal); + } + const result = await Promise.race([closed, deadline]); + return { ...result, stdout, stderr, requests, idleCritical }; + } finally { + clearTimeout(timer); + child.kill('SIGKILL'); + await closed; + server.closeAllConnections(); + await new Promise(resolveClose => server.close(() => resolveClose())); + rmSync(directory, { recursive: true, force: true }); + } +} + +// Windows does not deliver POSIX signals to the Node handler. +describe.skipIf(process.platform === 'win32')('batch conflict metadata signal receipts', () => { + for (const stage of ['request', 'body', 'idle'] as const) { + for (const signal of ['SIGINT', 'SIGTERM'] as const) { + for (const output of ['json', 'text'] as const) { + it(`${stage}, ${signal}, ${output}: preserves acknowledged receipts without cancelling`, async () => { + const result = await runBatch(stage, signal, output); + expect(result.code).toBe(signal === 'SIGINT' ? 130 : 143); + expect(result.signal).toBeNull(); + if (stage === 'idle') expect(result.idleCritical).toBe(false); + if (output === 'json') { + expect(JSON.parse(result.stdout)).toEqual({ + accepted: [{ testId: 'test_new', runId: 'run_new', status: 'running' }], + conflicts: batch.conflicts, + deferred: [], + skippedFrontend: batch.skippedFrontend, + skippedIntegration: batch.skippedIntegration, + }); + expect(result.stderr).toContain('"code": "INTERRUPTED"'); + } else { + expect(result.stdout).toContain('run_new running'); + } + expect(result.stderr).toContain(`Interrupted (${signal})`); + expect(result.stderr).toContain('testsprite test wait run_new'); + expect(result.requests.filter(r => r.method !== 'GET')).toEqual([ + { method: 'POST', path: '/api/cli/v1/tests/batch/run' }, + ]); + expect(result.requests.some(r => r.path.includes('waitSeconds'))).toBe(false); + }, 15_000); + } + } + } + + it('completes the normal accepted plus adopted run without changing resume behaviour', async () => { + const result = await runBatch('complete', 'SIGINT', 'json'); + expect(result.code).toBe(0); + const payload = JSON.parse(result.stdout); + expect( + payload.accepted.map((row: { runId: string; status: string }) => [row.runId, row.status]), + ).toEqual([ + ['run_new', 'passed'], + ['run_busy', 'passed'], + ]); + expect(payload.conflicts).toEqual([]); + expect(result.stderr).not.toContain('Interrupted'); + }, 15_000); + + for (const signal of ['SIGINT', 'SIGTERM'] as const) { + for (const output of ['json', 'text'] as const) { + it(`no accepted runs, ${signal}, ${output}: preserves unverified conflict IDs`, async () => { + const result = await runBatch('request', signal, output, 'conflicts-only'); + expect(result.code).toBe(signal === 'SIGINT' ? 130 : 143); + expect(result.signal).toBeNull(); + if (output === 'text') { + expect(result.stdout.trim()).toBe('run_busy unverified (not attached)'); + } else { + expect(JSON.parse(result.stdout)).toMatchObject({ + accepted: [], + conflicts: batch.conflicts, + }); + expect(JSON.parse(result.stdout)).not.toHaveProperty('notRunTestIds'); + } + expect(result.stderr).not.toContain('[auto-resume]'); + expect(result.requests.filter(r => r.method !== 'GET')).toEqual([ + { method: 'POST', path: '/api/cli/v1/tests/batch/run' }, + ]); + expect(result.requests.some(r => r.path.includes('waitSeconds'))).toBe(false); + }, 15_000); + } + + it(`stopped dispatch then metadata ${signal}: JSON names tests never sent`, async () => { + const result = await runBatch('request', signal, 'json', 'stopped-dispatch'); + expect(result.code).toBe(signal === 'SIGINT' ? 130 : 143); + expect(result.signal).toBeNull(); + expect(JSON.parse(result.stdout)).toMatchObject({ + accepted: [{ testId: 'test_new', runId: 'run_new', status: 'running' }], + conflicts: batch.conflicts, + notRunTestIds: ['test_50'], + }); + expect(result.stderr).toContain('1 test was not dispatched: test_50'); + expect(result.requests.filter(r => r.method !== 'GET')).toEqual([ + { method: 'POST', path: '/api/cli/v1/tests/batch/run' }, + { method: 'POST', path: '/api/cli/v1/tests/batch/run' }, + ]); + expect(result.requests.some(r => r.path.includes('waitSeconds'))).toBe(false); + }, 15_000); + } +});