From 646e9178b4dba12ad978f85c834d1cec80dd6046 Mon Sep 17 00:00:00 2001 From: Sri Charan Ratlan Date: Tue, 13 Oct 2026 15:05:57 +0530 Subject: [PATCH] fix(scripts): wait for workspace siblings to appear on npm before publishing bitgo problem/goal bitgo's prepack checked 3 times, 30 seconds apart, that every pinned workspace sibling was visible on npm, then failed the release. Lerna reports a package as published minutes (sometimes over an hour) before the registry shows it, so beta and stable releases often failed here and left partially published releases that needed reruns. implementation details - Replace the fixed 3-attempt check in ensureWorkspaceSiblingVersionsAvailable with a time budget: poll every 30 seconds until all siblings are visible or the budget is spent. Default 20 minutes, override with BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES (0 checks once without waiting). - Log how many siblings are still missing on each poll and, on timeout, fail with the full list of missing versions (same "still unavailable from npm" message as before). - Treat network errors, HTTP 429 and 5xx as transient and retry them up to 5 times in a row during the wait. Other unexpected statuses still fail at once. - Add injectable clock support and tests for waiting past the old limit, timeout, zero budget, transient errors and the environment override. verification - npx mocha --require tsx scripts/tests/generate-bitgo-shrinkwrap.test.ts: 13 passing. - prettier and eslint clean on the changed files. - Checked the wait against the real npm registry with read-only requests: returns at once when versions exist, fails after the budget naming a missing version. Not run in an actual publish. Ticket: WCI-1858 --- scripts/generate-bitgo-shrinkwrap.ts | 142 ++++++++++++++-- .../tests/generate-bitgo-shrinkwrap.test.ts | 159 ++++++++++++++++-- 2 files changed, 270 insertions(+), 31 deletions(-) diff --git a/scripts/generate-bitgo-shrinkwrap.ts b/scripts/generate-bitgo-shrinkwrap.ts index 319b0a9dd56..a2af1bd50c7 100644 --- a/scripts/generate-bitgo-shrinkwrap.ts +++ b/scripts/generate-bitgo-shrinkwrap.ts @@ -5,8 +5,11 @@ * * Runs as bitgo's `prepack` script, gated on BITGO_GENERATE_SHRINKWRAP=true. * Before npm resolves the isolated package, it verifies that every exact workspace - * sibling version is visible on the public registry. Missing versions are retried - * three times to absorb npm propagation delay before the release fails loudly. + * sibling version is visible on the public registry. Versions that are not visible yet + * are polled until they appear, up to a time budget (default 20 minutes, override with + * BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES), because the registry can take minutes to + * hours to show a version that lerna already reported as published. If the budget runs + * out, the release fails loudly and lists what is still missing. */ import fs from 'fs'; @@ -18,8 +21,12 @@ const rootDir = path.resolve(__dirname, '..'); const bitgoDir = path.join(rootDir, 'modules/bitgo'); const modulesDir = path.join(rootDir, 'modules'); const npmRegistryUrl = 'https://registry.npmjs.org'; -const siblingAvailabilityAttempts = 3; +const siblingAvailabilityTimeoutEnvVar = 'BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES'; +const defaultSiblingAvailabilityTimeoutMs = 20 * 60_000; const siblingAvailabilityDelayMs = 30_000; +// A long wait should survive a registry blip, but not a registry that is really down. +const maxConsecutiveRegistryCheckErrors = 5; +const maxMissingSiblingsLogged = 10; type PackageJson = { name?: unknown; @@ -41,6 +48,17 @@ export type RegistryResponse = { export type RegistryFetcher = (url: string) => Promise; export type Sleep = (delayMs: number) => Promise; +/** + * A failed registry lookup. `transient` marks failures worth retrying while waiting + * (network errors, HTTP 429 and 5xx); anything else, such as a 401/403, fails at once. + */ +export class RegistryCheckError extends Error { + constructor(message: string, readonly transient: boolean) { + super(message); + this.name = 'RegistryCheckError'; + } +} + /** * Workspace package versions, read from disk rather than a hardcoded `@bitgo/` * prefix — on alpha/beta these get rescoped to `@bitgo-beta/*`. @@ -92,7 +110,10 @@ export async function findUnavailableWorkspaceSiblings( response = await registryFetcher(url); } catch (error) { const message = error instanceof Error ? error.message : String(error); - throw new Error(`Failed to check npm availability for ${sibling.name}@${sibling.version}: ${message}`); + throw new RegistryCheckError( + `Failed to check npm availability for ${sibling.name}@${sibling.version}: ${message}`, + true + ); } if (response.status === 404 || response.status === 410) { @@ -100,9 +121,10 @@ export async function findUnavailableWorkspaceSiblings( } if (response.status < 200 || response.status >= 300) { const statusText = response.statusText ? ` ${response.statusText}` : ''; - throw new Error( + throw new RegistryCheckError( `Failed to check npm availability for ${sibling.name}@${sibling.version}: ` + - `HTTP ${response.status}${statusText}` + `HTTP ${response.status}${statusText}`, + response.status === 429 || response.status >= 500 ); } return undefined; @@ -115,12 +137,47 @@ export async function findUnavailableWorkspaceSiblings( export interface WorkspaceSiblingAvailabilityOptions { registryFetcher?: RegistryFetcher; sleep?: Sleep; - attempts?: number; + /** Wall-clock source in milliseconds. Injectable for tests. */ + now?: () => number; + /** + * Total time to keep waiting for missing siblings. Defaults to + * BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES, else 20 minutes. Zero checks once and does not wait. + */ + timeoutMs?: number; delayMs?: number; } const sleep: Sleep = (delayMs) => setTimeoutPromise(delayMs); +export function getSiblingAvailabilityTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[siblingAvailabilityTimeoutEnvVar]; + if (raw === undefined || raw.trim() === '') return defaultSiblingAvailabilityTimeoutMs; + const minutes = Number(raw); + if (!Number.isFinite(minutes) || minutes < 0) { + throw new Error(`${siblingAvailabilityTimeoutEnvVar} must be zero or more minutes, got "${raw}".`); + } + return minutes * 60_000; +} + +function formatDuration(ms: number): string { + if (ms < 60_000) return `${Math.round(ms / 1000)}s`; + const minutes = ms / 60_000; + return `${Number.isInteger(minutes) ? minutes : minutes.toFixed(1)} min`; +} + +function describeSiblings(siblings: readonly WorkspaceSibling[]): string { + const names = siblings.map(({ name, version }) => `${name}@${version}`); + const shown = names.slice(0, maxMissingSiblingsLogged).join(', '); + const hidden = names.length - maxMissingSiblingsLogged; + return hidden > 0 ? `${shown} and ${hidden} more` : shown; +} + +/** + * Waits until every sibling version is visible on npm, polling every `delayMs` until `timeoutMs` + * has been spent. Fails with the full list of still-missing versions when the budget runs out. + * Transient registry failures (network errors, 429, 5xx) are retried a few times in a row; + * anything else fails immediately. + */ export async function ensureWorkspaceSiblingVersionsAvailable( siblings: readonly WorkspaceSibling[], options: WorkspaceSiblingAvailabilityOptions = {} @@ -129,22 +186,71 @@ export async function ensureWorkspaceSiblingVersionsAvailable( const registryFetcher = options.registryFetcher ?? fetchRegistry; const sleepFn = options.sleep ?? sleep; - const attempts = options.attempts ?? siblingAvailabilityAttempts; + const now = options.now ?? Date.now; + const timeoutMs = options.timeoutMs ?? getSiblingAvailabilityTimeoutMs(); const delayMs = options.delayMs ?? siblingAvailabilityDelayMs; - if (attempts < 1) { - throw new Error(`Workspace sibling availability checks require at least one attempt, got ${attempts}.`); + if (!(timeoutMs >= 0)) { + throw new Error(`Workspace sibling availability timeout must be zero or more milliseconds, got ${timeoutMs}.`); + } + if (!(delayMs > 0)) { + throw new Error(`Workspace sibling availability delay must be more than zero milliseconds, got ${delayMs}.`); } - for (let attempt = 1; attempt <= attempts; attempt++) { - console.log(`Checking npm availability for workspace siblings (attempt ${attempt}/${attempts}).`); - const missing = await findUnavailableWorkspaceSiblings(siblings, registryFetcher); - if (missing.length === 0) return; + const startedAt = now(); + let checks = 0; + let consecutiveErrors = 0; + let lastMissing: WorkspaceSibling[] | undefined; + let lastError: Error | undefined; + + for (;;) { + checks += 1; + console.log( + `Checking npm availability for workspace siblings (check ${checks}, ${formatDuration( + now() - startedAt + )} elapsed).` + ); + try { + // After the first pass only re-check what was missing; a published version does not disappear. + const missing = await findUnavailableWorkspaceSiblings(lastMissing ?? siblings, registryFetcher); + consecutiveErrors = 0; + lastMissing = missing; + if (missing.length === 0) { + if (checks > 1) { + console.log( + `All ${siblings.length} workspace siblings are visible on npm after ${formatDuration(now() - startedAt)}.` + ); + } + return; + } + console.warn( + `Missing workspace sibling versions (${missing.length} of ${siblings.length}): ${describeSiblings(missing)}` + ); + } catch (error) { + consecutiveErrors += 1; + if ( + !(error instanceof RegistryCheckError) || + !error.transient || + consecutiveErrors > maxConsecutiveRegistryCheckErrors + ) { + throw error; + } + lastError = error; + console.warn( + `Transient npm registry error (${consecutiveErrors}/${maxConsecutiveRegistryCheckErrors}), will retry: ${error.message}` + ); + } - const missingPackages = missing.map(({ name, version }) => `${name}@${version}`); - console.warn(`Missing workspace sibling versions: ${missingPackages.join(', ')}`); - if (attempt === attempts) { + const elapsedMs = now() - startedAt; + if (elapsedMs + delayMs > timeoutMs) { + if (lastMissing === undefined) { + // Only reachable when every check so far failed with a transient registry error. + throw lastError ?? new Error('Workspace sibling availability could not be determined.'); + } + const missingPackages = lastMissing.map(({ name, version }) => `${name}@${version}`); throw new Error( - `Workspace sibling versions are still unavailable from npm after ${attempts} checks: ` + + `Workspace sibling versions are still unavailable from npm after ${checks} checks over ${formatDuration( + elapsedMs + )}: ` + `${missingPackages.join(', ')}. This is likely npm registry propagation delay; ` + 'retry the release in recovery mode after these versions become visible.' ); diff --git a/scripts/tests/generate-bitgo-shrinkwrap.test.ts b/scripts/tests/generate-bitgo-shrinkwrap.test.ts index 282ed03b0fb..578ce193625 100644 --- a/scripts/tests/generate-bitgo-shrinkwrap.test.ts +++ b/scripts/tests/generate-bitgo-shrinkwrap.test.ts @@ -6,6 +6,7 @@ import { describe, it } from 'mocha'; import { ensureWorkspaceSiblingVersionsAvailable, + getSiblingAvailabilityTimeoutMs, getWorkspacePackageVersions, getWorkspaceSiblingDependencies, type WorkspaceSibling, @@ -17,6 +18,17 @@ function response(status: number, statusText?: string) { return { status, statusText }; } +/** A clock that only moves when the code under test sleeps, so waits cost no real time. */ +function fakeClock() { + let time = 0; + return { + now: () => time, + sleep: async (delayMs: number) => { + time += delayMs; + }, + }; +} + describe('generate-bitgo-shrinkwrap workspace sibling availability', function () { it('checks exact scoped package versions and does not wait when all are available', async function () { const secondSibling: WorkspaceSibling = { name: '@bitgo-beta/account-lib', version: '2.3.4-beta.5' }; @@ -79,10 +91,55 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () assert.deepEqual(sleeps, [30_000, 30_000]); }); - it('performs three checks and reports every missing version', async function () { + it('only re-checks siblings that were missing on the previous check', async function () { const secondSibling: WorkspaceSibling = { name: '@bitgo-beta/account-lib', version: '2.3.4-beta.5' }; + const clock = fakeClock(); + const requestsByCheck: string[][] = [[]]; + let accountLibRequests = 0; + + await ensureWorkspaceSiblingVersionsAvailable([sibling, secondSibling], { + registryFetcher: async (url) => { + requestsByCheck[requestsByCheck.length - 1].push(url); + if (!url.includes('account-lib')) return response(200); + accountLibRequests += 1; + return response(accountLibRequests < 3 ? 404 : 200); + }, + now: clock.now, + sleep: async (delayMs) => { + await clock.sleep(delayMs); + requestsByCheck.push([]); + }, + }); + + const accountLibUrl = 'https://registry.npmjs.org/%40bitgo-beta%2Faccount-lib/2.3.4-beta.5'; + assert.equal(requestsByCheck.length, 3); + assert.equal(requestsByCheck[0].length, 2); + assert.deepEqual(requestsByCheck[1], [accountLibUrl]); + assert.deepEqual(requestsByCheck[2], [accountLibUrl]); + }); + + it('keeps waiting past the old three-check limit until the sibling appears', async function () { + const clock = fakeClock(); + let requests = 0; + + await ensureWorkspaceSiblingVersionsAvailable([sibling], { + registryFetcher: async () => { + requests += 1; + return response(requests < 11 ? 404 : 200, requests < 11 ? 'Not Found' : 'OK'); + }, + now: clock.now, + sleep: clock.sleep, + timeoutMs: 30 * 60_000, + }); + + assert.equal(requests, 11); + assert.equal(clock.now(), 10 * 30_000); + }); + + it('stops after the time budget and reports every missing version', async function () { + const secondSibling: WorkspaceSibling = { name: '@bitgo-beta/account-lib', version: '2.3.4-beta.5' }; + const clock = fakeClock(); let requests = 0; - const sleeps: number[] = []; await assert.rejects( () => @@ -91,11 +148,12 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () requests += 1; return response(404, 'Not Found'); }, - sleep: async (delayMs) => { - sleeps.push(delayMs); - }, + now: clock.now, + sleep: clock.sleep, + timeoutMs: 90_000, }), (error: Error) => { + assert.match(error.message, /still unavailable from npm after 4 checks over 1\.5 min/); assert.match(error.message, /@bitgo-beta\/sdk-core@1\.2\.3-beta\.4/); assert.match(error.message, /@bitgo-beta\/account-lib@2\.3\.4-beta\.5/); assert.match(error.message, /recovery mode/); @@ -103,11 +161,11 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () } ); - assert.equal(requests, 6); - assert.deepEqual(sleeps, [30_000, 30_000]); + assert.equal(requests, 8); + assert.equal(clock.now(), 90_000); }); - it('surfaces unexpected registry responses without retrying them', async function () { + it('checks once and does not wait when the time budget is zero', async function () { let requests = 0; const sleeps: number[] = []; @@ -116,22 +174,65 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () ensureWorkspaceSiblingVersionsAvailable([sibling], { registryFetcher: async () => { requests += 1; - return response(500, 'Internal Server Error'); + return response(404, 'Not Found'); }, sleep: async (delayMs) => { sleeps.push(delayMs); }, + timeoutMs: 0, }), - /HTTP 500 Internal Server Error/ + /still unavailable from npm after 1 checks/ ); assert.equal(requests, 1); assert.deepEqual(sleeps, []); }); - it('surfaces network errors without retrying them', async function () { + it('keeps waiting through transient registry errors', async function () { + const clock = fakeClock(); + let requests = 0; + + await ensureWorkspaceSiblingVersionsAvailable([sibling], { + registryFetcher: async () => { + requests += 1; + if (requests === 1) throw new Error('connection reset'); + if (requests === 2) return response(503, 'Service Unavailable'); + return response(200, 'OK'); + }, + now: clock.now, + sleep: clock.sleep, + timeoutMs: 10 * 60_000, + }); + + assert.equal(requests, 3); + assert.equal(clock.now(), 2 * 30_000); + }); + + it('gives up after repeated transient registry errors', async function () { + const clock = fakeClock(); + let requests = 0; + + await assert.rejects( + () => + ensureWorkspaceSiblingVersionsAvailable([sibling], { + registryFetcher: async () => { + requests += 1; + throw new Error('connection reset'); + }, + now: clock.now, + sleep: clock.sleep, + timeoutMs: 60 * 60_000, + }), + /connection reset/ + ); + + assert.equal(requests, 6); + assert.equal(clock.now(), 5 * 30_000); + }); + + it('reports the registry error when the budget runs out before any check succeeds', async function () { + const clock = fakeClock(); let requests = 0; - const sleeps: number[] = []; await assert.rejects( () => @@ -140,17 +241,49 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () requests += 1; throw new Error('connection reset'); }, + now: clock.now, + sleep: clock.sleep, + timeoutMs: 60_000, + }), + /connection reset/ + ); + + assert.equal(requests, 3); + }); + + it('surfaces registry responses that are not transient without retrying them', async function () { + let requests = 0; + const sleeps: number[] = []; + + await assert.rejects( + () => + ensureWorkspaceSiblingVersionsAvailable([sibling], { + registryFetcher: async () => { + requests += 1; + return response(403, 'Forbidden'); + }, sleep: async (delayMs) => { sleeps.push(delayMs); }, }), - /connection reset/ + /HTTP 403 Forbidden/ ); assert.equal(requests, 1); assert.deepEqual(sleeps, []); }); + it('reads the wait budget from the environment', function () { + const envVar = 'BITGO_SIBLING_AVAILABILITY_TIMEOUT_MINUTES'; + + assert.equal(getSiblingAvailabilityTimeoutMs({}), 20 * 60_000); + assert.equal(getSiblingAvailabilityTimeoutMs({ [envVar]: ' ' }), 20 * 60_000); + assert.equal(getSiblingAvailabilityTimeoutMs({ [envVar]: '90' }), 90 * 60_000); + assert.equal(getSiblingAvailabilityTimeoutMs({ [envVar]: '0' }), 0); + assert.throws(() => getSiblingAvailabilityTimeoutMs({ [envVar]: 'abc' }), /must be zero or more minutes/); + assert.throws(() => getSiblingAvailabilityTimeoutMs({ [envVar]: '-5' }), /must be zero or more minutes/); + }); + it('uses package names and versions from workspace manifests', function () { const modulesDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'bitgo-workspace-packages-')); try {