diff --git a/.github/workflows/assessment-test.yml b/.github/workflows/assessment-test.yml new file mode 100644 index 00000000..99c9bab2 --- /dev/null +++ b/.github/workflows/assessment-test.yml @@ -0,0 +1,19 @@ +name: Assessment tests + +on: + pull_request: + paths: + - scripts/assessment/** + - .github/workflows/assessment-test.yml + +permissions: + contents: read + +jobs: + assessment-test: + name: ASSESSMENT tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/npm-ci-via-cached-nvmrc + - run: npm run test:assessment diff --git a/.prettierignore b/.prettierignore index f238ae85..5fc35c7a 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,5 @@ /docs/localization/ja + +# Test fixtures are frozen inputs; formatting would change the bytes +# the assessment collection tests hash. +/scripts/assessment/test/fixtures/ diff --git a/package.json b/package.json index c0a727fc..fb39b47a 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "precheck:links": "npm run build", "seq": "bash -c 'for cmd in \"$@\"; do npm run $cmd || exit 1; done' - ", "serve": "npm run docus:serve", + "test:assessment": "node --test scripts/assessment/test/*.test.mjs", "test:unit": "node --experimental-strip-types --test 'lib/**/*.test.mts'", "test": "npm run check && npm run test:unit", "typecheck": "tsc", diff --git a/scripts/assessment/collect.mjs b/scripts/assessment/collect.mjs new file mode 100644 index 00000000..b93da451 --- /dev/null +++ b/scripts/assessment/collect.mjs @@ -0,0 +1,179 @@ +#!/usr/bin/env node +// Deterministic data collection for documentation assessments. +// Inventories one or more repository checkouts, extracts markdown +// links, optionally fetches sites and checks external links, and +// writes a content-hash manifest so every quantitative claim traces +// to a committed, re-runnable step and post-collection edits are +// detectable. +// +// node scripts/assessment/collect.mjs --repo [--repo ] +// [--site ] [--check-links] --out +// node scripts/assessment/collect.mjs verify --out + +import path from 'node:path'; +import process from 'node:process'; +import { buildInventory, markdownPaths } from './lib/inventory.mjs'; +import { buildLinkReport } from './lib/links.mjs'; +import { resolveHeadSha } from './lib/git.mjs'; +import { checkLinks, fetchSites } from './lib/sitefetch.mjs'; +import { verifyManifest, writeCollection } from './lib/manifest.mjs'; +import { stableStringify } from './lib/util.mjs'; + +const USAGE = + 'usage: collect.mjs [verify] --repo [--site ] [--check-links] --out '; + +function takeValue(argv, i, flag) { + const value = argv[i + 1]; + if (value == null || value.startsWith('--')) { + throw new Error(`missing value for ${flag}`); + } + return value; +} + +function parseArgs(argv) { + const args = { + verify: false, + repos: [], + sites: [], + checkLinks: false, + out: null, + }; + let i = 0; + if (argv[0] === 'verify') { + args.verify = true; + i = 1; + } + while (i < argv.length) { + const flag = argv[i]; + if (flag === '--repo') { + args.repos.push(takeValue(argv, i, flag)); + i += 2; + } else if (flag === '--site') { + args.sites.push(takeValue(argv, i, flag)); + i += 2; + } else if (flag === '--out') { + args.out = takeValue(argv, i, flag); + i += 2; + } else if (flag === '--check-links') { + args.checkLinks = true; + i += 1; + } else throw new Error(`unknown flag: ${flag}`); + } + if (!args.out) throw new Error('missing --out'); + if (!args.verify && args.repos.length === 0 && args.sites.length === 0) { + throw new Error('nothing to collect: pass --repo or --site'); + } + for (const repoPath of args.repos) { + if (outputRelPath(repoPath, args.out) === '') { + throw new Error('--out must not be a repository root'); + } + } + return args; +} + +// Relative posix path of the output directory inside a repository, so +// a rerun never inventories its own previous outputs; null when the +// output directory lives outside the repository. +function outputRelPath(repoPath, out) { + const rel = path.relative(path.resolve(repoPath), path.resolve(out)); + if (rel === '') return ''; + if (rel.startsWith('..') || path.isAbsolute(rel)) return null; + return rel.split(path.sep).join('/'); +} + +// The manifest records the command without --out: the output location +// is wherever the manifest lives, and omitting it keeps two runs into +// different directories byte-identical. +function commandLine(args) { + const parts = ['collect.mjs']; + for (const repo of args.repos) parts.push('--repo', repo); + for (const site of args.sites) parts.push('--site', site); + if (args.checkLinks) parts.push('--check-links'); + return parts.join(' '); +} + +async function collect(args) { + const outputs = {}; + const repos = []; + const repoReports = []; + const linkReports = []; + for (const repoPath of args.repos) { + const excludeRel = outputRelPath(repoPath, args.out); + const inventory = buildInventory(repoPath, { + exclude: excludeRel ? [excludeRel] : [], + }); + const report = buildLinkReport(repoPath, markdownPaths(inventory)); + repos.push({ path: repoPath, sha: resolveHeadSha(repoPath) }); + repoReports.push({ path: repoPath, ...inventory }); + linkReports.push({ path: repoPath, ...report }); + } + outputs['inventory.json'] = stableStringify({ repos: repoReports }) + '\n'; + outputs['links.json'] = stableStringify({ repos: linkReports }) + '\n'; + + const sites = []; + if (args.sites.length > 0) { + const fetched = await fetchSites(args.sites); + fetched.forEach((entry, index) => { + const { body, ...meta } = entry; + if (body) { + // A body implies the fetch succeeded, so the URL parses; on + // failure the entry (including an unparseable URL) is written + // through as evidence untouched. + const host = new URL(entry.url).host; + const name = `sites/${String(index + 1).padStart(3, '0')}-${host}.body`; + outputs[name] = body; + meta.bodyFile = name; + } + sites.push(meta); + }); + outputs['site-fetches.json'] = stableStringify({ sites }) + '\n'; + } + + if (args.checkLinks) { + const external = [ + ...new Set(linkReports.flatMap((report) => report.external)), + ].sort(); + outputs['link-status.json'] = + stableStringify({ checked: await checkLinks(external) }) + '\n'; + } + + writeCollection(args.out, { + commands: [commandLine(args)], + repos, + sites, + outputs, + }); + process.stdout.write(`collected into ${args.out}\n`); +} + +function verify(args) { + const result = verifyManifest(args.out); + if (result.ok) { + process.stdout.write('manifest ok\n'); + return; + } + for (const name of result.mismatched) { + process.stderr.write(`mismatched: ${name}\n`); + } + for (const name of result.missing) { + process.stderr.write(`missing: ${name}\n`); + } + for (const name of result.unexpected) { + process.stderr.write(`unexpected: ${name}\n`); + } + process.exit(1); +} + +async function main() { + let args; + try { + args = parseArgs(process.argv.slice(2)); + } catch (err) { + process.stderr.write(`${err.message}\n${USAGE}\n`); + process.exit(2); + } + if (args.verify) verify(args); + else await collect(args); +} + +await main(); diff --git a/scripts/assessment/lib/git.mjs b/scripts/assessment/lib/git.mjs new file mode 100644 index 00000000..11997089 --- /dev/null +++ b/scripts/assessment/lib/git.mjs @@ -0,0 +1,17 @@ +import { execFileSync } from 'node:child_process'; + +function defaultExec(cmd, args) { + return execFileSync(cmd, args, { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); +} + +export function resolveHeadSha(dir, execImpl = defaultExec) { + try { + const out = execImpl('git', ['-C', dir, 'rev-parse', 'HEAD']).trim(); + return /^[0-9a-f]{40}$/i.test(out) ? out : null; + } catch { + return null; + } +} diff --git a/scripts/assessment/lib/inventory.mjs b/scripts/assessment/lib/inventory.mjs new file mode 100644 index 00000000..4feab3fd --- /dev/null +++ b/scripts/assessment/lib/inventory.mjs @@ -0,0 +1,79 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { sha256Hex } from './util.mjs'; + +const MARKDOWN_EXTENSIONS = new Set(['.md', '.mdx', '.markdown']); +const SKIPPED_DIRECTORIES = new Set(['.git', 'node_modules']); + +function walk(root, rel, { files, symlinks, exclude }) { + const entries = fs.readdirSync(path.join(root, rel), { + withFileTypes: true, + }); + for (const entry of entries) { + const relPath = rel ? `${rel}/${entry.name}` : entry.name; + if (exclude.has(relPath)) continue; + if (entry.isSymbolicLink()) { + // Recorded, never followed: the link target string is the + // content, so links pointing outside the repository cannot pull + // outside bytes into the inventory. + symlinks.push(relPath); + } else if (entry.isDirectory()) { + if (!SKIPPED_DIRECTORIES.has(entry.name)) { + walk(root, relPath, { files, symlinks, exclude }); + } + } else if (entry.isFile()) { + files.push(relPath); + } + } +} + +export function buildInventory(root, { exclude = [] } = {}) { + const filePaths = []; + const symlinkPaths = []; + walk(root, '', { + files: filePaths, + symlinks: symlinkPaths, + exclude: new Set(exclude), + }); + filePaths.sort(); + symlinkPaths.sort(); + const files = filePaths.map((relPath) => { + const content = fs.readFileSync(path.join(root, relPath)); + return { path: relPath, bytes: content.length, sha256: sha256Hex(content) }; + }); + const symlinks = symlinkPaths.map((relPath) => { + const target = fs.readlinkSync(path.join(root, relPath)); + return { + path: relPath, + target, + bytes: Buffer.byteLength(target), + sha256: sha256Hex(target), + }; + }); + const byExtension = {}; + let markdownCount = 0; + for (const file of files) { + const ext = path.posix.extname(file.path).toLowerCase(); + if (ext) byExtension[ext] = (byExtension[ext] ?? 0) + 1; + if (MARKDOWN_EXTENSIONS.has(ext)) markdownCount += 1; + } + return { + files, + symlinks, + totals: { + fileCount: files.length, + symlinkCount: symlinks.length, + byteCount: files.reduce((n, f) => n + f.bytes, 0), + markdownCount, + byExtension, + }, + }; +} + +export function markdownPaths(inventory) { + return inventory.files + .map((f) => f.path) + .filter((p) => + MARKDOWN_EXTENSIONS.has(path.posix.extname(p).toLowerCase()), + ); +} diff --git a/scripts/assessment/lib/links.mjs b/scripts/assessment/lib/links.mjs new file mode 100644 index 00000000..fe995b75 --- /dev/null +++ b/scripts/assessment/lib/links.mjs @@ -0,0 +1,84 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +// Deliberate limitations, documented rather than hidden: destinations +// with unescaped parentheses need the angle-bracket form to be seen, +// and indented (non-fenced) code blocks are still scanned. A markdown +// AST would lift both at the cost of a new dependency. +const INLINE_LINK = /!?\[[^\]]*\]\(([^)\s<][^)\s]*)(?:\s+"[^"]*")?\)/g; +const ANGLE_LINK = /!?\[[^\]]*\]\(<([^>]+)>(?:\s+"[^"]*")?\)/g; +const REFERENCE_DEFINITION = /^\s*\[[^\]]+\]:\s*(\S+)/gm; +const AUTOLINK = /<(https?:\/\/[^>\s]+)>/g; + +const FENCE = /^ {0,3}(`{3,}|~{3,})/; + +// Blanks out fenced code blocks and inline code spans so example +// links inside them (common in documentation about documentation) do +// not pollute the inventory. +function stripCodeRegions(markdown) { + const out = []; + let fence = null; + for (const line of markdown.split('\n')) { + const opener = line.match(FENCE); + if (fence) { + if ( + opener && + opener[1][0] === fence[0] && + opener[1].length >= fence.length + ) { + fence = null; + } + out.push(''); + } else if (opener) { + fence = opener[1]; + out.push(''); + } else { + out.push(line.replace(/(`+)[^`]*\1/g, ' ')); + } + } + return out.join('\n'); +} + +function classify(url) { + if (url.startsWith('#')) return 'anchor'; + if (/^https?:\/\//i.test(url)) return 'external'; + return 'internal'; +} + +export function extractLinks(markdown) { + const source = stripCodeRegions(markdown); + const urls = new Set(); + for (const pattern of [ + INLINE_LINK, + ANGLE_LINK, + REFERENCE_DEFINITION, + AUTOLINK, + ]) { + for (const match of source.matchAll(pattern)) { + urls.add(match[1]); + } + } + return [...urls].sort().map((url) => ({ url, kind: classify(url) })); +} + +export function buildLinkReport(root, markdownRelPaths) { + const perFile = {}; + const byKind = { + external: new Set(), + internal: new Set(), + anchor: new Set(), + }; + for (const relPath of markdownRelPaths) { + const links = extractLinks( + fs.readFileSync(path.join(root, relPath), 'utf8'), + ); + perFile[relPath] = links; + for (const link of links) byKind[link.kind].add(link.url); + } + return { + perFile, + external: [...byKind.external].sort(), + internal: [...byKind.internal].sort(), + anchors: [...byKind.anchor].sort(), + }; +} diff --git a/scripts/assessment/lib/manifest.mjs b/scripts/assessment/lib/manifest.mjs new file mode 100644 index 00000000..2289701e --- /dev/null +++ b/scripts/assessment/lib/manifest.mjs @@ -0,0 +1,101 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { sha256Hex, stableStringify } from './util.mjs'; + +export const MANIFEST_NAME = 'manifest.json'; + +function hashEntry(content) { + const buf = Buffer.from(content); + return { bytes: buf.length, sha256: sha256Hex(buf) }; +} + +export function buildManifest({ commands, repos, sites, outputs }) { + const hashed = {}; + for (const name of Object.keys(outputs).sort()) { + hashed[name] = hashEntry(outputs[name]); + } + return { commands, sources: { repos, sites }, outputs: hashed }; +} + +function writeManifest(dir, manifest) { + fs.writeFileSync( + path.join(dir, MANIFEST_NAME), + stableStringify(manifest) + '\n', + ); +} + +export function writeCollection(dir, { commands, repos, sites, outputs }) { + const manifest = buildManifest({ commands, repos, sites, outputs }); + for (const [name, content] of Object.entries(outputs)) { + const target = path.join(dir, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content); + } + writeManifest(dir, manifest); + return manifest; +} + +// Adds one output produced by a separate step (for example the agent +// configuration snapshot) to an existing collection, seeding a fresh +// manifest when none exists. Commands are kept sorted and unique so +// repeated runs stay byte-identical; names in `remove` lose both +// their file and their manifest entry, keeping superseded evidence +// out of the record. +export function upsertOutput(dir, { command, name, content, remove = [] }) { + const manifestPath = path.join(dir, MANIFEST_NAME); + const manifest = fs.existsSync(manifestPath) + ? JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + : { commands: [], sources: { repos: [], sites: [] }, outputs: {} }; + manifest.commands = [...new Set([...manifest.commands, command])].sort(); + for (const staleName of remove) { + delete manifest.outputs[staleName]; + fs.rmSync(path.join(dir, staleName), { force: true }); + } + const target = path.join(dir, name); + fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.writeFileSync(target, content); + manifest.outputs[name] = hashEntry(content); + writeManifest(dir, manifest); + return manifest; +} + +function walkFiles(dir, rel, found) { + for (const entry of fs.readdirSync(path.join(dir, rel), { + withFileTypes: true, + })) { + const relPath = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) walkFiles(dir, relPath, found); + else found.push(relPath); + } +} + +export function verifyManifest(dir) { + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, MANIFEST_NAME), 'utf8'), + ); + const mismatched = []; + const missing = []; + for (const name of Object.keys(manifest.outputs).sort()) { + const target = path.join(dir, name); + if (!fs.existsSync(target)) { + missing.push(name); + continue; + } + if (sha256Hex(fs.readFileSync(target)) !== manifest.outputs[name].sha256) { + mismatched.push(name); + } + } + const present = []; + walkFiles(dir, '', present); + const listed = new Set([...Object.keys(manifest.outputs), MANIFEST_NAME]); + const unexpected = present.filter((name) => !listed.has(name)).sort(); + return { + ok: + mismatched.length === 0 && + missing.length === 0 && + unexpected.length === 0, + mismatched, + missing, + unexpected, + }; +} diff --git a/scripts/assessment/lib/sitefetch.mjs b/scripts/assessment/lib/sitefetch.mjs new file mode 100644 index 00000000..55baa9be --- /dev/null +++ b/scripts/assessment/lib/sitefetch.mjs @@ -0,0 +1,46 @@ +import { sha256Hex } from './util.mjs'; + +function isoDate(now) { + return now().toISOString().slice(0, 10); +} + +export async function fetchSites( + urls, + { fetchImpl = fetch, now = () => new Date() } = {}, +) { + const results = []; + for (const url of [...new Set(urls)].sort()) { + try { + const res = await fetchImpl(url); + const body = Buffer.from(await res.arrayBuffer()); + results.push({ + url, + status: res.status, + retrievedDate: isoDate(now), + sha256: sha256Hex(body), + body, + }); + } catch (err) { + results.push({ + url, + status: null, + retrievedDate: isoDate(now), + error: err.message, + }); + } + } + return results; +} + +export async function checkLinks(urls, { fetchImpl = fetch } = {}) { + const results = []; + for (const url of [...new Set(urls)].sort()) { + try { + const res = await fetchImpl(url, { method: 'HEAD', redirect: 'follow' }); + results.push({ url, status: res.status }); + } catch (err) { + results.push({ url, status: null, error: err.message }); + } + } + return results; +} diff --git a/scripts/assessment/lib/snapshot.mjs b/scripts/assessment/lib/snapshot.mjs new file mode 100644 index 00000000..bdde93b8 --- /dev/null +++ b/scripts/assessment/lib/snapshot.mjs @@ -0,0 +1,32 @@ +const API_VERSION = '2026-03-10'; + +// Endpoint: GET /repos/{owner}/{repo}/copilot/cloud-agent/configuration +// (public preview). Whether it answers from inside an agent session, +// and with which credential, is unverified; a failure here is +// recorded as evidence, not hidden. +export async function snapshotAgentConfig({ + owner, + repo, + token, + fetchImpl = fetch, + apiBase = 'https://api.github.com', +}) { + const url = `${apiBase}/repos/${owner}/${repo}/copilot/cloud-agent/configuration`; + const headers = { + accept: 'application/vnd.github+json', + 'x-github-api-version': API_VERSION, + }; + if (token) headers.authorization = `Bearer ${token}`; + try { + const res = await fetchImpl(url, { headers }); + if (res.status !== 200) { + return { + ok: false, + error: { status: res.status, message: `HTTP ${res.status}` }, + }; + } + return { ok: true, config: await res.json() }; + } catch (err) { + return { ok: false, error: { status: null, message: err.message } }; + } +} diff --git a/scripts/assessment/lib/util.mjs b/scripts/assessment/lib/util.mjs new file mode 100644 index 00000000..96af36e7 --- /dev/null +++ b/scripts/assessment/lib/util.mjs @@ -0,0 +1,23 @@ +import { createHash } from 'node:crypto'; + +export function sha256Hex(data) { + return createHash('sha256').update(data).digest('hex'); +} + +function sortDeep(value) { + if (Array.isArray(value)) return value.map(sortDeep); + if (value && typeof value === 'object' && value.constructor === Object) { + const out = {}; + for (const key of Object.keys(value).sort()) { + out[key] = sortDeep(value[key]); + } + return out; + } + return value; +} + +// Deterministic JSON: object keys sorted at every depth, arrays kept +// in order, two-space indent. +export function stableStringify(value) { + return JSON.stringify(sortDeep(value), null, 2); +} diff --git a/scripts/assessment/snapshot-agent-config.mjs b/scripts/assessment/snapshot-agent-config.mjs new file mode 100644 index 00000000..4628f067 --- /dev/null +++ b/scripts/assessment/snapshot-agent-config.mjs @@ -0,0 +1,93 @@ +#!/usr/bin/env node +// Captures the Copilot cloud agent configuration for a repository +// (MCP servers, firewall state, custom allowlist) as committed +// evidence alongside an assessment's data outputs: what the agent +// could reach when a draft was produced. +// +// GITHUB_TOKEN=... node scripts/assessment/snapshot-agent-config.mjs \ +// --repo owner/name --out +// +// On success writes agent-config.json; on failure writes +// agent-config.error.json and still exits 0: the failure record is +// the evidence, and setup steps should not abort on it. + +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { snapshotAgentConfig } from './lib/snapshot.mjs'; +import { upsertOutput } from './lib/manifest.mjs'; +import { stableStringify } from './lib/util.mjs'; + +const USAGE = + 'usage: snapshot-agent-config.mjs --repo --out '; + +function takeValue(argv, i, flag) { + const value = argv[i + 1]; + if (value == null || value.startsWith('--')) { + throw new Error(`missing value for ${flag}`); + } + return value; +} + +function parseArgs(argv) { + const args = { repo: null, out: null }; + let i = 0; + while (i < argv.length) { + const flag = argv[i]; + if (flag === '--repo') { + args.repo = takeValue(argv, i, flag); + i += 2; + } else if (flag === '--out') { + args.out = takeValue(argv, i, flag); + i += 2; + } else throw new Error(`unknown flag: ${flag}`); + } + if (!args.repo || !args.repo.includes('/')) { + throw new Error('missing --repo owner/name'); + } + if (!args.out) throw new Error('missing --out'); + return args; +} + +async function main() { + let args; + try { + args = parseArgs(process.argv.slice(2)); + } catch (err) { + process.stderr.write(`${err.message}\n${USAGE}\n`); + process.exit(2); + } + const [owner, repo] = args.repo.split('/'); + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN ?? ''; + const apiBase = process.env.GITHUB_API_URL || 'https://api.github.com'; + const result = await snapshotAgentConfig({ owner, repo, token, apiBase }); + fs.mkdirSync(args.out, { recursive: true }); + // Whichever way the call went, the opposite record from an earlier + // run is removed and the manifest updated, so a directory never + // carries both a configuration and an error for the same step. + const command = `snapshot-agent-config.mjs --repo ${args.repo}`; + if (result.ok) { + upsertOutput(args.out, { + command, + name: 'agent-config.json', + content: + stableStringify({ repository: args.repo, config: result.config }) + + '\n', + remove: ['agent-config.error.json'], + }); + process.stdout.write('agent configuration captured\n'); + } else { + upsertOutput(args.out, { + command, + name: 'agent-config.error.json', + content: + stableStringify({ repository: args.repo, error: result.error }) + '\n', + remove: ['agent-config.json'], + }); + process.stdout.write( + `agent configuration unavailable: ${result.error.message}\n`, + ); + } +} + +await main(); diff --git a/scripts/assessment/test/collect.e2e.test.mjs b/scripts/assessment/test/collect.e2e.test.mjs new file mode 100644 index 00000000..bcc46601 --- /dev/null +++ b/scripts/assessment/test/collect.e2e.test.mjs @@ -0,0 +1,128 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const cli = path.join(here, '..', 'collect.mjs'); +const fixture = path.join(here, 'fixtures', 'sample-project'); + +function run(args) { + return spawnSync(process.execPath, [cli, ...args], { encoding: 'utf8' }); +} + +function collectInto(dir) { + return run(['--repo', fixture, '--out', dir]); +} + +function readTree(dir) { + const files = {}; + const walk = (rel) => { + for (const entry of fs + .readdirSync(path.join(dir, rel), { withFileTypes: true }) + .sort((a, b) => a.name.localeCompare(b.name))) { + const relPath = rel ? `${rel}/${entry.name}` : entry.name; + if (entry.isDirectory()) walk(relPath); + else files[relPath] = fs.readFileSync(path.join(dir, relPath)); + } + }; + walk(''); + return files; +} + +test('collect twice produces byte-identical outputs', () => { + const base = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-collect-')); + const out1 = path.join(base, 'out1'); + const out2 = path.join(base, 'out2'); + for (const out of [out1, out2]) { + const r = collectInto(out); + assert.equal(r.status, 0, r.stderr); + } + const tree1 = readTree(out1); + const tree2 = readTree(out2); + assert.deepEqual(Object.keys(tree1), Object.keys(tree2)); + assert.ok(Object.keys(tree1).includes('inventory.json')); + assert.ok(Object.keys(tree1).includes('links.json')); + assert.ok(Object.keys(tree1).includes('manifest.json')); + for (const name of Object.keys(tree1)) { + assert.ok(tree1[name].equals(tree2[name]), `differs: ${name}`); + } +}); + +test('verify passes on untouched outputs and fails after tampering', () => { + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-verify-')); + assert.equal(collectInto(out).status, 0); + assert.equal(run(['verify', '--out', out]).status, 0); + const target = path.join(out, 'inventory.json'); + fs.writeFileSync(target, fs.readFileSync(target, 'utf8') + ' '); + const failed = run(['verify', '--out', out]); + assert.equal(failed.status, 1); + assert.match(failed.stderr, /inventory\.json/); +}); + +test('collect refuses unknown flags and missing arguments', () => { + assert.notEqual(run(['--repo', fixture]).status, 0); + assert.notEqual(run(['--bogus']).status, 0); +}); + +test('collect rejects a flag consumed as another flag value', () => { + const r = run(['--repo', fixture, '--out', '--check-links']); + assert.equal(r.status, 2); + assert.match(r.stderr, /--out/); + const trailing = run(['--repo', fixture, '--out']); + assert.equal(trailing.status, 2); + assert.match(trailing.stderr, /--out/); +}); + +test('collect rejects an output directory at the repository root', () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-root-')); + fs.writeFileSync(path.join(repo, 'README.md'), '# r\n'); + const r = run(['--repo', repo, '--out', repo]); + assert.equal(r.status, 2); + assert.match(r.stderr, /repository root/); +}); + +test('an unreachable site is recorded as evidence, not a crash', () => { + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-badsite-')); + const r = run(['--repo', fixture, '--site', 'not-a-url', '--out', out]); + assert.equal(r.status, 0, r.stderr); + const fetches = JSON.parse( + fs.readFileSync(path.join(out, 'site-fetches.json'), 'utf8'), + ); + assert.equal(fetches.sites.length, 1); + assert.equal(fetches.sites[0].url, 'not-a-url'); + assert.equal(fetches.sites[0].status, null); + assert.ok(fetches.sites[0].error); + assert.equal(run(['verify', '--out', out]).status, 0); +}); + +test('rerunning into an in-repository output stays byte-identical', () => { + const repo = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-inrepo-')); + fs.cpSync(fixture, repo, { recursive: true }); + const out = path.join(repo, 'collected'); + const first = spawnSync( + process.execPath, + [cli, '--repo', repo, '--out', out], + { encoding: 'utf8' }, + ); + assert.equal(first.status, 0, first.stderr); + const tree1 = readTree(out); + const second = spawnSync( + process.execPath, + [cli, '--repo', repo, '--out', out], + { encoding: 'utf8' }, + ); + assert.equal(second.status, 0, second.stderr); + const tree2 = readTree(out); + assert.deepEqual(Object.keys(tree1), Object.keys(tree2)); + for (const name of Object.keys(tree1)) { + assert.ok(tree1[name].equals(tree2[name]), `differs: ${name}`); + } + const inventory = JSON.parse(tree2['inventory.json'].toString()); + const paths = inventory.repos[0].files.map((f) => f.path); + assert.ok(!paths.some((p) => p.startsWith('collected')), paths.join(',')); + assert.equal(run(['verify', '--out', out]).status, 0); +}); diff --git a/scripts/assessment/test/fixtures/sample-project/README.md b/scripts/assessment/test/fixtures/sample-project/README.md new file mode 100644 index 00000000..e46073e8 --- /dev/null +++ b/scripts/assessment/test/fixtures/sample-project/README.md @@ -0,0 +1,10 @@ +# Sample project + +[External A](https://example.com/a) and the [guide](./docs/guide.md) and +a [section](#section). + +![sample](./assets/sample.png) + +## Section + +Body text. diff --git a/scripts/assessment/test/fixtures/sample-project/assets/sample.png b/scripts/assessment/test/fixtures/sample-project/assets/sample.png new file mode 100644 index 00000000..ed5cbaac Binary files /dev/null and b/scripts/assessment/test/fixtures/sample-project/assets/sample.png differ diff --git a/scripts/assessment/test/fixtures/sample-project/docs/guide.md b/scripts/assessment/test/fixtures/sample-project/docs/guide.md new file mode 100644 index 00000000..f4706b56 --- /dev/null +++ b/scripts/assessment/test/fixtures/sample-project/docs/guide.md @@ -0,0 +1,5 @@ +# Guide + +See the [spec][s] and . + +[s]: https://example.com/b diff --git a/scripts/assessment/test/git.test.mjs b/scripts/assessment/test/git.test.mjs new file mode 100644 index 00000000..4aa8648f --- /dev/null +++ b/scripts/assessment/test/git.test.mjs @@ -0,0 +1,29 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolveHeadSha } from '../lib/git.mjs'; + +const SHA = 'a'.repeat(40); + +test('resolveHeadSha returns the trimmed sha from git', () => { + const calls = []; + const exec = (cmd, args) => { + calls.push([cmd, ...args]); + return `${SHA}\n`; + }; + assert.equal(resolveHeadSha('/some/dir', exec), SHA); + assert.deepEqual(calls, [['git', '-C', '/some/dir', 'rev-parse', 'HEAD']]); +}); + +test('resolveHeadSha returns null when git fails', () => { + const exec = () => { + throw new Error('not a repository'); + }; + assert.equal(resolveHeadSha('/some/dir', exec), null); +}); + +test('resolveHeadSha returns null on malformed output', () => { + assert.equal( + resolveHeadSha('/some/dir', () => 'HEAD\n'), + null, + ); +}); diff --git a/scripts/assessment/test/inventory.test.mjs b/scripts/assessment/test/inventory.test.mjs new file mode 100644 index 00000000..5c79c94a --- /dev/null +++ b/scripts/assessment/test/inventory.test.mjs @@ -0,0 +1,102 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildInventory, markdownPaths } from '../lib/inventory.mjs'; + +const fixture = path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'fixtures', + 'sample-project', +); + +test('buildInventory lists files sorted by posix path with hashes', () => { + const inv = buildInventory(fixture); + assert.deepEqual( + inv.files.map((f) => f.path), + ['README.md', 'assets/sample.png', 'docs/guide.md'], + ); + for (const f of inv.files) { + assert.match(f.sha256, /^[0-9a-f]{64}$/); + assert.ok(f.bytes > 0); + } +}); + +test('buildInventory totals count files, bytes, and extensions', () => { + const inv = buildInventory(fixture); + assert.equal(inv.totals.fileCount, 3); + assert.equal(inv.totals.markdownCount, 2); + assert.deepEqual(inv.totals.byExtension, { '.md': 2, '.png': 1 }); + assert.equal( + inv.totals.byteCount, + inv.files.reduce((n, f) => n + f.bytes, 0), + ); +}); + +test('buildInventory skips .git and node_modules directories', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-inv-')); + fs.mkdirSync(path.join(root, '.git')); + fs.mkdirSync(path.join(root, 'node_modules', 'pkg'), { recursive: true }); + fs.writeFileSync(path.join(root, '.git', 'HEAD'), 'ref\n'); + fs.writeFileSync(path.join(root, 'node_modules', 'pkg', 'i.js'), '1\n'); + fs.writeFileSync(path.join(root, 'kept.md'), 'kept\n'); + const inv = buildInventory(root); + assert.deepEqual( + inv.files.map((f) => f.path), + ['kept.md'], + ); +}); + +test('buildInventory is deterministic across runs', () => { + const a = JSON.stringify(buildInventory(fixture)); + const b = JSON.stringify(buildInventory(fixture)); + assert.equal(a, b); +}); + +test('buildInventory excludes a named subtree', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-inv-')); + fs.mkdirSync(path.join(root, 'collected', 'sites'), { recursive: true }); + fs.writeFileSync(path.join(root, 'collected', 'manifest.json'), '{}\n'); + fs.writeFileSync(path.join(root, 'collected', 'sites', 'a.body'), 'x\n'); + fs.writeFileSync(path.join(root, 'kept.md'), 'kept\n'); + const inv = buildInventory(root, { exclude: ['collected'] }); + assert.deepEqual( + inv.files.map((f) => f.path), + ['kept.md'], + ); +}); + +test('buildInventory records symlinks without following them', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-inv-')); + fs.mkdirSync(path.join(root, 'docs')); + fs.writeFileSync(path.join(root, 'docs', 'real.md'), '# real\n'); + fs.symlinkSync('docs/real.md', path.join(root, 'alias.md')); + fs.symlinkSync('/etc', path.join(root, 'escape')); + const inv = buildInventory(root); + assert.deepEqual( + inv.files.map((f) => f.path), + ['docs/real.md'], + ); + assert.deepEqual( + inv.symlinks.map((s) => [s.path, s.target]), + [ + ['alias.md', 'docs/real.md'], + ['escape', '/etc'], + ], + ); + for (const s of inv.symlinks) { + assert.match(s.sha256, /^[0-9a-f]{64}$/); + assert.equal(s.bytes, Buffer.byteLength(s.target)); + } + assert.equal(inv.totals.fileCount, 1); + assert.equal(inv.totals.symlinkCount, 2); +}); + +test('markdownPaths never includes symlinked markdown', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-inv-')); + fs.writeFileSync(path.join(root, 'real.md'), '# real\n'); + fs.symlinkSync('real.md', path.join(root, 'alias.md')); + assert.deepEqual(markdownPaths(buildInventory(root)), ['real.md']); +}); diff --git a/scripts/assessment/test/links.test.mjs b/scripts/assessment/test/links.test.mjs new file mode 100644 index 00000000..eed5c575 --- /dev/null +++ b/scripts/assessment/test/links.test.mjs @@ -0,0 +1,88 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildLinkReport, extractLinks } from '../lib/links.mjs'; + +const fixture = path.join( + path.dirname(fileURLToPath(import.meta.url)), + 'fixtures', + 'sample-project', +); + +test('extractLinks finds inline links, images, references, autolinks', () => { + const md = [ + '[a](https://x.test/a) ![i](./img.png) [r][ref] ', + '[anchor](#top) [rel](./doc.md)', + '[ref]: https://x.test/ref', + ].join('\n'); + const links = extractLinks(md); + assert.deepEqual(links, [ + { url: '#top', kind: 'anchor' }, + { url: './doc.md', kind: 'internal' }, + { url: './img.png', kind: 'internal' }, + { url: 'https://x.test/a', kind: 'external' }, + { url: 'https://x.test/auto', kind: 'external' }, + { url: 'https://x.test/ref', kind: 'external' }, + ]); +}); + +test('extractLinks dedupes repeated urls', () => { + const links = extractLinks('[a](https://x.test/a) [b](https://x.test/a)'); + assert.equal(links.length, 1); +}); + +test('extractLinks ignores links inside fenced code blocks', () => { + const md = [ + 'Before [real](https://x.test/real).', + '```markdown', + '[fenced](https://x.test/fenced)', + '```', + '~~~', + '[tilde](https://x.test/tilde)', + '~~~', + 'After.', + ].join('\n'); + assert.deepEqual( + extractLinks(md).map((l) => l.url), + ['https://x.test/real'], + ); +}); + +test('extractLinks treats an unclosed fence as running to the end', () => { + const md = [ + '[real](https://x.test/real)', + '```', + '[fenced](https://x.test/fenced)', + ].join('\n'); + assert.deepEqual( + extractLinks(md).map((l) => l.url), + ['https://x.test/real'], + ); +}); + +test('extractLinks ignores links inside inline code spans', () => { + const md = + 'Use `[example](https://x.test/span)` then see [real](https://x.test/real).'; + assert.deepEqual( + extractLinks(md).map((l) => l.url), + ['https://x.test/real'], + ); +}); + +test('extractLinks reads angle-bracket destinations', () => { + const links = extractLinks('[spaced](<./my page.md>)'); + assert.deepEqual(links, [{ url: './my page.md', kind: 'internal' }]); +}); + +test('buildLinkReport aggregates per file and by kind', () => { + const report = buildLinkReport(fixture, ['README.md', 'docs/guide.md']); + assert.deepEqual(Object.keys(report.perFile), ['README.md', 'docs/guide.md']); + assert.deepEqual(report.external, [ + 'https://example.com/a', + 'https://example.com/b', + 'https://example.com/c', + ]); + assert.deepEqual(report.internal, ['./assets/sample.png', './docs/guide.md']); + assert.deepEqual(report.anchors, ['#section']); +}); diff --git a/scripts/assessment/test/manifest.test.mjs b/scripts/assessment/test/manifest.test.mjs new file mode 100644 index 00000000..87732862 --- /dev/null +++ b/scripts/assessment/test/manifest.test.mjs @@ -0,0 +1,140 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { + buildManifest, + upsertOutput, + verifyManifest, + writeCollection, +} from '../lib/manifest.mjs'; + +function tmpDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-manifest-')); +} + +const meta = { + commands: ['collect.mjs --repo sample'], + repos: [{ path: 'sample', sha: null }], + sites: [], +}; + +test('buildManifest hashes every output', () => { + const m = buildManifest({ + ...meta, + outputs: { 'inventory.json': '{}\n', 'links.json': '{}\n' }, + }); + assert.deepEqual(m.commands, meta.commands); + assert.deepEqual(Object.keys(m.outputs), ['inventory.json', 'links.json']); + for (const entry of Object.values(m.outputs)) { + assert.match(entry.sha256, /^[0-9a-f]{64}$/); + assert.ok(entry.bytes > 0); + } +}); + +test('writeCollection then verifyManifest passes', () => { + const dir = tmpDir(); + writeCollection(dir, { + ...meta, + outputs: { 'inventory.json': '{"a":1}\n', 'nested/links.json': '{}\n' }, + }); + const result = verifyManifest(dir); + assert.deepEqual(result, { + ok: true, + mismatched: [], + missing: [], + unexpected: [], + }); +}); + +test('verifyManifest flags a tampered output', () => { + const dir = tmpDir(); + writeCollection(dir, { ...meta, outputs: { 'inventory.json': '{}\n' } }); + fs.writeFileSync(path.join(dir, 'inventory.json'), '{"edited":true}\n'); + const result = verifyManifest(dir); + assert.equal(result.ok, false); + assert.deepEqual(result.mismatched, ['inventory.json']); +}); + +test('verifyManifest flags a missing output', () => { + const dir = tmpDir(); + writeCollection(dir, { ...meta, outputs: { 'inventory.json': '{}\n' } }); + fs.rmSync(path.join(dir, 'inventory.json')); + const result = verifyManifest(dir); + assert.equal(result.ok, false); + assert.deepEqual(result.missing, ['inventory.json']); +}); + +test('verifyManifest flags files the manifest does not list', () => { + const dir = tmpDir(); + writeCollection(dir, { ...meta, outputs: { 'inventory.json': '{}\n' } }); + fs.mkdirSync(path.join(dir, 'sites')); + fs.writeFileSync(path.join(dir, 'sites/stale.body'), 'left over'); + const result = verifyManifest(dir); + assert.equal(result.ok, false); + assert.deepEqual(result.unexpected, ['sites/stale.body']); +}); + +test('upsertOutput seeds a manifest when none exists', () => { + const dir = tmpDir(); + upsertOutput(dir, { + command: 'snapshot-agent-config.mjs --repo o/r', + name: 'agent-config.json', + content: '{"config":true}\n', + }); + const result = verifyManifest(dir); + assert.deepEqual(result, { + ok: true, + mismatched: [], + missing: [], + unexpected: [], + }); + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'), + ); + assert.deepEqual(manifest.commands, ['snapshot-agent-config.mjs --repo o/r']); + assert.ok(manifest.outputs['agent-config.json']); +}); + +test('upsertOutput extends an existing manifest and removes stale names', () => { + const dir = tmpDir(); + writeCollection(dir, { ...meta, outputs: { 'inventory.json': '{}\n' } }); + fs.writeFileSync(path.join(dir, 'agent-config.error.json'), '{"old":1}\n'); + upsertOutput(dir, { + command: 'snapshot-agent-config.mjs --repo o/r', + name: 'agent-config.json', + content: '{"config":true}\n', + remove: ['agent-config.error.json'], + }); + assert.ok(!fs.existsSync(path.join(dir, 'agent-config.error.json'))); + const result = verifyManifest(dir); + assert.deepEqual(result, { + ok: true, + mismatched: [], + missing: [], + unexpected: [], + }); + const manifest = JSON.parse( + fs.readFileSync(path.join(dir, 'manifest.json'), 'utf8'), + ); + assert.deepEqual(manifest.commands, [ + 'collect.mjs --repo sample', + 'snapshot-agent-config.mjs --repo o/r', + ]); +}); + +test('upsertOutput run twice leaves byte-identical manifests', () => { + const dir = tmpDir(); + const change = { + command: 'snapshot-agent-config.mjs --repo o/r', + name: 'agent-config.json', + content: '{"config":true}\n', + remove: ['agent-config.error.json'], + }; + upsertOutput(dir, change); + const first = fs.readFileSync(path.join(dir, 'manifest.json')); + upsertOutput(dir, change); + const second = fs.readFileSync(path.join(dir, 'manifest.json')); + assert.ok(first.equals(second)); +}); diff --git a/scripts/assessment/test/sitefetch.test.mjs b/scripts/assessment/test/sitefetch.test.mjs new file mode 100644 index 00000000..a1a2c184 --- /dev/null +++ b/scripts/assessment/test/sitefetch.test.mjs @@ -0,0 +1,52 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { checkLinks, fetchSites } from '../lib/sitefetch.mjs'; + +function response(status, body = '') { + return { + status, + arrayBuffer: async () => new TextEncoder().encode(body).buffer, + }; +} + +test('fetchSites records status, hash, body, and date per url', async () => { + const results = await fetchSites(['https://x.test/'], { + fetchImpl: async () => response(200, 'ok'), + now: () => new Date('2026-08-14T00:00:00Z'), + }); + assert.equal(results.length, 1); + const r = results[0]; + assert.equal(r.url, 'https://x.test/'); + assert.equal(r.status, 200); + assert.equal(r.retrievedDate, '2026-08-14'); + assert.match(r.sha256, /^[0-9a-f]{64}$/); + assert.equal(Buffer.from(r.body).toString(), 'ok'); +}); + +test('fetchSites records a failed fetch as an error entry', async () => { + const results = await fetchSites(['https://down.test/'], { + fetchImpl: async () => { + throw new Error('connect refused'); + }, + now: () => new Date('2026-08-14T00:00:00Z'), + }); + assert.equal(results[0].status, null); + assert.match(results[0].error, /connect refused/); +}); + +test('checkLinks reports status per url, sorted, errors captured', async () => { + const results = await checkLinks( + ['https://x.test/b', 'https://x.test/a', 'https://x.test/err'], + { + fetchImpl: async (url) => { + if (url.endsWith('/err')) throw new Error('boom'); + return response(url.endsWith('/a') ? 200 : 404); + }, + }, + ); + assert.deepEqual(results, [ + { url: 'https://x.test/a', status: 200 }, + { url: 'https://x.test/b', status: 404 }, + { url: 'https://x.test/err', status: null, error: 'boom' }, + ]); +}); diff --git a/scripts/assessment/test/snapshot-cli.test.mjs b/scripts/assessment/test/snapshot-cli.test.mjs new file mode 100644 index 00000000..7c390aae --- /dev/null +++ b/scripts/assessment/test/snapshot-cli.test.mjs @@ -0,0 +1,105 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import http from 'node:http'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const cli = path.join(here, '..', 'snapshot-agent-config.mjs'); +const collectCli = path.join(here, '..', 'collect.mjs'); +const fixture = path.join(here, 'fixtures', 'sample-project'); + +function run(args, env = {}) { + return spawnSync(process.execPath, [cli, ...args], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); +} + +// The stub API server lives in this process, so the CLI under test +// must run with an async spawn: spawnSync would block the event loop +// and deadlock the child against a server that can never answer. +function runAsync(args, env = {}) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [cli, ...args], { + env: { ...process.env, ...env }, + }); + let stderr = ''; + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + child.on('close', (status) => resolve({ status, stderr })); + }); +} + +function withServer(status, body) { + const server = http.createServer((req, res) => { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(body); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + resolve({ + base: `http://127.0.0.1:${server.address().port}`, + close: () => new Promise((done) => server.close(done)), + }); + }); + }); +} + +test('snapshot rejects a flag consumed as another flag value', () => { + const r = run(['--repo', 'cncf/techdocs', '--out', '--repo']); + assert.equal(r.status, 2); + assert.match(r.stderr, /--out/); +}); + +test('snapshot success replaces a stale error record and joins the manifest', async () => { + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-snap-')); + const collect = spawnSync( + process.execPath, + [collectCli, '--repo', fixture, '--out', out], + { encoding: 'utf8' }, + ); + assert.equal(collect.status, 0, collect.stderr); + fs.writeFileSync(path.join(out, 'agent-config.error.json'), '{"old":1}\n'); + + const server = await withServer(200, '{"is_firewall_enabled":true}'); + const r = await runAsync(['--repo', 'o/r', '--out', out], { + GITHUB_API_URL: server.base, + }); + await server.close(); + assert.equal(r.status, 0, r.stderr); + + assert.ok(fs.existsSync(path.join(out, 'agent-config.json'))); + assert.ok(!fs.existsSync(path.join(out, 'agent-config.error.json'))); + const verify = spawnSync( + process.execPath, + [collectCli, 'verify', '--out', out], + { encoding: 'utf8' }, + ); + assert.equal(verify.status, 0, verify.stderr); +}); + +test('snapshot failure replaces a stale success record and joins the manifest', async () => { + const out = fs.mkdtempSync(path.join(os.tmpdir(), 'aiaa-snap-')); + fs.writeFileSync(path.join(out, 'agent-config.json'), '{"old":1}\n'); + + const server = await withServer(500, '{}'); + const r = await runAsync(['--repo', 'o/r', '--out', out], { + GITHUB_API_URL: server.base, + }); + await server.close(); + assert.equal(r.status, 0, r.stderr); + + assert.ok(fs.existsSync(path.join(out, 'agent-config.error.json'))); + assert.ok(!fs.existsSync(path.join(out, 'agent-config.json'))); + const verify = spawnSync( + process.execPath, + [collectCli, 'verify', '--out', out], + { encoding: 'utf8' }, + ); + assert.equal(verify.status, 0, verify.stderr); +}); diff --git a/scripts/assessment/test/snapshot.test.mjs b/scripts/assessment/test/snapshot.test.mjs new file mode 100644 index 00000000..f410beef --- /dev/null +++ b/scripts/assessment/test/snapshot.test.mjs @@ -0,0 +1,69 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { snapshotAgentConfig } from '../lib/snapshot.mjs'; + +const okPayload = { + mcp_configuration: null, + enabled_tools: { + codeql: true, + copilot_code_review: true, + secret_scanning: true, + dependency_vulnerability_checks: true, + }, + require_actions_workflow_approval: true, + is_firewall_enabled: true, + is_firewall_recommended_allowlist_enabled: true, + custom_allowlist: [], + is_automations_enabled: true, + require_write_access_for_automation_triggers: true, +}; + +test('snapshotAgentConfig returns the configuration on 200', async () => { + const seen = {}; + const result = await snapshotAgentConfig({ + owner: 'cncf', + repo: 'techdocs', + token: 'tkn', + fetchImpl: async (url, opts) => { + seen.url = url; + seen.auth = opts.headers.authorization; + return { + status: 200, + json: async () => okPayload, + }; + }, + }); + assert.equal( + seen.url, + 'https://api.github.com/repos/cncf/techdocs/copilot/cloud-agent/configuration', + ); + assert.equal(seen.auth, 'Bearer tkn'); + assert.deepEqual(result, { ok: true, config: okPayload }); +}); + +test('snapshotAgentConfig reports an http error explicitly', async () => { + const result = await snapshotAgentConfig({ + owner: 'cncf', + repo: 'techdocs', + token: 'tkn', + fetchImpl: async () => ({ status: 404, json: async () => ({}) }), + }); + assert.deepEqual(result, { + ok: false, + error: { status: 404, message: 'HTTP 404' }, + }); +}); + +test('snapshotAgentConfig reports a network failure explicitly', async () => { + const result = await snapshotAgentConfig({ + owner: 'cncf', + repo: 'techdocs', + token: '', + fetchImpl: async () => { + throw new Error('no route'); + }, + }); + assert.equal(result.ok, false); + assert.equal(result.error.status, null); + assert.match(result.error.message, /no route/); +}); diff --git a/scripts/assessment/test/util.test.mjs b/scripts/assessment/test/util.test.mjs new file mode 100644 index 00000000..660559c0 --- /dev/null +++ b/scripts/assessment/test/util.test.mjs @@ -0,0 +1,22 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { sha256Hex, stableStringify } from '../lib/util.mjs'; + +test('stableStringify sorts object keys at every depth', () => { + const out = stableStringify({ b: 1, a: { d: 2, c: [3, { z: 4, y: 5 }] } }); + assert.equal( + out, + JSON.stringify({ a: { c: [3, { y: 5, z: 4 }], d: 2 }, b: 1 }, null, 2), + ); +}); + +test('stableStringify preserves array order', () => { + assert.equal(stableStringify([2, 1]), JSON.stringify([2, 1], null, 2)); +}); + +test('sha256Hex matches a known vector', () => { + assert.equal( + sha256Hex('abc'), + 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad', + ); +});