From ab3796a8c43189dd6e2c7b2279da9c40c01856be Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:33:24 +0800 Subject: [PATCH 01/24] Archive v0.27: immutable research collection manifests and public-Run validation --- .../migrations/0025_research_collections.sql | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 packages/database/migrations/0025_research_collections.sql diff --git a/packages/database/migrations/0025_research_collections.sql b/packages/database/migrations/0025_research_collections.sql new file mode 100644 index 0000000..0b5106c --- /dev/null +++ b/packages/database/migrations/0025_research_collections.sql @@ -0,0 +1,57 @@ +BEGIN; + +SET search_path TO modelapse, public; + +CREATE TABLE research_collections ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(btrim(title)) BETWEEN 3 AND 120), + description text, + filters jsonb NOT NULL CHECK (jsonb_typeof(filters) = 'object'), + run_ids uuid[] NOT NULL CHECK (cardinality(run_ids) BETWEEN 1 AND 50), + content_sha256 text NOT NULL CHECK (content_sha256 ~ '^[a-f0-9]{64}$'), + created_by text NOT NULL CHECK (char_length(btrim(created_by)) > 0), + created_at timestamptz NOT NULL DEFAULT now(), + selection_limit integer NOT NULL DEFAULT 50 CHECK (selection_limit = 50), + UNIQUE (content_sha256, title) +); + +CREATE INDEX research_collections_created_idx + ON research_collections (created_at DESC, id DESC); + +CREATE FUNCTION validate_research_collection_snapshot() +RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE + run_count integer; +BEGIN + IF cardinality(NEW.run_ids) <> ( + SELECT COUNT(DISTINCT value) FROM unnest(NEW.run_ids) AS value + ) THEN + RAISE EXCEPTION 'research collection contains duplicate Run IDs'; + END IF; + + SELECT COUNT(*) INTO run_count + FROM unnest(NEW.run_ids) AS item(run_id) + JOIN runs r ON r.id = item.run_id + JOIN test_cases tc ON tc.id = r.test_case_id + WHERE r.status = 'completed' + AND r.sealed_at IS NOT NULL + AND r.completed_at IS NOT NULL + AND tc.visibility = 'public' + AND tc.case_type <> 'calibration'; + + IF run_count <> cardinality(NEW.run_ids) THEN + RAISE EXCEPTION 'research collection requires sealed public non-calibration Runs'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER research_collections_validate +BEFORE INSERT ON research_collections +FOR EACH ROW EXECUTE FUNCTION validate_research_collection_snapshot(); + +CREATE TRIGGER research_collections_append_only +BEFORE UPDATE OR DELETE ON research_collections +FOR EACH ROW EXECUTE FUNCTION prevent_append_only_mutation(); + +COMMIT; From 41fb217c5f2b6ed1fbdf33ccc7f7c6036b994b24 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:33:54 +0800 Subject: [PATCH 02/24] Implement append-only operator-curated bounded research snapshots with manifest hashes --- .../persistence/src/research-collections.ts | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 packages/persistence/src/research-collections.ts diff --git a/packages/persistence/src/research-collections.ts b/packages/persistence/src/research-collections.ts new file mode 100644 index 0000000..07cade4 --- /dev/null +++ b/packages/persistence/src/research-collections.ts @@ -0,0 +1,190 @@ +import { createHash } from "node:crypto"; +import { Pool } from "pg"; +import { + validateArchiveResearchFilters, + type ArchiveResearchFilters, +} from "./archive-research.js"; +import type { + ArchiveRunView, + PgArchiveRepository, +} from "./archive-repository.js"; + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +export interface ResearchCollectionSummary { + readonly id: string; + readonly title: string; + readonly description: string | null; + readonly filters: ArchiveResearchFilters; + readonly runIds: readonly string[]; + readonly contentSha256: string; + readonly createdBy: string; + readonly createdAt: string; + readonly selectionLimit: 50; +} + +export interface ResearchCollectionDetail extends ResearchCollectionSummary { + readonly runs: readonly ArchiveRunView[]; +} + +export interface CreateResearchCollectionInput { + readonly title: string; + readonly description?: string; + readonly filters: ArchiveResearchFilters; + readonly actor: string; +} + +interface CollectionRow { + id: string; + title: string; + description: string | null; + filters: ArchiveResearchFilters; + run_ids: string[]; + content_sha256: string; + created_by: string; + created_at: Date; + selection_limit: 50; +} + +function view(row: CollectionRow): ResearchCollectionSummary { + return { + id: row.id, + title: row.title, + description: row.description, + filters: row.filters, + runIds: row.run_ids, + contentSha256: row.content_sha256, + createdBy: row.created_by, + createdAt: row.created_at.toISOString(), + selectionLimit: row.selection_limit, + }; +} + +const SELECT = ` + SELECT + id, title, description, filters, run_ids, content_sha256, + created_by, created_at, selection_limit + FROM modelapse.research_collections +`; + +export function researchCollectionDigest( + filters: ArchiveResearchFilters, + runIds: readonly string[], +): string { + const normalized = validateArchiveResearchFilters(filters); + const { cursor: _cursor, limit: _limit, ...facets } = normalized; + const canonical = { + schemaVersion: 1, + filters: { + accountTier: facets.accountTier ?? null, + cost: facets.cost, + evidence: facets.evidence, + modelId: facets.modelId ?? null, + providerSlug: facets.providerSlug ?? null, + region: facets.region ?? null, + serviceTier: facets.serviceTier ?? null, + testCaseId: facets.testCaseId ?? null, + }, + runIds, + }; + return createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); +} + +export class PgResearchCollections { + constructor( + private readonly pool: Pool, + private readonly archive: Pick, + ) {} + + static connect( + connectionString: string, + archive: Pick, + options: { readonly max?: number } = {}, + ): PgResearchCollections { + return new PgResearchCollections( + new Pool({ connectionString, max: options.max ?? 3 }), + archive, + ); + } + + async ping(): Promise { + await this.pool.query("SELECT 1"); + } + + async close(): Promise { + await this.pool.end(); + } + + async create(input: CreateResearchCollectionInput): Promise { + const title = input.title.trim(); + const actor = input.actor.trim(); + const description = input.description?.trim() || null; + if (title.length < 3 || title.length > 120 || !actor || actor.length > 100 || + (description !== null && description.length > 2000)) { + throw new Error("invalid_research_collection_metadata"); + } + // Cursor/offset captures would masquerade as a complete research slice. + if (input.filters.cursor) { + throw new Error("research_collection_cursor_not_allowed"); + } + const filters = validateArchiveResearchFilters({ + ...input.filters, + limit: 50, + }); + const page = await this.archive.researchRuns(filters); + if (page.hasMore) { + throw new Error("research_collection_exceeds_50_runs"); + } + if (page.runs.length === 0) { + throw new Error("research_collection_empty"); + } + const runIds = page.runs.map((run) => run.id); + const { limit: _limit, cursor: _cursor, ...frozen } = filters; + const contentSha256 = researchCollectionDigest(frozen, runIds); + const result = await this.pool.query( + `INSERT INTO modelapse.research_collections ( + title, description, filters, run_ids, content_sha256, created_by + ) + VALUES ($1, $2, $3::jsonb, $4::uuid[], $5, $6) + RETURNING + id, title, description, filters, run_ids, content_sha256, + created_by, created_at, selection_limit`, + [title, description, JSON.stringify(frozen), runIds, contentSha256, actor], + ); + const summary = view(result.rows[0]!); + return { ...summary, runs: page.runs }; + } + + async list(limit = 30): Promise { + if (!Number.isInteger(limit) || limit < 1 || limit > 50) { + throw new Error("invalid_limit"); + } + const result = await this.pool.query( + SELECT + " ORDER BY created_at DESC, id DESC LIMIT $1", + [limit], + ); + return result.rows.map(view); + } + + async get(id: string): Promise { + if (!UUID_RE.test(id)) { + throw new Error("invalid_research_collection_id"); + } + const result = await this.pool.query( + SELECT + " WHERE id = $1 LIMIT 1", + [id], + ); + const row = result.rows[0]; + if (!row) return null; + const runs = await this.archive.getRunsByIds(row.run_ids); + if (runs.length !== row.run_ids.length) { + throw new Error("research_collection_snapshot_incomplete"); + } + const summary = view(row); + if (researchCollectionDigest(summary.filters, summary.runIds) !== summary.contentSha256) { + throw new Error("research_collection_digest_mismatch"); + } + return { ...summary, runs }; + } +} From 5ec5f54de41a0c320f2b64c347e1fdac9d487b76 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:34:06 +0800 Subject: [PATCH 03/24] Read exact immutable Run identities in saved collection order --- .../persistence/src/archive-repository.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/persistence/src/archive-repository.ts b/packages/persistence/src/archive-repository.ts index ce4b4bf..1d47a8d 100644 --- a/packages/persistence/src/archive-repository.ts +++ b/packages/persistence/src/archive-repository.ts @@ -1219,6 +1219,26 @@ export class PgArchiveRepository { return result.rows.map(runView); } + async getRunsByIds( + runIds: readonly string[], + ): Promise { + if (runIds.length > 50) throw new Error("invalid_research_run_count"); + if (runIds.length === 0) return []; + const result = await this.pool.query( + RUN_SELECT + + ` + WHERE r.id = ANY($1::uuid[]) + AND tc.visibility = 'public' + AND tc.case_type <> 'calibration' + AND r.status = 'completed' + AND r.sealed_at IS NOT NULL + AND r.completed_at IS NOT NULL + ORDER BY array_position($1::uuid[], r.id)`, + [runIds], + ); + return result.rows.map(runView); + } + async researchRuns( input: ArchiveResearchFilters = {}, ): Promise { From acd920bb1539278a35bf4b6ba54f956f62bf206d Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:34:29 +0800 Subject: [PATCH 04/24] Add sourced JSON/CSV research exports with spreadsheet formula escaping --- packages/persistence/src/research-export.ts | 133 ++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 packages/persistence/src/research-export.ts diff --git a/packages/persistence/src/research-export.ts b/packages/persistence/src/research-export.ts new file mode 100644 index 0000000..0f9c179 --- /dev/null +++ b/packages/persistence/src/research-export.ts @@ -0,0 +1,133 @@ +import type { ArchiveRunView } from "./archive-repository.js"; +import type { ResearchCollectionDetail } from "./research-collections.js"; + +export type ResearchExportFormat = "json" | "csv"; + +interface ResearchExportRow { + runId: string; + runUrl: string; + modelId: string | null; + provider: string; + model: string; + testCaseId: string; + test: string; + completedAt: string | null; + evidenceLevel: string | null; + evaluationStatus: string | null; + exactMatch: boolean | null; + executionPath: string; + region: string | null; + accountTier: string | null; + effectiveServiceTier: string | null; + serviceAssurance: string | null; + providerPolicySourceId: string | null; + runnerAccessSourceId: string | null; + pricingSourceId: string | null; + currency: string | null; + estimatedNativeCost: string | null; + qualificationCaveats: readonly string[]; + costCaveats: readonly string[]; +} + +export function researchExportRow(run: ArchiveRunView): ResearchExportRow { + return { + runId: run.id, + runUrl: "/runs/" + run.id, + modelId: run.model.id, + provider: run.provider.slug, + model: run.model.canonicalSlug ?? run.requestedModel, + testCaseId: run.test.testCaseId, + test: [ + run.test.familySlug, run.test.variantSlug, + run.test.version, run.test.caseSlug, + ].join("/"), + completedAt: run.completedAt, + evidenceLevel: run.evidenceLevel, + evaluationStatus: run.evaluation?.status ?? null, + exactMatch: run.evaluation?.exactMatch ?? null, + executionPath: run.executionPath, + region: run.executionQualification?.executionRegion ?? null, + accountTier: run.executionQualification?.accountTier ?? null, + effectiveServiceTier: + run.executionQualification?.returnedServiceTier ?? + run.executionQualification?.serviceTier ?? null, + serviceAssurance: run.executionQualification?.serviceAssurance ?? null, + providerPolicySourceId: + run.executionQualification?.providerPolicyObservation?.sourceId ?? null, + runnerAccessSourceId: + run.executionQualification?.runnerAccessObservation?.sourceId ?? null, + pricingSourceId: run.cost?.pricingObservation?.sourceId ?? null, + currency: run.cost?.pricing?.currency ?? null, + estimatedNativeCost: run.cost?.estimatedNativeCost ?? null, + qualificationCaveats: run.executionQualification?.caveats ?? + ["qualification_missing"], + costCaveats: run.cost?.caveats ?? ["cost_evidence_missing"], + }; +} + +const csvKeys = [ + "runId", "runUrl", "modelId", "provider", "model", "testCaseId", "test", + "completedAt", "evidenceLevel", "evaluationStatus", "exactMatch", + "executionPath", "region", "accountTier", "effectiveServiceTier", + "serviceAssurance", "providerPolicySourceId", "runnerAccessSourceId", + "pricingSourceId", "currency", "estimatedNativeCost", + "qualificationCaveats", "costCaveats", +] as const satisfies readonly (keyof ResearchExportRow)[]; + +function csvCell(value: unknown): string { + const plain = Array.isArray(value) + ? value.join(";") + : value === null || value === undefined ? "" : String(value); + // Spreadsheet applications can evaluate cells starting with these characters. + // Neutralize potential formula injection without losing the original evidence text. + const safe = /^[\s]*[=+@\-]/.test(plain) ? "'" + plain : plain; + return '"' + safe.replaceAll('"', '""') + '"'; +} + +export function exportResearchCollection( + collection: ResearchCollectionDetail, + format: ResearchExportFormat, +): { readonly body: string; readonly mediaType: string; readonly filename: string } { + const filename = "modelapse-research-" + collection.id + "." + format; + const rows = collection.runs.map(researchExportRow); + if (format === "json") { + return { + filename, + mediaType: "application/json; charset=utf-8", + body: JSON.stringify({ + schemaVersion: 1, + manifest: { + id: collection.id, + title: collection.title, + description: collection.description, + createdAt: collection.createdAt, + filters: collection.filters, + runIds: collection.runIds, + contentSha256: collection.contentSha256, + selectionLimit: collection.selectionLimit, + scope: "sealed_public_non_calibration", + }, + interpretation: { + resultOrder: "captured_completed_at_desc_run_id_desc", + matchedComparability: "not_evaluated", + calibrationEligibility: "not_evaluated", + unknownCostIsFree: false, + rankingsProvided: false, + }, + rows, + }, null, 2) + "\n", + }; + } + if (format !== "csv") throw new Error("invalid_research_export_format"); + const csv = [ + csvKeys.map(csvCell).join(","), + ...rows.map((row) => + csvKeys.map((key) => csvCell(row[key])).join(","), + ), + ].join("\r\n") + "\r\n"; + return { + filename, + mediaType: "text/csv; charset=utf-8", + body: csv, + }; +} From 7ad6d58728a4a44a96532f4af9a6ff64e58a4ff4 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:34:49 +0800 Subject: [PATCH 05/24] Export research collection persistence and sourced JSON/CSV helpers --- packages/persistence/src/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/persistence/src/index.ts b/packages/persistence/src/index.ts index e9484a2..15ae99d 100644 --- a/packages/persistence/src/index.ts +++ b/packages/persistence/src/index.ts @@ -8,3 +8,5 @@ export * from "./cost-ledger.js"; export * from "./calibration-repository.js"; export * from "./comparability-repository.js"; export * from "./archive-research.js"; +export * from "./research-collections.js"; +export * from "./research-export.js"; From d2e74673789424213e8e3a67cbdda9d80b893953 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:34:53 +0800 Subject: [PATCH 06/24] Wire public Research Collection repository into API lifecycle --- apps/api/src/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index e589418..5b2383d 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -22,6 +22,7 @@ import { PgComparabilityRepository, PgCostLedger, PgRunRepository, + PgResearchCollections, } from "@modelapse/persistence"; import { createApp } from "./app.js"; @@ -35,6 +36,7 @@ const jobs = PgRunJobQueue.connect(databaseUrl); const fleet = PgExecutionFleet.connect(databaseUrl); const planner = PgRunPlanner.connect(databaseUrl); const archive = PgArchiveRepository.connect(databaseUrl); +const researchCollections = PgResearchCollections.connect(databaseUrl, archive); const costLedger = PgCostLedger.connect(databaseUrl); const calibration = PgCalibrationRepository.connect(databaseUrl); const comparability = PgComparabilityRepository.connect(databaseUrl); @@ -56,6 +58,7 @@ const app = createApp({ fleet, planner, archive, + researchCollections, costLedger, calibration, comparability, @@ -91,6 +94,7 @@ function shutdown(signal: string): void { fleet.close(), planner.close(), archive.close(), + researchCollections.close(), costLedger.close(), calibration.close(), comparability.close(), From c6289c519452f15b0a3c5790e0f969d6a4dcab47 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:35:17 +0800 Subject: [PATCH 07/24] Expose immutable research collection list/read/export and operator-only capture API --- apps/api/src/app.ts | 107 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 7704b40..b5c60df 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -9,10 +9,11 @@ import { type PgRunPlanner, type RunJob, } from "@modelapse/control-plane"; -import { validateArchiveResearchFilters } from "@modelapse/persistence"; +import { exportResearchCollection, validateArchiveResearchFilters } from "@modelapse/persistence"; import type { ArchiveResearchFilters, PgArchiveRepository, + PgResearchCollections, PgCalibrationRepository, PgComparabilityRepository, PgCostLedger, @@ -108,6 +109,10 @@ type ComparabilityRepository = Pick< "ping" | "listPolicies" | "recordPolicy" >; +type ResearchCollectionsRepository = Pick< + PgResearchCollections, + "ping" | "list" | "get" | "create" +>; type ArchiveRepository = Pick< PgArchiveRepository, | "ping" @@ -129,6 +134,7 @@ export interface AppDependencies { readonly fleet?: ControlFleet; readonly planner?: ControlPlanner; readonly archive?: ArchiveRepository; + readonly researchCollections?: ResearchCollectionsRepository; readonly catalogDiscovery?: CatalogDiscoveryRepository; readonly catalogDriftReview?: CatalogDriftReviewRepository; readonly catalogIdentityCase?: CatalogIdentityCaseRepository; @@ -312,6 +318,105 @@ export function createApp(deps: AppDependencies) { } }); + app.get("/v1/archive/research/collections", async (c) => { + if (!deps.researchCollections) { + return c.json({ error: "research_collections_unavailable" }, 503); + } + const rawLimit = c.req.query("limit"); + const limit = rawLimit === undefined ? 30 : Number(rawLimit); + if (!Number.isInteger(limit) || limit < 1 || limit > 50) { + return c.json({ error: "invalid_limit" }, 400); + } + return c.json({ collections: await deps.researchCollections.list(limit) }); + }); + + app.get("/v1/archive/research/collections/:collectionId", async (c) => { + if (!deps.researchCollections) { + return c.json({ error: "research_collections_unavailable" }, 503); + } + const id = c.req.param("collectionId"); + if (!UUID_RE.test(id)) { + return c.json({ error: "invalid_research_collection_id" }, 400); + } + const collection = await deps.researchCollections.get(id); + return collection + ? c.json({ collection }) + : c.json({ error: "research_collection_not_found" }, 404); + }); + + app.get("/v1/archive/research/collections/:collectionId/export", async (c) => { + if (!deps.researchCollections) { + return c.json({ error: "research_collections_unavailable" }, 503); + } + const id = c.req.param("collectionId"); + if (!UUID_RE.test(id)) { + return c.json({ error: "invalid_research_collection_id" }, 400); + } + const format = c.req.query("format") ?? "json"; + if (format !== "csv" && format !== "json") { + return c.json({ error: "invalid_research_export_format" }, 400); + } + const collection = await deps.researchCollections.get(id); + if (!collection) return c.json({ error: "research_collection_not_found" }, 404); + const exported = exportResearchCollection(collection, format); + c.header("Content-Type", exported.mediaType); + c.header("Content-Disposition", `attachment; filename="${exported.filename}"`); + c.header("Cache-Control", "public, max-age=300"); + return c.body(exported.body); + }); + + app.post("/v1/control/research/collections", async (c) => { + if (!deps.researchCollections || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + let raw: unknown; + try { + raw = await c.req.json(); + } catch { + return c.json({ error: "invalid_json" }, 400); + } + const body = apiRecord(raw); + const filters = apiRecord(body?.filters); + const title = body?.title; + const actor = body?.actor; + const description = body?.description; + if ( + typeof title !== "string" || + typeof actor !== "string" || + (description !== undefined && typeof description !== "string") || + !filters || + Object.keys(filters).some((key) => ![ + "providerSlug", "modelId", "testCaseId", "evidence", "region", + "accountTier", "serviceTier", "cost" + ].includes(key)) || + Object.values(filters).some((value) => typeof value !== "string") + ) { + return c.json({ error: "invalid_research_collection_request" }, 400); + } + try { + const validatedFilters = validateArchiveResearchFilters( + filters as ArchiveResearchFilters, + ); + const collection = await deps.researchCollections.create({ + title, + actor, + ...(typeof description === "string" ? { description } : {}), + filters: validatedFilters, + }); + return c.json({ collection }, 201); + } catch (error) { + if (error instanceof Error && ( + error.message.startsWith("invalid_") || + error.message.startsWith("research_collection_") + )) { + return c.json({ error: error.message }, 400); + } + throw error; + } + }); + app.get("/v1/archive/changes", async (c) => { if (!deps.archive) { return c.json({ error: "archive_unavailable" }, 503); From 6d4e71a86e858b6980b86def6ec46d9c86619c5f Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:35:48 +0800 Subject: [PATCH 08/24] Add Web server functions for saving, sharing and downloading immutable research collections --- apps/web/src/modelapse.ts | 136 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index a02be36..cf7ba13 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -564,6 +564,21 @@ export interface ArchiveContextTransition { readonly caveats: readonly string[]; } +export interface ResearchCollectionSummary { + readonly id: string; + readonly title: string; + readonly description: string | null; + readonly filters: SearchArchiveResearchInput; + readonly runIds: readonly string[]; + readonly contentSha256: string; + readonly createdBy: string; + readonly createdAt: string; + readonly selectionLimit: 50; +} +export interface ResearchCollectionDetail extends ResearchCollectionSummary { + readonly runs: readonly ArchiveRun[]; +} + export interface ArchiveResearchPage { readonly runs: readonly ArchiveRun[]; readonly nextCursor: string | null; @@ -1538,6 +1553,21 @@ export interface SearchArchiveResearchInput { readonly limit?: number; } +interface CaptureResearchCollectionInput { + readonly operatorToken: string; + readonly title: string; + readonly description?: string; + readonly filters: SearchArchiveResearchInput; +} + +interface ResearchCollectionIdInput { + readonly collectionId: string; +} + +interface ResearchCollectionExportInput extends ResearchCollectionIdInput { + readonly format: "json" | "csv"; +} + interface ReadArchiveHistoryInput { readonly modelId: string; readonly testCaseId: string; @@ -2225,6 +2255,49 @@ function parseArchiveResearchInput(value: unknown): SearchArchiveResearchInput { return input as SearchArchiveResearchInput; } +function parseResearchCollectionId(value: unknown): ResearchCollectionIdInput { + if (!isRecord(value) || + typeof value.collectionId !== "string" || + !UUID_RE.test(value.collectionId)) { + throw new Error("Collection ID must be a UUID"); + } + return { collectionId: value.collectionId }; +} + +function parseResearchCollectionExport(value: unknown): ResearchCollectionExportInput { + const { collectionId } = parseResearchCollectionId(value); + if (!isRecord(value) || + (value.format !== "json" && value.format !== "csv")) { + throw new Error("Research export format must be json or csv"); + } + return { collectionId, format: value.format }; +} + +function parseCaptureResearchCollection(value: unknown): CaptureResearchCollectionInput { + const operatorToken = parseOperatorToken(value); + if (!isRecord(value) || + typeof value.title !== "string" || + value.title.trim().length < 3 || + value.title.trim().length > 120 || + (value.description !== undefined && + (typeof value.description !== "string" || + value.description.length > 2000))) { + throw new Error("Research collection title or description is invalid"); + } + const filters = parseArchiveResearchInput(value.filters); + if (filters.cursor) { + throw new Error("Saving a cursor page is not permitted"); + } + return { + operatorToken, + title: value.title.trim(), + ...(typeof value.description === "string" + ? { description: value.description.trim() } + : {}), + filters, + }; +} + function parseArchiveComparisonInput(value: unknown): CompareArchiveInput { if (!isRecord(value)) throw new Error("Archive comparison request must be an object"); @@ -2351,6 +2424,69 @@ export const searchArchiveResearch = createServerFn({ method: "POST" }) return result.research; }); +export const getResearchCollections = createServerFn({ method: "GET" }).handler( + async (): Promise => { + const result = await requestJson<{ + collections: readonly ResearchCollectionSummary[]; + }>("/v1/archive/research/collections?limit=30"); + return result.collections; + }, +); + +export const getResearchCollection = createServerFn({ method: "POST" }) + .validator(parseResearchCollectionId) + .handler(async ({ data }): Promise => { + try { + const result = await requestJson<{ collection: ResearchCollectionDetail }>( + `/v1/archive/research/collections/${data.collectionId}`, + ); + return result.collection; + } catch (error) { + if (error instanceof ApiRequestError && error.status === 404) return null; + throw error; + } + }); + +export const captureResearchCollection = createServerFn({ method: "POST" }) + .validator(parseCaptureResearchCollection) + .handler(async ({ data }): Promise => { + requireOperator(data.operatorToken); + const result = await requestJson<{ collection: ResearchCollectionDetail }>( + "/v1/control/research/collections", + { + control: true, + method: "POST", + body: { + title: data.title, + ...(data.description ? { description: data.description } : {}), + filters: data.filters, + actor: "web-operator", + }, + }, + ); + return result.collection; + }); + +export const downloadResearchCollection = createServerFn({ method: "POST" }) + .validator(parseResearchCollectionExport) + .handler(async ({ data }): Promise<{ body: string; filename: string; mediaType: string }> => { + const path = `/v1/archive/research/collections/${data.collectionId}/export?format=${data.format}`; + const response = await fetch(new URL(path, apiOrigin()), { + headers: { accept: data.format === "csv" ? "text/csv" : "application/json" }, + cache: "no-store", + }); + if (!response.ok) { + throw new ApiRequestError(response.status, "Research export unavailable"); + } + return { + body: await response.text(), + filename: `modelapse-research-${data.collectionId}.${data.format}`, + mediaType: data.format === "csv" + ? "text/csv; charset=utf-8" + : "application/json; charset=utf-8", + }; + }); + export const getArchiveComparabilityPolicies = createServerFn({ method: "GET" }).handler( async (): Promise => { const result = await requestJson<{ policies: readonly ArchiveComparabilityPolicy[] }>( From d9ce0337a94d7bb83beca88a7dc67a2a2c567cc1 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:36:19 +0800 Subject: [PATCH 09/24] Add public saved research collection gallery --- apps/web/src/routes/research.collections.tsx | 56 ++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 apps/web/src/routes/research.collections.tsx diff --git a/apps/web/src/routes/research.collections.tsx b/apps/web/src/routes/research.collections.tsx new file mode 100644 index 0000000..b453e07 --- /dev/null +++ b/apps/web/src/routes/research.collections.tsx @@ -0,0 +1,56 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { getResearchCollections } from "../modelapse"; + +export const Route = createFileRoute("/research/collections")({ + loader: () => getResearchCollections(), + component: ResearchCollectionIndex, +}); + +function ResearchCollectionIndex() { + const collections = Route.useLoaderData(); + return ( +
+
+ + M + ModelapseAI Model Test & Evolution Archive + + +
+
+
+ Research/Saved collections +
+

ARCHIVE v0.27 · REPRODUCIBLE RESEARCH

+

Published research snapshots

+

+ These operator-curated collections freeze a finite set of sealed public + non-calibration Run IDs. Dynamic filters may find newer Runs later, + but an existing collection never silently changes membership. +

+
+
+

IMMUTABLE MANIFESTS

Recently captured collections

+
+ {collections.map((collection) => ( +
+
+ {collection.title} + {collection.runIds.length} frozen Runs +
+ {collection.description ?

{collection.description}

: null} +

Captured {collection.createdAt.replace("T", " ").replace(".000Z", "Z")}

+ Manifest SHA-256: {collection.contentSha256} +

Inspect snapshot and exports →

+
+ ))} + {!collections.length ?
No public snapshots yet. An operator can capture a bounded research slice from the Research Explorer.
: null} +
+
+ +
+ ); +} From 3bc7de2baa774683fee8af78e6f21f2fea0d6f65 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:36:22 +0800 Subject: [PATCH 10/24] Add verifiable research snapshot detail and JSON/CSV download controls --- .../research.collections.$collectionId.tsx | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 apps/web/src/routes/research.collections.$collectionId.tsx diff --git a/apps/web/src/routes/research.collections.$collectionId.tsx b/apps/web/src/routes/research.collections.$collectionId.tsx new file mode 100644 index 0000000..17646c5 --- /dev/null +++ b/apps/web/src/routes/research.collections.$collectionId.tsx @@ -0,0 +1,145 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { useState } from "react"; +import { + downloadResearchCollection, + getResearchCollection, +} from "../modelapse"; + +export const Route = createFileRoute("/research/collections/$collectionId")({ + loader: ({ params }) => getResearchCollection({ + data: { collectionId: params.collectionId }, + }), + component: ResearchCollectionDetailPage, +}); + +function ResearchCollectionDetailPage() { + const collection = Route.useLoaderData(); + const [exporting, setExporting] = useState<"csv" | "json" | null>(null); + const [error, setError] = useState(null); + + async function exportSnapshot(format: "csv" | "json"): Promise { + if (!collection) return; + setExporting(format); + setError(null); + try { + const result = await downloadResearchCollection({ + data: { collectionId: collection.id, format }, + }); + const blob = new Blob([result.body], { type: result.mediaType }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = result.filename; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Export failed"); + } finally { + setExporting(null); + } + } + + return ( +
+
+ + M + ModelapseAI Model Test & Evolution Archive + + +
+
+
+ Research/ + Collections/ + {collection?.id.slice(0, 8) ?? "Unknown"} +
+

ARCHIVE v0.27 · FROZEN EVIDENCE SET

+

{collection?.title ?? "Research collection not found"}

+

+ {collection?.description ?? + "The collection manifest records exact sealed Run identities, not an automatically refreshing query."} +

+
+ {collection ? ( + <> +
+
+

REPRODUCIBILITY

Manifest & exports

+ {collection.runIds.length} captured Runs +
+

+ Captured {collection.createdAt} · scope: sealed public non-calibration + · selection limit {collection.selectionLimit}. The manifest stays unchanged + when newer Runs arrive. Comparability and calibration eligibility have + not been evaluated for this collection. +

+

SHA-256 manifest: {collection.contentSha256}

+

+ Frozen filters: {JSON.stringify(collection.filters)} +

+
+ + + typeof value === "string") + .map(([key, value]) => [ + key === "providerSlug" ? "provider" : key, String(value), + ]), + ).toString()}`}>Re-run these filters (live) → +
+ {error ?
{error}
: null} +
+
+
+

CAPTURED SOURCE RECORDS

Run evidence

+
+
+ {collection.runs.map((run) => ( +
+
+ {run.provider.name} · {run.model.marketingName ?? run.requestedModel} + {run.evidenceLevel ?? "unknown evidence"} +
+

+ {run.test.familySlug}/{run.test.caseSlug} · {run.executionPath} + {" · "}region {run.executionQualification?.executionRegion ?? "unknown"} + {" · "}account {run.executionQualification?.accountTier ?? "unknown"} + {" · "}tier {run.executionQualification?.returnedServiceTier ?? + run.executionQualification?.serviceTier ?? "unknown"} +

+

+ Pricing {run.cost?.estimatedNativeCost ?? "unknown (not free)"} + {" "}{run.cost?.pricing?.currency ?? ""} + {" · "}Qualification caveats: {run.executionQualification?.caveats.join(" · ") || + "unknown/none reported"} + {" · "}Cost caveats: {run.cost?.caveats.join(" · ") || + "unknown/none reported"} +

+ Verify Run → + {" · "} + {run.model.id ? History → : null} +
+ ))} +
+
+ + ) : ( +
No public research collection exists for this ID.
+ )} + +
+ ); +} From 1f031826315652667f17a27c130d1b8129c2daf0 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:36:43 +0800 Subject: [PATCH 11/24] Allow operator to publish bounded immutable snapshot from live research filters --- apps/web/src/routes/research.tsx | 95 ++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/apps/web/src/routes/research.tsx b/apps/web/src/routes/research.tsx index ff9d458..1338ae8 100644 --- a/apps/web/src/routes/research.tsx +++ b/apps/web/src/routes/research.tsx @@ -1,7 +1,9 @@ import { createFileRoute } from "@tanstack/react-router"; +import { useState } from "react"; import { getArchiveCatalog, searchArchiveResearch, + captureResearchCollection, type ArchiveRun, type SearchArchiveResearchInput, } from "../modelapse"; @@ -90,6 +92,44 @@ function evaluation(run: ArchiveRun): string { function ArchiveResearchPage() { const { catalog, research } = Route.useLoaderData(); const filters = Route.useSearch(); + const [operatorToken, setOperatorToken] = useState(""); + const [collectionTitle, setCollectionTitle] = useState(""); + const [collectionNote, setCollectionNote] = useState(""); + const [capturing, setCapturing] = useState(false); + const [collectionId, setCollectionId] = useState(null); + const [captureError, setCaptureError] = useState(null); + + async function captureCollection(): Promise { + setCaptureError(null); + setCollectionId(null); + setCapturing(true); + try { + const collection = await captureResearchCollection({ + data: { + operatorToken, + title: collectionTitle, + ...(collectionNote.trim() ? { description: collectionNote.trim() } : {}), + filters: { + ...(filters.provider ? { providerSlug: filters.provider } : {}), + ...(filters.modelId ? { modelId: filters.modelId } : {}), + ...(filters.testCaseId ? { testCaseId: filters.testCaseId } : {}), + ...(filters.evidence ? { evidence: filters.evidence } : {}), + ...(filters.region ? { region: filters.region } : {}), + ...(filters.accountTier ? { accountTier: filters.accountTier } : {}), + ...(filters.serviceTier ? { serviceTier: filters.serviceTier } : {}), + ...(filters.cost ? { cost: filters.cost } : {}), + }, + }, + }); + setCollectionId(collection.id); + } catch (caught) { + setCaptureError( + caught instanceof Error ? caught.message : "Collection capture failed", + ); + } finally { + setCapturing(false); + } + } const providerOptions = [...new Map( catalog.models.map((model) => [model.provider.slug, model.provider.name]), )].sort(([a], [b]) => a.localeCompare(b)); @@ -119,6 +159,7 @@ function ArchiveResearchPage() { @@ -221,6 +262,60 @@ function ArchiveResearchPage() {

+
+
+

ARCHIVE v0.27 · CURATED RESEARCH

Save an immutable collection

+ Browse saved collections → +
+

+ Research filters above are live and can find new Runs. Saving creates a + public immutable snapshot of the current matching Run IDs (1–50 only). + Capture requires operator access. Empty or oversized slices are rejected + rather than silently truncated. Collections never include calibration Runs. +

+
+ Operator · capture this filtered slice +
+ + + +
+ + Saved manifests store no Provider credentials or account identifiers. +
+
+ {captureError ?
{captureError}
: null} + {collectionId ? ( +
+ Snapshot captured ·{" "} + + Open immutable collection → + +
+ ) : null} +
+
+
From 8df93c13271168c18802ffc0add20421e6211031 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:37:12 +0800 Subject: [PATCH 12/24] Test reproducible manifest digest, sourced exports and CSV formula injection defense --- .../test/research-collections.test.ts | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 packages/persistence/test/research-collections.test.ts diff --git a/packages/persistence/test/research-collections.test.ts b/packages/persistence/test/research-collections.test.ts new file mode 100644 index 0000000..efd2d99 --- /dev/null +++ b/packages/persistence/test/research-collections.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it } from "vitest"; +import { + researchCollectionDigest, + type ResearchCollectionDetail, +} from "../src/research-collections.js"; +import { exportResearchCollection } from "../src/research-export.js"; +import type { ArchiveRunView } from "../src/archive-repository.js"; + +const runId = "00000000-0000-4000-8000-000000000401"; +const collectionId = "00000000-0000-4000-8000-000000000402"; +const mockRun = { + id: runId, + provider: { slug: "=HYPERLINK(\"evil\")", name: "Fixture" }, + model: { id: null, canonicalSlug: "test", marketingName: "Test" }, + test: { + testCaseId: "00000000-0000-4000-8000-000000000403", + familySlug: "smoke", variantSlug: "text", version: "1.0.0", + caseSlug: "exact", + }, + requestedModel: "test", + completedAt: "2026-10-08T00:00:01.000Z", + evidenceLevel: "E4", + evaluation: { status: "completed", exactMatch: true }, + executionPath: "first_party_direct", + executionQualification: { + executionRegion: "US", accountTier: "paid", serviceTier: "default", + returnedServiceTier: "default", serviceAssurance: "documented_default", + providerPolicyObservation: { sourceId: "source-policy" }, + runnerAccessObservation: { sourceId: "source-runner" }, + caveats: ["operator_uncertain"], + }, + cost: { + pricingObservation: { sourceId: "source-price" }, + pricing: { currency: "USD" }, + estimatedNativeCost: "0.001", + caveats: ["pricing_tier_generic"], + }, +} as ArchiveRunView; + +function collection(): ResearchCollectionDetail { + const filters = { providerSlug: "deepseek", evidence: "E4+" as const }; + return { + id: collectionId, + title: "Fixture snapshot", + description: "A bounded historical Run set", + createdAt: "2026-10-08T00:10:00.000Z", + createdBy: "test", + selectionLimit: 50, + filters, + runIds: [runId], + contentSha256: researchCollectionDigest(filters, [runId]), + runs: [mockRun], + }; +} + +describe("research snapshot manifest and exports", () => { + it("identifies exact filters and ordered Run memberships without clock dependence", () => { + const a = researchCollectionDigest({ providerSlug: "deepseek", evidence: "E4+" }, [runId]); + const b = researchCollectionDigest({ evidence: "E4+", providerSlug: "deepseek" }, [runId]); + expect(a).toBe(b); + expect(a).toMatch(/^[a-f0-9]{64}$/); + expect( + researchCollectionDigest({ providerSlug: "openai", evidence: "E4+" }, [runId]), + ).not.toBe(a); + expect( + researchCollectionDigest({ providerSlug: "deepseek", evidence: "E4+" }, [ + "00000000-0000-4000-8000-000000000404", runId, + ]), + ).not.toBe(a); + }); + + it("exports source IDs, unknown-safe caveats and a verifiable manifest", () => { + const json = exportResearchCollection(collection(), "json"); + expect(json.mediaType).toContain("application/json"); + const output = JSON.parse(json.body) as { + manifest: { contentSha256: string; runIds: string[] }; + rows: { providerPolicySourceId: string; pricingSourceId: string; qualificationCaveats: string[] }[]; + interpretation: { rankingsProvided: boolean }; + }; + expect(output.manifest.runIds).toEqual([runId]); + expect(output.manifest.contentSha256).toBe(collection().contentSha256); + expect(output.rows[0]).toMatchObject({ + providerPolicySourceId: "source-policy", + pricingSourceId: "source-price", + qualificationCaveats: ["operator_uncertain"], + }); + expect(output.interpretation.rankingsProvided).toBe(false); + }); + + it("produces CSV with one record and defangs spreadsheet formula injection", () => { + const csv = exportResearchCollection(collection(), "csv"); + expect(csv.mediaType).toContain("text/csv"); + expect(csv.body.split("\r\n")).toHaveLength(3); + expect(csv.body).toContain("\"'\=HYPERLINK".replace("\\=", "=")); + expect(csv.body).toContain("pricing_tier_generic"); + expect(csv.body).toContain("source-price"); + }); + + it("rejects unknown export formats", () => { + expect(() => + exportResearchCollection(collection(), "html" as "json"), + ).toThrow("invalid_research_export_format"); + }); +}); From 7d40ee7a9d76625842a5c35a42502bcf0c3010ea Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:37:28 +0800 Subject: [PATCH 13/24] Simplify CSV formula-injection test literal --- packages/persistence/test/research-collections.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistence/test/research-collections.test.ts b/packages/persistence/test/research-collections.test.ts index efd2d99..0037f4d 100644 --- a/packages/persistence/test/research-collections.test.ts +++ b/packages/persistence/test/research-collections.test.ts @@ -91,7 +91,7 @@ describe("research snapshot manifest and exports", () => { const csv = exportResearchCollection(collection(), "csv"); expect(csv.mediaType).toContain("text/csv"); expect(csv.body.split("\r\n")).toHaveLength(3); - expect(csv.body).toContain("\"'\=HYPERLINK".replace("\\=", "=")); + expect(csv.body).toContain("\"'=HYPERLINK"); expect(csv.body).toContain("pricing_tier_generic"); expect(csv.body).toContain("source-price"); }); From 91c5d776113df1886d9a02ae82ac9bc4a8204d2e Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:37:53 +0800 Subject: [PATCH 14/24] Test PostgreSQL research snapshot save/reopen, immutable manifest and forged Run rejection --- .../test/comparability.integration.test.ts | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/packages/persistence/test/comparability.integration.test.ts b/packages/persistence/test/comparability.integration.test.ts index 05ec314..000cfe0 100644 --- a/packages/persistence/test/comparability.integration.test.ts +++ b/packages/persistence/test/comparability.integration.test.ts @@ -4,6 +4,7 @@ import { afterAll, describe, expect, it } from "vitest"; import { PgArchiveRepository, PgComparabilityRepository, + PgResearchCollections, } from "../src/index.js"; const DATABASE_URL = @@ -14,10 +15,12 @@ describe("Archive comparability policy", () => { const pool = new Pool({ connectionString: DATABASE_URL }); const archive = PgArchiveRepository.connect(DATABASE_URL, { max: 2 }); const policies = PgComparabilityRepository.connect(DATABASE_URL, { max: 2 }); + const collections = PgResearchCollections.connect(DATABASE_URL, archive, { max: 2 }); afterAll(async () => { await archive.close(); await policies.close(); + await collections.close(); await pool.end(); }); @@ -540,6 +543,57 @@ describe("Archive comparability policy", () => { ), ).toBe(true); + const collection = await collections.create({ + title: "Captured comparison " + suffix, + actor: "comparability-integration", + description: "Frozen Run identities for a single canonical Test", + filters: { + providerSlug: "comparability-provider-" + suffix + "-2", + testCaseId: benchmarkCase.rows[0]!.id, + evidence: "E4+", + }, + }); + expect(collection.runIds).toHaveLength(2); + expect(collection.runs.map((run) => run.id)).toEqual(collection.runIds); + expect(collection.contentSha256).toMatch(/^[a-f0-9]{64}$/); + const reopened = await collections.get(collection.id); + expect(reopened?.contentSha256).toBe(collection.contentSha256); + expect(reopened?.runs.map((run) => run.id)).toEqual(collection.runIds); + expect((await collections.list()).some((item) => item.id === collection.id)).toBe(true); + + await expect( + collections.create({ + title: "Invalid historical cursor", + actor: "comparability-integration", + filters: { + modelId: models[1]!, + cursor: research.nextCursor!, + }, + }), + ).rejects.toThrow("research_collection_cursor_not_allowed"); + + await expect( + pool.query( + `UPDATE modelapse.research_collections + SET title = 'rewritten' + WHERE id = $1`, + [collection.id], + ), + ).rejects.toThrow(/append-only/i); + + await expect( + pool.query( + `INSERT INTO modelapse.research_collections + (title, filters, run_ids, content_sha256, created_by) + VALUES ($1, '{}'::jsonb, $2::uuid[], $3, 'integration')`, + [ + "Forged private Run set " + suffix, + [randomUUID()], + "a".repeat(64), + ], + ), + ).rejects.toThrow(/sealed public non-calibration Runs/i); + const custom = await policies.recordPolicy({ version: "comparability-integration-" + suffix, minimumEvidenceLevel: "E4", From ce6be72a17bfcb5e8e41efbede91a6dbb95ae111 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:38:15 +0800 Subject: [PATCH 15/24] Test public research collection browsing/export, operator-only capture and validation --- apps/api/test/archive-api.test.ts | 107 ++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) diff --git a/apps/api/test/archive-api.test.ts b/apps/api/test/archive-api.test.ts index 75f8325..9098fa5 100644 --- a/apps/api/test/archive-api.test.ts +++ b/apps/api/test/archive-api.test.ts @@ -733,3 +733,110 @@ describe("Archive read API", () => { }); }); + + +describe("Archive v0.27 immutable research collections API", () => { + it("allows public browsing and export, but requires operator auth for capture", async () => { + const id = "00000000-0000-4000-8000-000000000488"; + const collection = { + id, + title: "Source-backed research snapshot", + description: "Immutable selection of one first-party Run", + filters: { modelId: MODEL_ID, evidence: "E4+" as const }, + runIds: [RUN_ID], + contentSha256: "a".repeat(64), + createdBy: "test-operator", + createdAt: "2026-10-08T00:00:00.000Z", + selectionLimit: 50 as const, + runs: [archiveRun], + }; + let createdInput: unknown; + const app = createApp({ + runs: baseRuns(), + controlToken: "control-secret", + researchCollections: { + ping: async () => undefined, + list: async () => [{ ...collection }], + get: async (collectionId) => collectionId === id ? collection : null, + create: async (input) => { + createdInput = input; + return collection; + }, + }, + }); + + const listed = await app.request("/v1/archive/research/collections"); + expect(listed.status).toBe(200); + await expect(listed.json()).resolves.toMatchObject({ + collections: [{ id, title: collection.title, runIds: [RUN_ID] }], + }); + const detail = await app.request( + "/v1/archive/research/collections/" + id, + ); + expect(detail.status).toBe(200); + await expect(detail.json()).resolves.toMatchObject({ + collection: { id, runs: [{ id: RUN_ID }] }, + }); + + const json = await app.request( + "/v1/archive/research/collections/" + id + "/export?format=json", + ); + expect(json.status).toBe(200); + expect(json.headers.get("content-disposition")).toContain(".json"); + await expect(json.json()).resolves.toMatchObject({ + manifest: { id, runIds: [RUN_ID], scope: "sealed_public_non_calibration" }, + interpretation: { rankingsProvided: false }, + rows: [{ runId: RUN_ID, pricingSourceId: null }], + }); + const csv = await app.request( + "/v1/archive/research/collections/" + id + "/export?format=csv", + ); + expect(csv.status).toBe(200); + expect(csv.headers.get("content-type")).toContain("text/csv"); + expect(await csv.text()).toContain(RUN_ID); + + const unauthorized = await app.request("/v1/control/research/collections", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ title: "Bad snapshot", actor: "guest", filters: {} }), + }); + expect(unauthorized.status).toBe(401); + const authorizedHeaders = { + authorization: "Bearer control-secret", + "content-type": "application/json", + }; + const created = await app.request("/v1/control/research/collections", { + method: "POST", + headers: authorizedHeaders, + body: JSON.stringify({ + title: "Snapshot from API", + actor: "test-operator", + filters: { modelId: MODEL_ID, evidence: "E4+" }, + }), + }); + expect(created.status).toBe(201); + expect(createdInput).toMatchObject({ + title: "Snapshot from API", + actor: "test-operator", + filters: { modelId: MODEL_ID, evidence: "E4+" }, + }); + + for (const path of [ + "/v1/archive/research/collections/nope", + "/v1/archive/research/collections/" + id + "/export?format=html", + "/v1/archive/research/collections?limit=51", + ]) { + expect((await app.request(path)).status, path).toBe(400); + } + const invalidCapture = await app.request("/v1/control/research/collections", { + method: "POST", + headers: authorizedHeaders, + body: JSON.stringify({ + title: "Forged", + actor: "test", + filters: { injectedFilter: "abc" }, + }), + }); + expect(invalidCapture.status).toBe(400); + }); +}); From ced2f90f2ad20339a68e2a4506ea5c6d786bc2f6 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:38:44 +0800 Subject: [PATCH 16/24] Use explicit column alias for PostgreSQL snapshot ID uniqueness check --- packages/database/migrations/0025_research_collections.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/database/migrations/0025_research_collections.sql b/packages/database/migrations/0025_research_collections.sql index 0b5106c..05e097c 100644 --- a/packages/database/migrations/0025_research_collections.sql +++ b/packages/database/migrations/0025_research_collections.sql @@ -24,7 +24,8 @@ DECLARE run_count integer; BEGIN IF cardinality(NEW.run_ids) <> ( - SELECT COUNT(DISTINCT value) FROM unnest(NEW.run_ids) AS value + SELECT COUNT(DISTINCT input.run_id) + FROM unnest(NEW.run_ids) AS input(run_id) ) THEN RAISE EXCEPTION 'research collection contains duplicate Run IDs'; END IF; From 57ea473cf7851b765ef66d130e42bebe3f7ebcff Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:38:47 +0800 Subject: [PATCH 17/24] Type live research filter URL tuples for snapshot detail --- apps/web/src/routes/research.collections.$collectionId.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/src/routes/research.collections.$collectionId.tsx b/apps/web/src/routes/research.collections.$collectionId.tsx index 17646c5..40b0905 100644 --- a/apps/web/src/routes/research.collections.$collectionId.tsx +++ b/apps/web/src/routes/research.collections.$collectionId.tsx @@ -97,7 +97,7 @@ function ResearchCollectionDetailPage() { .filter(([, value]) => typeof value === "string") .map(([key, value]) => [ key === "providerSlug" ? "provider" : key, String(value), - ]), + ] as [string, string]), ).toString()}`}>Re-run these filters (live) →
{error ?
{error}
: null} From 1bc26dbcc190cb6a1085bee970597506a3cf8dd9 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:38:50 +0800 Subject: [PATCH 18/24] Run v0.27 research digest and export tests in CI --- packages/persistence/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistence/package.json b/packages/persistence/package.json index 932f3ce..10a3091 100644 --- a/packages/persistence/package.json +++ b/packages/persistence/package.json @@ -10,7 +10,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", - "test": "vitest run test/record-provider-run.test.ts test/archive-context-transitions.test.ts test/archive-research.test.ts", + "test": "vitest run test/record-provider-run.test.ts test/archive-context-transitions.test.ts test/archive-research.test.ts test/research-collections.test.ts", "test:integration": "vitest run test/integration.test.ts test/comparability.integration.test.ts" }, "dependencies": { From b3e8c7ea88287fed509aa1d8adfa547f5952de9f Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:39:18 +0800 Subject: [PATCH 19/24] Document Archive v0.27 immutable snapshots and sourced exports --- docs/research-collections-exports.md | 92 ++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 docs/research-collections-exports.md diff --git a/docs/research-collections-exports.md b/docs/research-collections-exports.md new file mode 100644 index 0000000..6db7733 --- /dev/null +++ b/docs/research-collections-exports.md @@ -0,0 +1,92 @@ +# Research Collections & Evidence Exports — Archive v0.27 + +Archive v0.27 introduces **operator-curated, publicly shareable immutable Run selections** and sourced JSON/CSV exports. + +## What is saved? + +A dynamic Research Explorer URL (`/research?…`) is a query. As the Archive collects new Runs, repeating that query may return different records. + +A Research Collection (`/research/collections/:id`) freezes **a specific ordered list of up to 50 sealed Run IDs**, the original validated filters, title/note, creator label and capture time. Existing membership never changes as more Runs arrive. + +A collection is a *selection snapshot*, not a byte-for-byte copy of the entire public Run view. It references sealed, immutable Run facts, qualification envelopes and cost evidence. Derived policy interpretations may evolve separately. Export includes a content digest for the selection manifest, not a hash of the exported JSON/CSV bytes. + +Creation fails closed when: + +- no matching Run exists; +- more than 50 matching Runs exist (no silent first-page truncation); +- a caller supplies a pagination cursor (a partial page is not a complete slice); +- filter, title or creator metadata is invalid; +- one of the selected IDs is not a completed, sealed, public, non-calibration Run. + +## Control and public endpoints + +**Operator-only, token authenticated:** + +`POST /v1/control/research/collections` + +Body example: + +```json +{ + "title": "US paid-account snapshot", + "description": "Bounded first-party Run selection", + "actor": "archive-operator", + "filters": { + "providerSlug": "deepseek", + "evidence": "E4+", + "region": "US", + "cost": "unknown" + } +} +``` + +The API uses the same validated facets as `GET /v1/archive/research`, but caps the capture at 50 and refuses a partial result. + +**Public, read-only:** + +```text +GET /v1/archive/research/collections?limit=30 +GET /v1/archive/research/collections/:collectionId +GET /v1/archive/research/collections/:collectionId/export?format=json +GET /v1/archive/research/collections/:collectionId/export?format=csv +``` + +The export response uses the appropriate MIME type and attachment filename. + +## Manifest identity + +`research_collections` is append-only. PostgreSQL validates every selected Run's sealed/public/non-calibration status on insert and rejects duplicate Run IDs. + +The SHA-256 `contentSha256` uses canonical JSON of: + +- manifest schema version; +- normalized Provider, Model, Test, evidence, region, account, service tier and cost facets; +- ordered Run UUIDs. + +It excludes mutable presentation details such as title and capture timestamp. The repository verifies the digest again when the collection is read. This hash proves the identity of **the selected filters and Run IDs**, not a cryptographic attestation by the Provider or a guarantee that the entire externally downloaded file has never been modified. + +## Exports and evidence language + +Each exported row retains immutable identifiers, execution path, completion time, E-level, deterministic evaluator outcome, region/account/effective service tier/service assurance, referenced first-party access/policy evidence source IDs, pricing source ID, native currency/cost estimate and qualification/cost caveats. + +JSON includes the full manifest, explicit interpretation scope and structured rows. + +CSV includes one flat sourced row per Run; all cells are quoted, embedded quotes escaped, and values beginning with spreadsheet formula triggers are prefixed to prevent formula execution in spreadsheet software. + +Cost unavailable remains empty or null, **never zero or free**. CSV/JSON is not a ranking, not a matched comparability set, and not proof of model degradation. Calibration Tests are excluded from snapshot membership; calibration eligibility itself is not silently inferred. + +## UI + +- `/research`: live filters plus a collapsed Operator-only capture form; +- `/research/collections`: public recently published collections; +- `/research/collections/:id`: fixed Run membership, manifest digest, filter replay link and JSON/CSV download. + +Replaying a collection's filters launches a **new live search**, not an update to the frozen collection. Public viewing does not require operator credentials. Provider secrets and personal account identifiers never enter the collection. + +## Tests and deployment + +Migration `0025_research_collections.sql` adds the append-only manifest table and Run status/visibility validation trigger. It is idempotently applied by the existing migration runner. + +Tests cover digest determinism, CSV formula escaping, public GET/export vs operator POST boundaries, re-open of a captured collection in PostgreSQL, cursor rejection and append-only/forged Run constraints. + +If a selection grows beyond 50, operators must narrow it (for example by Provider, exact Test, or historical context). Future phases may introduce explicit multi-page snapshot transactions, but may not silently save only a partial result. From abf617d611f04584012cdd966c2e2b8317932b92 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:39:20 +0800 Subject: [PATCH 20/24] ADR 0023: freeze research selection IDs, not live query results --- docs/adr/0023-immutable-research-selection.md | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 docs/adr/0023-immutable-research-selection.md diff --git a/docs/adr/0023-immutable-research-selection.md b/docs/adr/0023-immutable-research-selection.md new file mode 100644 index 0000000..0f63a6d --- /dev/null +++ b/docs/adr/0023-immutable-research-selection.md @@ -0,0 +1,28 @@ +# ADR 0023 — Saved research collections freeze Run identity, not live query results + +## Status + +Accepted for Archive v0.27. + +## Context + +Archive v0.26 offers shareable filter URLs, but rerunning a query after new collection may return different Runs. A published research citation must not silently expand or drift. + +Anonymous persistence endpoints would also permit unauthenticated public content creation and abuse. + +## Decision + +1. Define research collections as append-only manifests of validated filters and **ordered sealed public non-calibration Run IDs**. +2. Allow collection capture only through the operator-authenticated control API. +3. Make saved collection reading and structured JSON/CSV export public. +4. Cap captures at 50 Runs and reject empty, oversized, or cursor-restricted selections. +5. Compute a canonical SHA-256 selection-manifest digest; verify it when reading. +6. Validate snapshot membership with PostgreSQL trigger rules and prevent updates/deletes. +7. Keep CSV exports safe from spreadsheet formula injection, preserve evidence source references and unknown cost as unknown. +8. Distinguish query replay from fixed Run membership and from derived policy eligibility. +9. Treat the manifest hash as selection identity, not as a provider-signed result or a hash of all response blobs. +10. Never change the immutable Run, Evaluation, qualification or cost ledger as part of research snapshot creation. + +## Consequences + +Public citations can reference stable evidence selections while allowing live faceted discovery to evolve. Longer research collections need a future explicit bounded-batch protocol instead of truncating queries without disclosure. From c648b169d4dbf8e36cb0089cf63ce4fff0275f53 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:39:32 +0800 Subject: [PATCH 21/24] Mark v0.27 immutable collections and sourced exports implemented; outline v0.28 research facets --- docs/archive-roadmap.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index 6a1e370..7abbaa7 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -309,9 +309,17 @@ The new `/research` public surface supports server-side bounded browsing of seal Every research row shows qualification and cost caveats and links into immutable Run, Model, Test, History and same-Test Compare. Reproducible URL filters, deterministic keyset pagination and dedicated indexes support broader discovery without ranking Models or equating collection volume with quality. -### Archive v0.27+ — Extended research workflows +### Archive v0.27 — Immutable Research Collections & Sourced Exports -Consider saved/shareable research collections, sourced CSV/JSON exports, typed facet discovery, and richer policy-aware comparisons while preserving the evidence-first, no-aggregate-ranking boundary. Continue to make registration, account, region and calibration restrictions explicit. +**Implemented:** see `docs/research-collections-exports.md` and ADR 0023. + +Operators can publish bounded, immutable collections of up to 50 sealed public non-calibration Run IDs. Every public collection retains the normalized query filters, an ordered Run manifest and a SHA-256 selection digest; new Runs never silently alter its membership. + +Public collection pages support sharing, live filter replay and source-backed JSON/CSV exports with qualification, access, pricing, region/tier and evidence caveats. Empty, oversized and cursor-partial captures fail closed. The export is not a comparability or quality ranking. + +### Archive v0.28+ — Typed Research Facets & Policy-Aware Workflows + +Next, consider facet discovery with observed counts, policy-version anchored research annotations and reproducible multi-page collection protocols. Explicitly distinguish current Provider policy from historical Run state; never aggregate heterogeneous Test results into a leaderboard. ## Scheduling principle From ba3c119d63144352be7dcc704cb62dc17aaadf80 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:39:57 +0800 Subject: [PATCH 22/24] Allow independent append-only captures of an identical research selection --- packages/database/migrations/0025_research_collections.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/database/migrations/0025_research_collections.sql b/packages/database/migrations/0025_research_collections.sql index 05e097c..be0f9ca 100644 --- a/packages/database/migrations/0025_research_collections.sql +++ b/packages/database/migrations/0025_research_collections.sql @@ -11,8 +11,7 @@ CREATE TABLE research_collections ( content_sha256 text NOT NULL CHECK (content_sha256 ~ '^[a-f0-9]{64}$'), created_by text NOT NULL CHECK (char_length(btrim(created_by)) > 0), created_at timestamptz NOT NULL DEFAULT now(), - selection_limit integer NOT NULL DEFAULT 50 CHECK (selection_limit = 50), - UNIQUE (content_sha256, title) + selection_limit integer NOT NULL DEFAULT 50 CHECK (selection_limit = 50) ); CREATE INDEX research_collections_created_idx From 1edb7a52c2861d5133dccf80813a922df3e4f9be Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:41:41 +0800 Subject: [PATCH 23/24] Complete strongly typed Run fixture for research export tests --- .../test/research-collections.test.ts | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/packages/persistence/test/research-collections.test.ts b/packages/persistence/test/research-collections.test.ts index 0037f4d..43d41d6 100644 --- a/packages/persistence/test/research-collections.test.ts +++ b/packages/persistence/test/research-collections.test.ts @@ -8,9 +8,10 @@ import type { ArchiveRunView } from "../src/archive-repository.js"; const runId = "00000000-0000-4000-8000-000000000401"; const collectionId = "00000000-0000-4000-8000-000000000402"; -const mockRun = { +const mockRun: ArchiveRunView = { id: runId, - provider: { slug: "=HYPERLINK(\"evil\")", name: "Fixture" }, + status: "completed", + provider: { id: "00000000-0000-4000-8000-000000000405", slug: "=HYPERLINK(\"evil\")", name: "Fixture" }, model: { id: null, canonicalSlug: "test", marketingName: "Test" }, test: { testCaseId: "00000000-0000-4000-8000-000000000403", @@ -18,24 +19,52 @@ const mockRun = { caseSlug: "exact", }, requestedModel: "test", + returnedModel: "test", + runnerBuild: "research-export-test", + createdAt: "2026-10-08T00:00:00.000Z", + sealedAt: "2026-10-08T00:00:03.000Z", completedAt: "2026-10-08T00:00:01.000Z", evidenceLevel: "E4", - evaluation: { status: "completed", exactMatch: true }, + evaluation: { + id: "00000000-0000-4000-8000-000000000406", + status: "completed", exactMatch: true, + evaluatorSlug: "exact-text", evaluatorVersion: "1.0.0", + evaluatorKind: "deterministic", definitionSha256: "a".repeat(64), + rawResultSha256: null, + }, executionPath: "first_party_direct", executionQualification: { + selectedAt: "2026-10-08T00:00:00.000Z", + executionEnvironment: null, + requestedServiceTier: "default", + contextKey: null, executionRegion: "US", accountTier: "paid", serviceTier: "default", returnedServiceTier: "default", serviceAssurance: "documented_default", - providerPolicyObservation: { sourceId: "source-policy" }, - runnerAccessObservation: { sourceId: "source-runner" }, + providerPolicyObservation: { + id: "00000000-0000-4000-8000-000000000407", + sourceId: "source-policy", accessState: "available", + }, + runnerAccessObservation: { + id: "00000000-0000-4000-8000-000000000408", + sourceId: "source-runner", accessState: "available", + }, caveats: ["operator_uncertain"], }, cost: { - pricingObservation: { sourceId: "source-price" }, - pricing: { currency: "USD" }, + selectedAt: "2026-10-08T00:00:00.000Z", + pricingObservation: { + id: "00000000-0000-4000-8000-000000000409", + sourceId: "source-price", + }, + pricing: { + currency: "USD", + inputPerMillion: "1.0", outputPerMillion: "2.0", perRequest: null, + }, + usage: null, estimatedNativeCost: "0.001", caveats: ["pricing_tier_generic"], }, -} as ArchiveRunView; +}; function collection(): ResearchCollectionDetail { const filters = { providerSlug: "deepseek", evidence: "E4+" as const }; From f9d4d091a8d86a05a4793012232eaf4271f493a7 Mon Sep 17 00:00:00 2001 From: WangEn Date: Thu, 8 Oct 2026 11:43:10 +0800 Subject: [PATCH 24/24] Complete Test identity in strongly typed research export fixture --- packages/persistence/test/research-collections.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/persistence/test/research-collections.test.ts b/packages/persistence/test/research-collections.test.ts index 43d41d6..e70db1f 100644 --- a/packages/persistence/test/research-collections.test.ts +++ b/packages/persistence/test/research-collections.test.ts @@ -15,7 +15,7 @@ const mockRun: ArchiveRunView = { model: { id: null, canonicalSlug: "test", marketingName: "Test" }, test: { testCaseId: "00000000-0000-4000-8000-000000000403", - familySlug: "smoke", variantSlug: "text", version: "1.0.0", + familySlug: "smoke", familyName: "Smoke", variantSlug: "text", variantName: "Text", version: "1.0.0", caseSlug: "exact", }, requestedModel: "test",