diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index dc1e034..b03b8a4 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -29,7 +29,10 @@ import type { PgCatalogPresence, PgCatalogPresenceReview, PgCatalogRemoteIdCase, + PgProviderExpansion, PgProviderTestability, + RecordProviderExpansionPolicyInput, + RecordProviderCapabilityInput, RecordProviderTestabilityObservationInput, } from "@modelapse/catalog-admin"; @@ -81,6 +84,15 @@ type ProviderTestabilityRepository = Pick< PgProviderTestability, "listProviders" | "getProvider" | "recordObservation" >; +type ProviderExpansionRepository = Pick< + PgProviderExpansion, + | "ping" + | "listProviders" + | "getProvider" + | "listPolicies" + | "recordPolicy" + | "recordCapability" +>; type CostLedgerRepository = Pick< PgCostLedger, "ping" | "listDailyCosts" | "listBudgetStatus" | "recordBudgetPolicy" @@ -123,6 +135,7 @@ export interface AppDependencies { readonly catalogPresenceReview?: CatalogPresenceReviewRepository; readonly catalogRemoteIdCase?: CatalogRemoteIdCaseRepository; readonly providerTestability?: ProviderTestabilityRepository; + readonly providerExpansion?: ProviderExpansionRepository; readonly costLedger?: CostLedgerRepository; readonly calibration?: CalibrationRepository; readonly comparability?: ComparabilityRepository; @@ -225,6 +238,7 @@ export function createApp(deps: AppDependencies) { if (deps.costLedger) await deps.costLedger.ping(); if (deps.calibration) await deps.calibration.ping(); if (deps.comparability) await deps.comparability.ping(); + if (deps.providerExpansion) await deps.providerExpansion.ping(); return c.json({ ready: true, service: "modelapse-api", @@ -627,6 +641,213 @@ export function createApp(deps: AppDependencies) { } }); + app.get("/v1/control/provider-expansion", async (c) => { + if (!deps.providerExpansion || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + const policyVersion = c.req.query("policyVersion"); + if ( + policyVersion && + !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion) + ) { + return c.json({ error: "invalid_policy_version" }, 400); + } + return c.json({ + providers: await deps.providerExpansion.listProviders(policyVersion), + }); + }); + + app.get("/v1/control/provider-expansion/policies", async (c) => { + if (!deps.providerExpansion || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + return c.json({ policies: await deps.providerExpansion.listPolicies() }); + }); + + app.get("/v1/control/provider-expansion/:providerId", async (c) => { + if (!deps.providerExpansion || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + + const providerId = c.req.param("providerId"); + const policyVersion = c.req.query("policyVersion"); + if (!UUID_RE.test(providerId)) { + return c.json({ error: "invalid_provider_id" }, 400); + } + if ( + policyVersion && + !/^[a-z0-9][a-z0-9._-]*$/.test(policyVersion) + ) { + return c.json({ error: "invalid_policy_version" }, 400); + } + + const provider = await deps.providerExpansion.getProvider( + providerId, + policyVersion, + ); + if (!provider) { + return c.json({ error: "provider_expansion_provider_not_found" }, 404); + } + return c.json({ provider }); + }); + + app.post("/v1/control/provider-expansion/capabilities", async (c) => { + if (!deps.providerExpansion || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + + let raw: unknown; + try { + raw = await c.req.json(); + } catch { + return c.json({ error: "invalid_json" }, 400); + } + const body = apiRecord(raw); + const providerId = body?.providerId; + const capability = body?.capability; + const supportState = body?.supportState; + const sourceId = body?.sourceId; + const declaredAt = body?.declaredAt; + const actor = body?.actor; + const note = body?.note; + const capabilityKeys = [ + "returned_model_metadata", + "model_version_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", + "catalog_model_list", + ]; + + if ( + typeof providerId !== "string" || + !UUID_RE.test(providerId) || + typeof capability !== "string" || + !capabilityKeys.includes(capability) || + (supportState !== "supported" && supportState !== "unsupported") || + (sourceId !== undefined && + (typeof sourceId !== "string" || !UUID_RE.test(sourceId))) || + (declaredAt !== undefined && + (typeof declaredAt !== "string" || + !Number.isFinite(Date.parse(declaredAt)))) || + typeof actor !== "string" || + !actor.trim() || + (note !== undefined && typeof note !== "string") + ) { + return c.json({ error: "invalid_provider_capability" }, 400); + } + + try { + const input = { + providerId, + capability: capability as RecordProviderCapabilityInput["capability"], + supportState, + ...(typeof sourceId === "string" ? { sourceId } : {}), + ...(typeof declaredAt === "string" ? { declaredAt } : {}), + actor, + ...(typeof note === "string" ? { note } : {}), + } satisfies RecordProviderCapabilityInput; + return c.json( + { capability: await deps.providerExpansion.recordCapability(input) }, + 201, + ); + } catch (error) { + return c.json( + { + error: "provider_capability_rejected", + message: + error instanceof Error + ? error.message + : "Provider capability declaration rejected", + }, + 409, + ); + } + }); + + app.post("/v1/control/provider-expansion/policies", async (c) => { + if (!deps.providerExpansion || !controlToken) { + return c.json({ error: "control_plane_disabled" }, 503); + } + const authIssue = controlAuthIssue(c.req.header("authorization"), controlToken); + if (authIssue) return c.json(controlAuthError(authIssue), 401); + + let raw: unknown; + try { + raw = await c.req.json(); + } catch { + return c.json({ error: "invalid_json" }, 400); + } + const body = apiRecord(raw); + const requiredCapabilities = apiStringArray(body?.requiredCapabilities); + const version = body?.version; + const actor = body?.actor; + const note = body?.note; + const booleanKeys = [ + "requireIdentityProvenance", + "requireCatalogCollection", + "requireProviderPolicy", + "requirePricingEvidence", + "requireRunnerContext", + "requireDirectRun", + "requireCalibration", + ] as const; + + if ( + typeof version !== "string" || + !/^[a-z0-9][a-z0-9._-]*$/.test(version) || + !requiredCapabilities || + requiredCapabilities.length === 0 || + booleanKeys.some((key) => typeof body?.[key] !== "boolean") || + typeof actor !== "string" || + !actor.trim() || + (note !== undefined && typeof note !== "string") + ) { + return c.json({ error: "invalid_provider_expansion_policy" }, 400); + } + + try { + const input = { + version, + requiredCapabilities: + requiredCapabilities as RecordProviderExpansionPolicyInput["requiredCapabilities"], + requireIdentityProvenance: body!.requireIdentityProvenance as boolean, + requireCatalogCollection: body!.requireCatalogCollection as boolean, + requireProviderPolicy: body!.requireProviderPolicy as boolean, + requirePricingEvidence: body!.requirePricingEvidence as boolean, + requireRunnerContext: body!.requireRunnerContext as boolean, + requireDirectRun: body!.requireDirectRun as boolean, + requireCalibration: body!.requireCalibration as boolean, + actor, + ...(typeof note === "string" ? { note } : {}), + } satisfies RecordProviderExpansionPolicyInput; + return c.json( + { policy: await deps.providerExpansion.recordPolicy(input) }, + 201, + ); + } catch (error) { + return c.json( + { + error: "provider_expansion_policy_rejected", + message: + error instanceof Error + ? error.message + : "Provider expansion policy rejected", + }, + 400, + ); + } + }); + app.get("/v1/control/provider-testability", async (c) => { if (!deps.providerTestability || !controlToken) { return c.json({ error: "control_plane_disabled" }, 503); diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 6502b18..e589418 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -8,6 +8,7 @@ import { PgCatalogPresence, PgCatalogPresenceReview, PgCatalogRemoteIdCase, + PgProviderExpansion, PgProviderTestability, } from "@modelapse/catalog-admin"; import { @@ -46,6 +47,7 @@ const catalogPresence = PgCatalogPresence.connect(databaseUrl); const catalogPresenceReview = PgCatalogPresenceReview.connect(databaseUrl); const catalogRemoteIdCase = PgCatalogRemoteIdCase.connect(databaseUrl); const providerTestability = PgProviderTestability.connect(databaseUrl); +const providerExpansion = PgProviderExpansion.connect(databaseUrl); const controlToken = process.env.MODELAPSE_CONTROL_TOKEN; const port = Number(process.env.PORT ?? "3000"); const app = createApp({ @@ -66,6 +68,7 @@ const app = createApp({ catalogPresenceReview, catalogRemoteIdCase, providerTestability, + providerExpansion, ...(controlToken ? { controlToken } : {}), }); @@ -100,6 +103,7 @@ function shutdown(signal: string): void { catalogPresenceReview.close(), catalogRemoteIdCase.close(), providerTestability.close(), + providerExpansion.close(), ]).finally(() => { if (error) { console.error(error); diff --git a/apps/api/test/control-plane.test.ts b/apps/api/test/control-plane.test.ts index e38cd87..e909d8c 100644 --- a/apps/api/test/control-plane.test.ts +++ b/apps/api/test/control-plane.test.ts @@ -1232,3 +1232,211 @@ describe("Calibration and comparability policy API", () => { }); }); }); + + +describe("Provider Expansion control API", () => { + it("requires operator auth and exposes readiness plus append-only capability declarations", async () => { + const providerId = "00000000-0000-4000-8000-000000000401"; + const policy = { + id: "00000000-0000-4000-8000-000000000402", + version: "provider-expansion-v1", + requiredCapabilities: [ + "returned_model_metadata", + "model_version_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", + ] as const, + requireIdentityProvenance: true, + requireCatalogCollection: true, + requireProviderPolicy: true, + requirePricingEvidence: true, + requireRunnerContext: true, + requireDirectRun: true, + requireCalibration: true, + actor: "migration", + note: null, + createdAt: "2026-10-06T00:00:00.000Z", + }; + const summary = { + provider: { + id: providerId, + slug: "fixture-provider", + name: "Fixture Provider", + }, + policyVersion: policy.version, + status: "incomplete" as const, + blockerCount: 1, + caveatCount: 0, + activeModelCount: 1, + latestDirectRunAt: null, + latestCalibrationAt: null, + }; + const detail = { + ...summary, + generatedAt: "2026-10-06T01:00:00.000Z", + policy, + blockers: ["calibration_evidence_missing"], + caveats: [], + gates: [ + { + key: "calibration_coverage" as const, + status: "fail" as const, + summary: "no calibration assessment", + blockers: ["calibration_evidence_missing"], + caveats: [], + evidenceIds: [], + }, + ], + capabilities: [ + { + capability: "returned_model_metadata" as const, + supportState: "supported" as const, + eventId: "00000000-0000-4000-8000-000000000403", + sourceId: null, + declaredAt: "2026-10-06T00:30:00.000Z", + actor: "fixture", + note: null, + observed: true, + }, + ], + models: [], + fleet: [], + latestCalibration: null, + }; + let capabilityInput: unknown; + let policyInput: unknown; + + const app = createApp({ + runs: baseRuns(), + controlToken: "control-secret", + providerExpansion: { + ping: async () => undefined, + listProviders: async () => [summary], + getProvider: async () => detail, + listPolicies: async () => [policy], + recordCapability: async (input) => { + capabilityInput = input; + return { + capability: input.capability, + supportState: input.supportState, + eventId: "00000000-0000-4000-8000-000000000404", + sourceId: input.sourceId ?? null, + declaredAt: input.declaredAt ?? "2026-10-06T02:00:00.000Z", + actor: input.actor, + note: input.note ?? null, + observed: false, + }; + }, + recordPolicy: async (input) => { + policyInput = input; + return { + ...policy, + version: input.version, + requiredCapabilities: input.requiredCapabilities, + requireIdentityProvenance: input.requireIdentityProvenance, + requireCatalogCollection: input.requireCatalogCollection, + requireProviderPolicy: input.requireProviderPolicy, + requirePricingEvidence: input.requirePricingEvidence, + requireRunnerContext: input.requireRunnerContext, + requireDirectRun: input.requireDirectRun, + requireCalibration: input.requireCalibration, + actor: input.actor, + note: input.note ?? null, + }; + }, + }, + }); + + const unauthorized = await app.request("/v1/control/provider-expansion"); + expect(unauthorized.status).toBe(401); + + const headers = { authorization: "Bearer control-secret" }; + const listed = await app.request("/v1/control/provider-expansion", { + headers, + }); + expect(listed.status).toBe(200); + await expect(listed.json()).resolves.toMatchObject({ + providers: [ + { + provider: { id: providerId, slug: "fixture-provider" }, + status: "incomplete", + blockerCount: 1, + }, + ], + }); + + const read = await app.request( + `/v1/control/provider-expansion/${providerId}`, + { headers }, + ); + expect(read.status).toBe(200); + await expect(read.json()).resolves.toMatchObject({ + provider: { + policyVersion: "provider-expansion-v1", + blockers: ["calibration_evidence_missing"], + }, + }); + + const capability = await app.request( + "/v1/control/provider-expansion/capabilities", + { + method: "POST", + headers: { + ...headers, + "content-type": "application/json", + }, + body: JSON.stringify({ + providerId, + capability: "service_tier_metadata", + supportState: "unsupported", + actor: "api-test", + note: "Provider does not expose service tier metadata.", + }), + }, + ); + expect(capability.status).toBe(201); + expect(capabilityInput).toMatchObject({ + providerId, + capability: "service_tier_metadata", + supportState: "unsupported", + actor: "api-test", + }); + + const createdPolicy = await app.request( + "/v1/control/provider-expansion/policies", + { + method: "POST", + headers: { + ...headers, + "content-type": "application/json", + }, + body: JSON.stringify({ + version: "provider-expansion-v2", + requiredCapabilities: [ + "returned_model_metadata", + "provider_request_id", + ], + requireIdentityProvenance: true, + requireCatalogCollection: true, + requireProviderPolicy: true, + requirePricingEvidence: true, + requireRunnerContext: true, + requireDirectRun: true, + requireCalibration: true, + actor: "api-test", + }), + }, + ); + expect(createdPolicy.status).toBe(201); + expect(policyInput).toMatchObject({ + version: "provider-expansion-v2", + requiredCapabilities: [ + "returned_model_metadata", + "provider_request_id", + ], + actor: "api-test", + }); + }); +}); diff --git a/apps/web/src/modelapse.ts b/apps/web/src/modelapse.ts index 4f398f6..d6fe48f 100644 --- a/apps/web/src/modelapse.ts +++ b/apps/web/src/modelapse.ts @@ -1254,6 +1254,103 @@ export interface ProviderTestabilityProvider { readonly history: readonly ProviderTestabilityObservation[]; } +export type ProviderExpansionCapabilityKey = + | "returned_model_metadata" + | "model_version_metadata" + | "provider_request_id" + | "provider_response_id" + | "service_tier_metadata" + | "token_usage" + | "catalog_model_list"; + +export interface ProviderExpansionProviderSummary { + readonly provider: { + readonly id: string; + readonly slug: string; + readonly name: string; + }; + readonly policyVersion: string; + readonly status: "ready" | "limited" | "incomplete"; + readonly blockerCount: number; + readonly caveatCount: number; + readonly activeModelCount: number; + readonly latestDirectRunAt: string | null; + readonly latestCalibrationAt: string | null; +} + +export interface ProviderExpansionProvider + extends ProviderExpansionProviderSummary { + readonly generatedAt: string; + readonly policy: { + readonly id: string; + readonly version: string; + readonly requiredCapabilities: readonly ProviderExpansionCapabilityKey[]; + readonly requireIdentityProvenance: boolean; + readonly requireCatalogCollection: boolean; + readonly requireProviderPolicy: boolean; + readonly requirePricingEvidence: boolean; + readonly requireRunnerContext: boolean; + readonly requireDirectRun: boolean; + readonly requireCalibration: boolean; + readonly actor: string; + readonly note: string | null; + readonly createdAt: string; + }; + readonly blockers: readonly string[]; + readonly caveats: readonly string[]; + readonly gates: readonly { + readonly key: + | "identity_provenance" + | "catalog_collection" + | "access_region_evidence" + | "pricing_evidence" + | "runner_account_context" + | "direct_execution_evidence" + | "calibration_coverage" + | "capability_contract"; + readonly status: "pass" | "limited" | "fail"; + readonly summary: string; + readonly blockers: readonly string[]; + readonly caveats: readonly string[]; + readonly evidenceIds: readonly string[]; + }[]; + readonly capabilities: readonly { + readonly capability: ProviderExpansionCapabilityKey; + readonly supportState: "supported" | "unsupported" | null; + readonly eventId: string | null; + readonly sourceId: string | null; + readonly declaredAt: string | null; + readonly actor: string | null; + readonly note: string | null; + readonly observed: boolean; + }[]; + readonly models: readonly { + readonly id: string; + readonly canonicalSlug: string; + readonly marketingName: string; + readonly status: string; + readonly canonicalSourceId: string | null; + readonly sourcedDirectBindingId: string | null; + readonly endpointSourceId: string | null; + readonly bindingSourceId: string | null; + }[]; + readonly fleet: readonly { + readonly environmentId: string; + readonly slug: string; + readonly region: string; + readonly accountTier: string | null; + readonly serviceTier: string | null; + readonly serviceAssurance: string; + readonly capabilityEventId: string; + }[]; + readonly latestCalibration: { + readonly runId: string; + readonly status: "pass" | "anomaly" | "unknown"; + readonly repeatedAnomaly: boolean; + readonly completedAt: string; + } | null; +} + export interface ControlJob { readonly id: string; readonly kind: string; @@ -1344,6 +1441,17 @@ interface ProviderTestabilityProviderInput extends OperatorInput { readonly providerId: string; } +interface ProviderExpansionProviderInput extends OperatorInput { + readonly providerId: string; +} + +interface RecordProviderCapabilityWebInput + extends ProviderExpansionProviderInput { + readonly capability: ProviderExpansionCapabilityKey; + readonly supportState: "supported" | "unsupported"; + readonly note?: string; +} + interface RecordProviderTestabilityObservationWebInput extends OperatorInput { readonly providerId: string; readonly modelId?: string; @@ -1776,6 +1884,59 @@ function parseCatalogRemoteIdCaseInput(value: unknown): CatalogRemoteIdCaseInput return { operatorToken, providerId, remoteModelId: remoteModelId.trim() }; } +function parseProviderExpansionProviderInput( + value: unknown, +): ProviderExpansionProviderInput { + const operatorToken = parseOperatorToken(value); + if (!isRecord(value)) { + throw new Error("Provider expansion request must be an object"); + } + const providerId = value.providerId; + if (typeof providerId !== "string" || !UUID_RE.test(providerId)) { + throw new Error("providerId must be a UUID"); + } + return { operatorToken, providerId }; +} + +function parseProviderCapabilityInput( + value: unknown, +): RecordProviderCapabilityWebInput { + const base = parseProviderExpansionProviderInput(value); + if (!isRecord(value)) { + throw new Error("Provider capability request must be an object"); + } + const capability = value.capability; + const supportState = value.supportState; + const note = value.note; + const capabilities = new Set([ + "returned_model_metadata", + "model_version_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", + "catalog_model_list", + ]); + if ( + typeof capability !== "string" || + !capabilities.has(capability as ProviderExpansionCapabilityKey) + ) { + throw new Error("capability is invalid"); + } + if (supportState !== "supported" && supportState !== "unsupported") { + throw new Error("supportState is invalid"); + } + if (note !== undefined && typeof note !== "string") { + throw new Error("note must be a string"); + } + return { + ...base, + capability: capability as ProviderExpansionCapabilityKey, + supportState, + ...(typeof note === "string" && note.trim() ? { note: note.trim() } : {}), + }; +} + function parseProviderTestabilityProviderInput( value: unknown, ): ProviderTestabilityProviderInput { @@ -2213,6 +2374,51 @@ export const compareArchive = createServerFn({ method: "POST" }) } }); +export const getProviderExpansionProviders = createServerFn({ method: "POST" }) + .validator(parseOperatorInput) + .handler(async ({ data }): Promise => { + requireOperator(data.operatorToken); + const result = await requestJson<{ + providers: readonly ProviderExpansionProviderSummary[]; + }>("/v1/control/provider-expansion", { control: true }); + return result.providers; + }); + +export const getProviderExpansionProvider = createServerFn({ method: "POST" }) + .validator(parseProviderExpansionProviderInput) + .handler(async ({ data }): Promise => { + requireOperator(data.operatorToken); + try { + const result = await requestJson<{ provider: ProviderExpansionProvider }>( + `/v1/control/provider-expansion/${data.providerId}`, + { control: true }, + ); + return result.provider; + } catch (error) { + if (error instanceof ApiRequestError && error.status === 404) return null; + throw error; + } + }); + +export const recordProviderExpansionCapability = createServerFn({ method: "POST" }) + .validator(parseProviderCapabilityInput) + .handler(async ({ data }) => { + requireOperator(data.operatorToken); + return requestJson<{ + capability: ProviderExpansionProvider["capabilities"][number]; + }>("/v1/control/provider-expansion/capabilities", { + method: "POST", + control: true, + body: { + providerId: data.providerId, + capability: data.capability, + supportState: data.supportState, + actor: "web-operator", + ...(data.note ? { note: data.note } : {}), + }, + }); + }); + export const getProviderTestabilityProviders = createServerFn({ method: "POST" }) .validator(parseOperatorInput) .handler(async ({ data }): Promise => { diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index bfe15cb..0fbfb3d 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -168,6 +168,7 @@ function ModelapseHome() { Run control Catalog Inbox Provider Testability + Provider Expansion Catalog Integrity
diff --git a/apps/web/src/routes/provider-expansion.tsx b/apps/web/src/routes/provider-expansion.tsx new file mode 100644 index 0000000..73fd8fc --- /dev/null +++ b/apps/web/src/routes/provider-expansion.tsx @@ -0,0 +1,411 @@ +import { createFileRoute } from "@tanstack/react-router"; +import { useState } from "react"; +import { + getProviderExpansionProvider, + getProviderExpansionProviders, + recordProviderExpansionCapability, + type ProviderExpansionCapabilityKey, + type ProviderExpansionProvider, + type ProviderExpansionProviderSummary, +} from "../modelapse"; + +export const Route = createFileRoute("/provider-expansion")({ + component: ProviderExpansionPage, +}); + +function stamp(value: string | null): string { + if (!value) return "—"; + return new Date(value).toISOString().replace("T", " ").replace(".000Z", "Z"); +} + +function statusClass( + value: "ready" | "limited" | "incomplete" | "pass" | "fail", +): string { + if (value === "ready" || value === "pass") return "badge badge-pass"; + if (value === "incomplete" || value === "fail") return "badge badge-fail"; + return "badge"; +} + +function ProviderExpansionPage() { + const [operatorToken, setOperatorToken] = useState(""); + const [providers, setProviders] = useState< + readonly ProviderExpansionProviderSummary[] + >([]); + const [selectedProviderId, setSelectedProviderId] = useState(""); + const [detail, setDetail] = useState(null); + const [capability, setCapability] = + useState("returned_model_metadata"); + const [supportState, setSupportState] = + useState<"supported" | "unsupported">("supported"); + const [note, setNote] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [message, setMessage] = useState(null); + + async function loadProvider(providerId: string): Promise { + const provider = await getProviderExpansionProvider({ + data: { operatorToken, providerId }, + }); + setSelectedProviderId(providerId); + setDetail(provider); + } + + async function loadProviders(): Promise { + if (!operatorToken) return; + setBusy(true); + setError(null); + setMessage(null); + try { + const result = await getProviderExpansionProviders({ + data: { operatorToken }, + }); + setProviders(result); + const next = + selectedProviderId && + result.some((item) => item.provider.id === selectedProviderId) + ? selectedProviderId + : result[0]?.provider.id ?? ""; + if (next) await loadProvider(next); + else setDetail(null); + } catch (caught) { + setProviders([]); + setDetail(null); + setError(caught instanceof Error ? caught.message : "Provider Expansion load failed"); + } finally { + setBusy(false); + } + } + + async function appendCapability(): Promise { + if (!detail || !operatorToken) return; + setBusy(true); + setError(null); + setMessage(null); + try { + await recordProviderExpansionCapability({ + data: { + operatorToken, + providerId: detail.provider.id, + capability, + supportState, + ...(note.trim() ? { note: note.trim() } : {}), + }, + }); + setNote(""); + await loadProvider(detail.provider.id); + const refreshed = await getProviderExpansionProviders({ + data: { operatorToken }, + }); + setProviders(refreshed); + setMessage("Capability declaration appended."); + } catch (caught) { + setError(caught instanceof Error ? caught.message : "Capability declaration failed"); + } finally { + setBusy(false); + } + } + + return ( +
+
+ + M + + Modelapse + AI Model Test & Evolution Archive + + + +
+ +
+
+ Operator + / + Provider Expansion +
+
+
+

ARCHIVE v0.24 · PROVIDER EXPANSION PLAYBOOK

+

HTTP success is not onboarding completion.

+

+ Readiness is derived from identity provenance, catalog collection, + access and pricing evidence, runner context, verified execution, + calibration, and explicit adapter capability declarations. + No percentage readiness score is computed. +

+
+
+
+ +
+
+
+

OPERATOR ACCESS

+

Load provider onboarding state

+
+ control-plane only +
+
+ +
+ + + Credentials for Providers remain in controlled workers and are never + returned by this view. + +
+
+ {error ?
{error}
: null} + {message ?
{message}
: null} +
+ + {providers.length ? ( +
+
+
+

PROVIDER MATRIX

+

Expansion state

+
+ categorical · no score +
+
+ {providers.map((item) => ( + + ))} +
+
+ ) : null} + + {detail ? ( + <> +
+
+
+

PLAYBOOK RESULT

+

{detail.provider.name}

+
+ {detail.status} +
+ +
+ {detail.policy.version} + + projected {stamp(detail.generatedAt)} · latest direct{" "} + {stamp(detail.latestDirectRunAt)} · latest calibration{" "} + {stamp(detail.latestCalibrationAt)} + +
+ + {detail.blockers.length ? ( +
+ Onboarding blockers + {detail.blockers.join(" · ")} +
+ ) : null} + {detail.caveats.length ? ( +
+ Known limitations + {detail.caveats.join(" · ")} +
+ ) : null} + +
+ {detail.gates.map((item) => ( +
+
+ {item.key.replaceAll("_", " ")} + {item.status} +
+

{item.summary}

+ {item.blockers.length ? ( +

Blockers: {item.blockers.join(" · ")}

+ ) : null} + {item.caveats.length ? ( +

Caveats: {item.caveats.join(" · ")}

+ ) : null} + {item.evidenceIds.length} evidence reference(s) +
+ ))} +
+
+ +
+
+
+

ADAPTER CONTRACT

+

Supported and unsupported capture

+
+ append-only declarations +
+
+ {detail.capabilities.map((item) => ( +
+
+ {item.capability.replaceAll("_", " ")} +
+ + {item.supportState ?? "undeclared"} + + + {item.observed ? "observed" : "not observed"} + +
+
+

+ event {item.eventId?.slice(0, 8) ?? "—"} · source{" "} + {item.sourceId?.slice(0, 8) ?? "—"} · declared{" "} + {stamp(item.declaredAt)} +

+ {item.note ?

{item.note}

: null} +
+ ))} +
+
+ +
+
+
+

APPEND CAPABILITY

+

Declare a Provider capability

+
+ never overwrite history +
+
+ + + +
+ + + A later declaration changes only the current projection. Earlier + declarations remain auditable. + +
+
+
+ +
+
+
+

CONTROLLED ENVIRONMENTS

+

Fleet context

+
+
+
+ {detail.fleet.map((environment) => ( +
+ {environment.slug} +

+ {environment.region} · account {environment.accountTier ?? "unknown"} + {" · "}service {environment.serviceTier ?? "default/unspecified"} + {" · "}{environment.serviceAssurance.replaceAll("_", " ")} +

+
+ ))} + {!detail.fleet.length ? ( +
No enabled fleet capability for this Provider.
+ ) : null} +
+
+ + ) : null} + + +
+ ); +} diff --git a/docker/api.Dockerfile b/docker/api.Dockerfile index beb87e7..519ffe6 100644 --- a/docker/api.Dockerfile +++ b/docker/api.Dockerfile @@ -46,6 +46,10 @@ COPY --from=build /app/packages/domain/package.json ./packages/domain/package.js COPY --from=build /app/packages/domain/dist ./packages/domain/dist COPY --from=build /app/packages/provider-adapter/package.json ./packages/provider-adapter/package.json COPY --from=build /app/packages/provider-adapter/dist ./packages/provider-adapter/dist +COPY --from=build /app/packages/provider-openai/package.json ./packages/provider-openai/package.json +COPY --from=build /app/packages/provider-openai/dist ./packages/provider-openai/dist +COPY --from=build /app/packages/provider-deepseek/package.json ./packages/provider-deepseek/package.json +COPY --from=build /app/packages/provider-deepseek/dist ./packages/provider-deepseek/dist COPY --from=build /app/packages/attestation/package.json ./packages/attestation/package.json COPY --from=build /app/packages/attestation/dist ./packages/attestation/dist COPY --from=build /app/packages/blob-store/package.json ./packages/blob-store/package.json diff --git a/docs/adr/0020-provider-onboarding-evidence-gated.md b/docs/adr/0020-provider-onboarding-evidence-gated.md new file mode 100644 index 0000000..7bbe786 --- /dev/null +++ b/docs/adr/0020-provider-onboarding-evidence-gated.md @@ -0,0 +1,28 @@ +# ADR 0020 — Provider onboarding is evidence-gated + +## Status + +Accepted for Archive v0.24. + +## Context + +A working HTTP request proves only that one request succeeded. It does not establish canonical identity, first-party provenance, access conditions, pricing, runner account context, response metadata coverage, service-health calibration or the limits of the Provider Adapter. + +Treating transport success as onboarding completion would make later benchmark comparisons appear stronger than their evidence. + +## Decision + +1. Provider onboarding is evaluated under a named append-only Provider Expansion policy. +2. Readiness is derived from existing identity, catalog, testability, pricing, Fleet, Run and calibration evidence. +3. Readiness uses categorical `ready`, `limited`, and `incomplete` states; no percentage score is introduced. +4. Runtime Provider Adapter metadata capabilities are exhaustive and must be declared `supported` or `unsupported`. +5. A capability declared `supported` must be observed in verified first-party execution before its gate passes. +6. An explicit `unsupported` declaration satisfies documentation completeness but remains a visible limitation. +7. Provider capability declarations are append-only events; later declarations change only the current projection. +8. Production bootstrap may seed adapter manifests only when no current declaration exists and must never overwrite operator evidence. +9. Provider credentials and personal account identifiers are outside the readiness data model. +10. Readiness never mutates historical Runs, Evaluations, qualification envelopes or calibration assessments. + +## Consequences + +New Provider work now has an auditable definition of done. Provider-specific limitations remain visible instead of being normalized away, and v0.25+ public comparison surfaces can distinguish a Provider that is genuinely evidence-ready from one that merely has a functioning HTTP adapter. diff --git a/docs/archive-roadmap.md b/docs/archive-roadmap.md index d541ffe..c911da0 100644 --- a/docs/archive-roadmap.md +++ b/docs/archive-roadmap.md @@ -278,6 +278,8 @@ The grouping policy must be versioned and inspectable. ### Archive v0.24 — Provider Expansion Playbook +**Implemented:** see `docs/provider-expansion-playbook.md` and ADR 0020. + New Provider Adapters should not be considered complete merely because the HTTP request works. Provider onboarding should require: diff --git a/docs/provider-expansion-playbook.md b/docs/provider-expansion-playbook.md new file mode 100644 index 0000000..b060b2d --- /dev/null +++ b/docs/provider-expansion-playbook.md @@ -0,0 +1,184 @@ +# Provider Expansion Playbook + +Archive v0.24 turns Provider onboarding into an evidence-gated process. + +A new Provider Adapter is **not complete because one HTTP request succeeds**. Modelapse evaluates onboarding under a named, append-only Provider Expansion policy and exposes the missing evidence directly to operators. + +## Status model + +Provider readiness is categorical, not a percentage score: + +- `incomplete` — one or more required evidence gates are missing; +- `limited` — required gates are complete, but a documented limitation remains; +- `ready` — all required gates are complete with no current limitation under the selected policy. + +A Provider can therefore be fully documented yet still be `limited`, for example when model-version metadata is explicitly unsupported. + +## provider-expansion-v1 + +The initial policy requires: + +1. first-party identity provenance; +2. catalog collection evidence; +3. access / registration / region policy evidence; +4. sourced pricing evidence; +5. controlled runner account + fleet context; +6. at least one sealed E4/E5 first-party direct Run; +7. calibration coverage; +8. an explicit runtime capability contract. + +The policy is stored in `provider_expansion_policies`. New rules require a new policy version; existing policy rows are append-only. + +## Evidence gates + +### Identity provenance + +Every active/preview canonical Model must have: + +- `canonical_source_id`; +- a current first-party direct execution binding; +- a source-backed Provider endpoint; +- a source-backed execution binding. + +One sourced endpoint is not enough if another active Model lacks a sourced binding. + +### Catalog collection + +At least one enabled catalog observer source must have successful or partial snapshot evidence. + +Model-list evidence is expected. If the Provider genuinely has no usable model-list surface, operators may explicitly declare `catalog_model_list=unsupported`; this is surfaced as a limitation rather than silently treated as coverage. + +### Access and region evidence + +Current `provider_policy` observations must cover the Provider or every active Model. + +The gate requires: + +- known access state; +- known region policy; +- source-backed observations. + +Restricted or unavailable access remains evidence, but produces a limitation rather than being rewritten as “normal”. + +### Pricing evidence + +Pricing is complete only when current Provider-wide pricing exists or every active Model is covered by a current sourced pricing observation. + +Zero is allowed when the sourced price is actually zero. Missing pricing is not interpreted as free. + +### Runner account context + +At least one current runner-access observation must identify: + +- available access; +- account tier; +- documented service assurance. + +At least one enabled Fleet environment for the Provider must also expose: + +- region; +- account tier; +- documented service assurance; +- enabled first-party-direct capability. + +Secrets and personal account identifiers are never part of the readiness payload. + +### Verified direct execution + +At least one sealed completed `first_party_direct` Run must carry E4 or E5 evidence. + +This proves that the controlled execution path works, but it does not by itself prove representative service quality. + +### Calibration + +At least one calibration assessment must exist. + +A pass satisfies coverage. An anomaly also satisfies “coverage exists” but remains a caveat. A repeated anomaly is surfaced more strongly; it is still not automatically described as intentional degradation. + +### Capability contract + +Every required runtime capability must be declared explicitly: + +- `returned_model_metadata`; +- `model_version_metadata`; +- `provider_request_id`; +- `provider_response_id`; +- `service_tier_metadata`; +- `token_usage`. + +Declarations are append-only `provider_capability_events`. + +States are: + +- `supported`; +- `unsupported`. + +There is no implicit “probably supported”. Missing declarations are blockers. + +For capabilities declared `supported`, readiness also requires at least one verified direct Run that actually demonstrates capture. A supported-but-never-observed claim remains incomplete. + +An explicit `unsupported` declaration satisfies completeness but makes the Provider `limited`. + +## Adapter descriptor contract + +`ProviderAdapterDescriptor.capabilities` is exhaustive for runtime metadata capture. + +Current examples: + +- OpenAI declares service-tier metadata supported; +- DeepSeek declares service-tier metadata unsupported; +- Anthropic declares service-tier metadata unsupported; +- all three currently declare model-version metadata unsupported. + +OpenAI and DeepSeek production bootstrap persist these manifests into the capability ledger only when no declaration exists yet. A later operator declaration remains authoritative in the current projection; bootstrap never overwrites it. + +## Operator API + +Authenticated control-plane endpoints: + +```text +GET /v1/control/provider-expansion +GET /v1/control/provider-expansion/:providerId +GET /v1/control/provider-expansion/policies +POST /v1/control/provider-expansion/capabilities +POST /v1/control/provider-expansion/policies +``` + +The Web operator view is available at: + +```text +/provider-expansion +``` + +It exposes gates, blockers, limitations, evidence-reference counts, capability declarations and controlled Fleet context. + +## Provider onboarding sequence + +A practical onboarding order is: + +```text +adapter descriptor + explicit capabilities +→ Provider / endpoint source +→ canonical Model + execution binding source +→ catalog observer source + snapshot +→ access / registration / region policy +→ pricing +→ runner-access observation +→ controlled Fleet environment +→ E4 first-party Run +→ calibration Run +→ Provider Expansion review +``` + +This order is operational guidance, not a historical rewrite rule. + +## Invariants + +1. Readiness is a derived view, never a mutation of Run/Evaluation history. +2. Provider credentials never appear in the readiness model. +3. Unsupported capabilities are explicit facts, not missing fields. +4. “Supported” requires observed capture before the capability gate passes. +5. A restricted Provider may be documented completely while remaining limited. +6. One successful request does not make a Provider ready. +7. Policy evolution adds new versions instead of rewriting old policy rows. +8. Readiness status is not a model-quality score. diff --git a/packages/catalog-admin/package.json b/packages/catalog-admin/package.json index 5cfba38..e112849 100644 --- a/packages/catalog-admin/package.json +++ b/packages/catalog-admin/package.json @@ -11,7 +11,7 @@ "build": "tsc -p tsconfig.json", "typecheck": "tsc -p tsconfig.json --noEmit", "test": "vitest run test/openai-smoke.test.ts test/service-calibration.test.ts test/catalog-observer.test.ts", - "test:integration": "vitest run test/bootstrap.integration.test.ts", + "test:integration": "vitest run test/bootstrap.integration.test.ts test/provider-expansion.integration.test.ts", "bootstrap:openai-smoke": "node dist/src/cli.js bootstrap-openai-smoke", "bootstrap:deepseek-smoke": "node dist/src/cli.js bootstrap-deepseek-smoke", "bootstrap:deepseek-flash-model": "node dist/src/cli.js bootstrap-deepseek-flash-model", @@ -24,7 +24,10 @@ "dependencies": { "@modelapse/blob-store": "*", "@modelapse/testpack-sdk": "*", - "pg": "8.23.0" + "pg": "8.23.0", + "@modelapse/provider-openai": "*", + "@modelapse/provider-deepseek": "*", + "@modelapse/provider-adapter": "*" }, "devDependencies": { "@modelapse/persistence": "*", diff --git a/packages/catalog-admin/src/catalog.ts b/packages/catalog-admin/src/catalog.ts index 3751bb3..8abf0aa 100644 --- a/packages/catalog-admin/src/catalog.ts +++ b/packages/catalog-admin/src/catalog.ts @@ -2,6 +2,9 @@ import type { BlobDescriptor, BlobStore, } from "@modelapse/blob-store"; +import type { ProviderAdapterDescriptor } from "@modelapse/provider-adapter"; +import { DeepSeekResponsesAdapter } from "@modelapse/provider-deepseek"; +import { OpenAIResponsesAdapter } from "@modelapse/provider-openai"; import { Pool, type PoolClient } from "pg"; import { OPENAI_SMOKE_CASE_SLUG, @@ -90,6 +93,51 @@ async function registerBlob( } } +async function ensureProviderCapabilityManifest( + client: PoolClient, + input: { + readonly providerId: string; + readonly descriptor: ProviderAdapterDescriptor; + readonly sourceId: string; + }, +): Promise { + for (const [capability, supportState] of Object.entries( + input.descriptor.capabilities, + )) { + await client.query( + `INSERT INTO modelapse.provider_capability_events + ( + provider_id, + capability, + support_state, + source_id, + actor, + note + ) + SELECT + $1, + $2, + $3, + $4, + 'catalog-bootstrap', + $5 + WHERE NOT EXISTS ( + SELECT 1 + FROM modelapse.provider_capability_current current + WHERE current.provider_id = $1 + AND current.capability = $2 + )`, + [ + input.providerId, + capability, + supportState, + input.sourceId, + "Seeded from adapter " + input.descriptor.id, + ], + ); + } +} + async function ensureSource( client: PoolClient, input: { @@ -865,6 +913,16 @@ export class PgCatalogAdmin { }); const providerId = await ensureProvider(client); + const adapterSourceId = await ensureSource(client, { + sourceType: "modelapse_definition", + url: "https://github.com/imwarn/modelapse/blob/main/packages/provider-openai/src/index.ts", + title: "Modelapse OpenAI provider adapter", + }); + await ensureProviderCapabilityManifest(client, { + providerId, + descriptor: new OpenAIResponsesAdapter().descriptor, + sourceId: adapterSourceId, + }); const endpointId = await ensureEndpoint( client, providerId, @@ -943,6 +1001,16 @@ export class PgCatalogAdmin { name: "DeepSeek", homepage: "https://www.deepseek.com/", }); + const adapterSourceId = await ensureSource(client, { + sourceType: "modelapse_definition", + url: "https://github.com/imwarn/modelapse/blob/main/packages/provider-deepseek/src/index.ts", + title: "Modelapse DeepSeek provider adapter", + }); + await ensureProviderCapabilityManifest(client, { + providerId, + descriptor: new DeepSeekResponsesAdapter().descriptor, + sourceId: adapterSourceId, + }); const endpointId = await ensureNamedDirectEndpoint(client, { providerId, baseUrl: "https://api.deepseek.com", diff --git a/packages/catalog-admin/src/index.ts b/packages/catalog-admin/src/index.ts index c73c470..283ed9d 100644 --- a/packages/catalog-admin/src/index.ts +++ b/packages/catalog-admin/src/index.ts @@ -21,3 +21,4 @@ export * from "./catalog-remote-id-case.js"; export * from "./provider-testability.js"; export * from "./service-calibration.js"; export * from "./service-calibration-catalog.js"; +export * from "./provider-expansion.js"; diff --git a/packages/catalog-admin/src/provider-expansion.ts b/packages/catalog-admin/src/provider-expansion.ts new file mode 100644 index 0000000..3cdc159 --- /dev/null +++ b/packages/catalog-admin/src/provider-expansion.ts @@ -0,0 +1,1149 @@ +import { Pool } from "pg"; + +export const providerExpansionCapabilityKeys = [ + "returned_model_metadata", + "model_version_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", + "catalog_model_list", +] as const; + +export type ProviderExpansionCapabilityKey = + (typeof providerExpansionCapabilityKeys)[number]; + +export type ProviderExpansionCapabilitySupport = + | "supported" + | "unsupported"; + +export type ProviderExpansionStatus = + | "ready" + | "limited" + | "incomplete"; + +export type ProviderExpansionGateStatus = + | "pass" + | "limited" + | "fail"; + +export interface ProviderExpansionPolicy { + readonly id: string; + readonly version: string; + readonly requiredCapabilities: readonly ProviderExpansionCapabilityKey[]; + readonly requireIdentityProvenance: boolean; + readonly requireCatalogCollection: boolean; + readonly requireProviderPolicy: boolean; + readonly requirePricingEvidence: boolean; + readonly requireRunnerContext: boolean; + readonly requireDirectRun: boolean; + readonly requireCalibration: boolean; + readonly actor: string; + readonly note: string | null; + readonly createdAt: string; +} + +export interface ProviderExpansionCapability { + readonly capability: ProviderExpansionCapabilityKey; + readonly supportState: ProviderExpansionCapabilitySupport | null; + readonly eventId: string | null; + readonly sourceId: string | null; + readonly declaredAt: string | null; + readonly actor: string | null; + readonly note: string | null; + readonly observed: boolean; +} + +export interface ProviderExpansionGate { + readonly key: + | "identity_provenance" + | "catalog_collection" + | "access_region_evidence" + | "pricing_evidence" + | "runner_account_context" + | "direct_execution_evidence" + | "calibration_coverage" + | "capability_contract"; + readonly status: ProviderExpansionGateStatus; + readonly summary: string; + readonly blockers: readonly string[]; + readonly caveats: readonly string[]; + readonly evidenceIds: readonly string[]; +} + +export interface ProviderExpansionProviderSummary { + readonly provider: { + readonly id: string; + readonly slug: string; + readonly name: string; + }; + readonly policyVersion: string; + readonly status: ProviderExpansionStatus; + readonly blockerCount: number; + readonly caveatCount: number; + readonly activeModelCount: number; + readonly latestDirectRunAt: string | null; + readonly latestCalibrationAt: string | null; +} + +export interface ProviderExpansionProvider + extends ProviderExpansionProviderSummary { + readonly generatedAt: string; + readonly policy: ProviderExpansionPolicy; + readonly blockers: readonly string[]; + readonly caveats: readonly string[]; + readonly gates: readonly ProviderExpansionGate[]; + readonly capabilities: readonly ProviderExpansionCapability[]; + readonly models: readonly { + readonly id: string; + readonly canonicalSlug: string; + readonly marketingName: string; + readonly status: string; + readonly canonicalSourceId: string | null; + readonly sourcedDirectBindingId: string | null; + readonly endpointSourceId: string | null; + readonly bindingSourceId: string | null; + }[]; + readonly fleet: readonly { + readonly environmentId: string; + readonly slug: string; + readonly region: string; + readonly accountTier: string | null; + readonly serviceTier: string | null; + readonly serviceAssurance: string; + readonly capabilityEventId: string; + }[]; + readonly latestCalibration: { + readonly runId: string; + readonly status: "pass" | "anomaly" | "unknown"; + readonly repeatedAnomaly: boolean; + readonly completedAt: string; + } | null; +} + +export interface RecordProviderCapabilityInput { + readonly providerId: string; + readonly capability: ProviderExpansionCapabilityKey; + readonly supportState: ProviderExpansionCapabilitySupport; + readonly sourceId?: string; + readonly declaredAt?: string; + readonly actor: string; + readonly note?: string; +} + +export interface RecordProviderExpansionPolicyInput { + readonly version: string; + readonly requiredCapabilities: readonly ProviderExpansionCapabilityKey[]; + readonly requireIdentityProvenance: boolean; + readonly requireCatalogCollection: boolean; + readonly requireProviderPolicy: boolean; + readonly requirePricingEvidence: boolean; + readonly requireRunnerContext: boolean; + readonly requireDirectRun: boolean; + readonly requireCalibration: boolean; + readonly actor: string; + readonly note?: string; +} + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const VERSION_RE = /^[a-z0-9][a-z0-9._-]*$/; +const CAPABILITY_KEYS = new Set(providerExpansionCapabilityKeys); + +function normalizeTimestamp(value: string | undefined): string { + if (!value) return new Date().toISOString(); + const parsed = new Date(value); + if (Number.isNaN(parsed.valueOf())) { + throw new Error("declaredAt must be an ISO-8601 timestamp"); + } + return parsed.toISOString(); +} + +function requiredText(value: string, label: string): string { + const normalized = value.trim(); + if (!normalized) throw new Error(label + " is required"); + return normalized; +} + +function optionalText(value: string | undefined): string | null { + const normalized = value?.trim(); + return normalized ? normalized : null; +} + +function policyView(row: { + id: string; + version: string; + required_capabilities: ProviderExpansionCapabilityKey[]; + require_identity_provenance: boolean; + require_catalog_collection: boolean; + require_provider_policy: boolean; + require_pricing_evidence: boolean; + require_runner_context: boolean; + require_direct_run: boolean; + require_calibration: boolean; + actor: string; + note: string | null; + created_at: Date; +}): ProviderExpansionPolicy { + return { + id: row.id, + version: row.version, + requiredCapabilities: row.required_capabilities, + requireIdentityProvenance: row.require_identity_provenance, + requireCatalogCollection: row.require_catalog_collection, + requireProviderPolicy: row.require_provider_policy, + requirePricingEvidence: row.require_pricing_evidence, + requireRunnerContext: row.require_runner_context, + requireDirectRun: row.require_direct_run, + requireCalibration: row.require_calibration, + actor: row.actor, + note: row.note, + createdAt: row.created_at.toISOString(), + }; +} + +const POLICY_SELECT = ` + SELECT + id, + version, + required_capabilities, + require_identity_provenance, + require_catalog_collection, + require_provider_policy, + require_pricing_evidence, + require_runner_context, + require_direct_run, + require_calibration, + actor, + note, + created_at + FROM modelapse.provider_expansion_policies +`; + +function gate(input: { + key: ProviderExpansionGate["key"]; + complete: boolean; + limited?: boolean; + summary: string; + blockers?: readonly string[]; + caveats?: readonly string[]; + evidenceIds?: readonly string[]; +}): ProviderExpansionGate { + return { + key: input.key, + status: !input.complete ? "fail" : input.limited ? "limited" : "pass", + summary: input.summary, + blockers: input.blockers ?? [], + caveats: input.caveats ?? [], + evidenceIds: [...new Set(input.evidenceIds ?? [])], + }; +} + +export class PgProviderExpansion { + constructor(private readonly pool: Pool) {} + + static connect( + connectionString: string, + options: { readonly max?: number } = {}, + ): PgProviderExpansion { + return new PgProviderExpansion( + new Pool({ connectionString, max: options.max ?? 3 }), + ); + } + + async close(): Promise { + await this.pool.end(); + } + + async ping(): Promise { + await this.pool.query("SELECT 1"); + } + + async listPolicies(): Promise { + const result = await this.pool.query( + POLICY_SELECT + " ORDER BY created_at DESC, version DESC, id DESC", + ); + return result.rows.map((row) => policyView(row)); + } + + async getPolicy( + version?: string, + ): Promise { + const result = version + ? await this.pool.query( + POLICY_SELECT + " WHERE version = $1 LIMIT 1", + [version], + ) + : await this.pool.query( + POLICY_SELECT + + " ORDER BY created_at DESC, version DESC, id DESC LIMIT 1", + ); + return result.rows[0] ? policyView(result.rows[0]) : null; + } + + async recordPolicy( + input: RecordProviderExpansionPolicyInput, + ): Promise { + const version = input.version.trim(); + if (!VERSION_RE.test(version)) { + throw new Error("version must be a lowercase policy identifier"); + } + const actor = requiredText(input.actor, "actor"); + const capabilities = [...new Set(input.requiredCapabilities)]; + if ( + capabilities.length === 0 || + capabilities.some((capability) => !CAPABILITY_KEYS.has(capability)) + ) { + throw new Error("requiredCapabilities contains an invalid capability"); + } + + const result = await this.pool.query( + `INSERT INTO modelapse.provider_expansion_policies + ( + version, + required_capabilities, + require_identity_provenance, + require_catalog_collection, + require_provider_policy, + require_pricing_evidence, + require_runner_context, + require_direct_run, + require_calibration, + actor, + note + ) + VALUES ( + $1, + $2::text[], + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10, + $11 + ) + RETURNING + id, + version, + required_capabilities, + require_identity_provenance, + require_catalog_collection, + require_provider_policy, + require_pricing_evidence, + require_runner_context, + require_direct_run, + require_calibration, + actor, + note, + created_at`, + [ + version, + capabilities, + input.requireIdentityProvenance, + input.requireCatalogCollection, + input.requireProviderPolicy, + input.requirePricingEvidence, + input.requireRunnerContext, + input.requireDirectRun, + input.requireCalibration, + actor, + optionalText(input.note), + ], + ); + return policyView(result.rows[0]!); + } + + async recordCapability( + input: RecordProviderCapabilityInput, + ): Promise { + if (!UUID_RE.test(input.providerId)) { + throw new Error("providerId must be a UUID"); + } + if (!CAPABILITY_KEYS.has(input.capability)) { + throw new Error("capability is invalid"); + } + if ( + input.supportState !== "supported" && + input.supportState !== "unsupported" + ) { + throw new Error("supportState is invalid"); + } + if (input.sourceId && !UUID_RE.test(input.sourceId)) { + throw new Error("sourceId must be a UUID"); + } + const actor = requiredText(input.actor, "actor"); + const result = await this.pool.query<{ + id: string; + capability: ProviderExpansionCapabilityKey; + support_state: ProviderExpansionCapabilitySupport; + source_id: string | null; + declared_at: Date; + actor: string; + note: string | null; + }>( + `INSERT INTO modelapse.provider_capability_events + ( + provider_id, + capability, + support_state, + source_id, + declared_at, + actor, + note + ) + SELECT + provider.id, + $2, + $3, + $4::uuid, + $5, + $6, + $7 + FROM modelapse.providers provider + WHERE provider.id = $1 + AND ( + $4::uuid IS NULL OR EXISTS ( + SELECT 1 + FROM modelapse.source_records + WHERE id = $4 + ) + ) + RETURNING + id, + capability, + support_state, + source_id, + declared_at, + actor, + note`, + [ + input.providerId, + input.capability, + input.supportState, + input.sourceId ?? null, + normalizeTimestamp(input.declaredAt), + actor, + optionalText(input.note), + ], + ); + const row = result.rows[0]; + if (!row) throw new Error("provider or source record not found"); + + return { + capability: row.capability, + supportState: row.support_state, + eventId: row.id, + sourceId: row.source_id, + declaredAt: row.declared_at.toISOString(), + actor: row.actor, + note: row.note, + observed: false, + }; + } + + async listProviders( + policyVersion?: string, + ): Promise { + const providerResult = await this.pool.query<{ + id: string; + }>("SELECT id FROM modelapse.providers ORDER BY slug"); + const details = await Promise.all( + providerResult.rows.map((row) => + this.getProvider(row.id, policyVersion), + ), + ); + return details + .filter( + (item): item is ProviderExpansionProvider => item !== null, + ) + .map((item) => ({ + provider: item.provider, + policyVersion: item.policyVersion, + status: item.status, + blockerCount: item.blockerCount, + caveatCount: item.caveatCount, + activeModelCount: item.activeModelCount, + latestDirectRunAt: item.latestDirectRunAt, + latestCalibrationAt: item.latestCalibrationAt, + })); + } + + async getProvider( + providerId: string, + policyVersion?: string, + ): Promise { + if (!UUID_RE.test(providerId)) { + throw new Error("providerId must be a UUID"); + } + + const policy = await this.getPolicy(policyVersion); + if (!policy) return null; + + const [ + clock, + providerResult, + modelsResult, + catalogResult, + testabilityResult, + fleetResult, + runEvidenceResult, + calibrationResult, + capabilitiesResult, + ] = await Promise.all([ + this.pool.query<{ generated_at: Date }>("SELECT now() AS generated_at"), + this.pool.query<{ id: string; slug: string; name: string }>( + `SELECT id, slug, name + FROM modelapse.providers + WHERE id = $1 + LIMIT 1`, + [providerId], + ), + this.pool.query<{ + id: string; + canonical_slug: string; + marketing_name: string; + status: string; + canonical_source_id: string | null; + binding_id: string | null; + endpoint_source_id: string | null; + binding_source_id: string | null; + }>( + `SELECT + model.id, + model.canonical_slug, + model.marketing_name, + model.status, + model.canonical_source_id, + binding.id AS binding_id, + endpoint.source_id AS endpoint_source_id, + binding.source_id AS binding_source_id + FROM modelapse.models model + LEFT JOIN LATERAL ( + SELECT current_binding.* + FROM modelapse.model_execution_bindings current_binding + JOIN modelapse.provider_endpoints current_endpoint + ON current_endpoint.id = current_binding.endpoint_id + AND current_endpoint.provider_id = model.provider_id + AND current_endpoint.path = 'first_party_direct' + AND (current_endpoint.valid_from IS NULL OR current_endpoint.valid_from <= now()) + AND (current_endpoint.valid_to IS NULL OR current_endpoint.valid_to > now()) + WHERE current_binding.model_id = model.id + AND current_binding.valid_from <= now() + AND (current_binding.valid_to IS NULL OR current_binding.valid_to > now()) + ORDER BY current_binding.valid_from DESC, current_binding.id DESC + LIMIT 1 + ) binding ON true + LEFT JOIN modelapse.provider_endpoints endpoint + ON endpoint.id = binding.endpoint_id + WHERE model.provider_id = $1 + AND model.status IN ('preview', 'active') + ORDER BY model.canonical_slug`, + [providerId], + ), + this.pool.query<{ + source_id: string; + source_key: string; + source_kind: "model_list" | "docs"; + parser: string; + latest_run_id: string | null; + latest_status: string | null; + latest_source_record_id: string | null; + latest_completed_at: Date | null; + }>( + `SELECT + source.id AS source_id, + source.source_key, + source.source_kind, + source.parser, + latest.run_id AS latest_run_id, + latest.status AS latest_status, + latest.source_record_id AS latest_source_record_id, + latest.completed_at AS latest_completed_at + FROM modelapse.catalog_observer_sources source + LEFT JOIN LATERAL ( + SELECT + run.id AS run_id, + run.status, + snapshot.source_record_id, + run.completed_at + FROM modelapse.catalog_collection_runs run + JOIN modelapse.catalog_source_snapshots snapshot + ON snapshot.collection_run_id = run.id + WHERE run.observer_source_id = source.id + AND run.status IN ('succeeded', 'partial') + ORDER BY run.completed_at DESC NULLS LAST, run.started_at DESC, run.id DESC + LIMIT 1 + ) latest ON true + WHERE source.provider_id = $1 + AND source.enabled + ORDER BY source.source_key`, + [providerId], + ), + this.pool.query<{ + id: string; + model_id: string | null; + subject_kind: "provider_policy" | "runner_access"; + access_state: string; + region_policy: string; + account_tier: string | null; + service_tier: string | null; + service_assurance: string; + pricing_currency: string | null; + input_price_per_million: string | null; + output_price_per_million: string | null; + request_price: string | null; + source_id: string; + }>( + `SELECT + observation.id, + observation.model_id, + observation.subject_kind, + observation.access_state, + observation.region_policy, + observation.account_tier, + observation.service_tier, + observation.service_assurance, + observation.pricing_currency, + observation.input_price_per_million::text, + observation.output_price_per_million::text, + observation.request_price::text, + observation.source_id + FROM modelapse.provider_testability_current observation + WHERE observation.provider_id = $1 + AND observation.execution_path = 'first_party_direct' + ORDER BY observation.subject_kind, observation.model_id NULLS FIRST`, + [providerId], + ), + this.pool.query<{ + environment_id: string; + slug: string; + region: string; + account_tier: string | null; + service_tier: string | null; + service_assurance: string; + capability_event_id: string; + }>( + `SELECT + environment.id AS environment_id, + environment.slug, + environment.region, + environment.account_tier, + environment.service_tier, + environment.service_assurance, + capability.id AS capability_event_id + FROM modelapse.execution_environment_capabilities_current capability + JOIN modelapse.execution_environments environment + ON environment.id = capability.environment_id + JOIN modelapse.execution_environment_current_state state + ON state.environment_id = environment.id + WHERE capability.provider_id = $1 + AND capability.execution_path = 'first_party_direct' + AND capability.enabled + AND state.enabled + ORDER BY capability.selection_priority, environment.slug`, + [providerId], + ), + this.pool.query<{ + latest_completed_at: Date | null; + run_ids: string[]; + returned_model_observed: boolean; + model_version_observed: boolean; + provider_request_id_observed: boolean; + provider_response_id_observed: boolean; + service_tier_observed: boolean; + token_usage_observed: boolean; + }>( + `SELECT + max(run.completed_at) AS latest_completed_at, + COALESCE(array_agg(run.id ORDER BY run.completed_at DESC) + FILTER (WHERE run.id IS NOT NULL), '{}'::uuid[]) AS run_ids, + COALESCE(bool_or(run.returned_model IS NOT NULL), false) + AS returned_model_observed, + COALESCE(bool_or(metadata.model_version_string IS NOT NULL), false) + AS model_version_observed, + COALESCE(bool_or(metadata.provider_request_id IS NOT NULL), false) + AS provider_request_id_observed, + COALESCE(bool_or(metadata.provider_response_id IS NOT NULL), false) + AS provider_response_id_observed, + COALESCE(bool_or( + qualification.returned_service_tier IS NOT NULL + OR metadata.metadata ? 'serviceTier' + ), false) AS service_tier_observed, + COALESCE(bool_or(metadata.usage IS NOT NULL), false) + AS token_usage_observed + FROM modelapse.runs run + JOIN modelapse.run_evidence_summary evidence + ON evidence.run_id = run.id + AND evidence.level IN ('E4', 'E5') + LEFT JOIN modelapse.provider_run_metadata metadata + ON metadata.run_id = run.id + LEFT JOIN modelapse.run_execution_qualification qualification + ON qualification.run_id = run.id + WHERE run.provider_id = $1 + AND run.execution_path = 'first_party_direct' + AND run.status = 'completed' + AND run.sealed_at IS NOT NULL`, + [providerId], + ), + this.pool.query<{ + run_id: string; + status: "pass" | "anomaly" | "unknown"; + repeated_anomaly: boolean; + completed_at: Date; + }>( + `SELECT + assessment.run_id, + assessment.status, + assessment.repeated_anomaly, + run.completed_at + FROM modelapse.calibration_run_assessments assessment + JOIN modelapse.runs run ON run.id = assessment.run_id + WHERE run.provider_id = $1 + AND run.status = 'completed' + AND run.sealed_at IS NOT NULL + ORDER BY run.completed_at DESC, assessment.run_id DESC + LIMIT 1`, + [providerId], + ), + this.pool.query<{ + id: string; + capability: ProviderExpansionCapabilityKey; + support_state: ProviderExpansionCapabilitySupport; + source_id: string | null; + declared_at: Date; + actor: string; + note: string | null; + }>( + `SELECT + id, + capability, + support_state, + source_id, + declared_at, + actor, + note + FROM modelapse.provider_capability_current + WHERE provider_id = $1 + ORDER BY capability`, + [providerId], + ), + ]); + + const providerRow = providerResult.rows[0]; + if (!providerRow) return null; + + const models = modelsResult.rows.map((row) => ({ + id: row.id, + canonicalSlug: row.canonical_slug, + marketingName: row.marketing_name, + status: row.status, + canonicalSourceId: row.canonical_source_id, + sourcedDirectBindingId: row.binding_id, + endpointSourceId: row.endpoint_source_id, + bindingSourceId: row.binding_source_id, + })); + + const currentCapabilities = new Map( + capabilitiesResult.rows.map((row) => [row.capability, row] as const), + ); + const catalogModelListObserved = catalogResult.rows.some( + (row) => + row.source_kind === "model_list" && + row.latest_source_record_id !== null, + ); + const catalogAnyObserved = catalogResult.rows.some( + (row) => row.latest_source_record_id !== null, + ); + + const runEvidence = runEvidenceResult.rows[0] ?? { + latest_completed_at: null, + run_ids: [], + returned_model_observed: false, + model_version_observed: false, + provider_request_id_observed: false, + provider_response_id_observed: false, + service_tier_observed: false, + token_usage_observed: false, + }; + + const observedByCapability: Record< + ProviderExpansionCapabilityKey, + boolean + > = { + returned_model_metadata: runEvidence.returned_model_observed, + model_version_metadata: runEvidence.model_version_observed, + provider_request_id: runEvidence.provider_request_id_observed, + provider_response_id: runEvidence.provider_response_id_observed, + service_tier_metadata: runEvidence.service_tier_observed, + token_usage: runEvidence.token_usage_observed, + catalog_model_list: catalogModelListObserved, + }; + + const capabilities: ProviderExpansionCapability[] = + providerExpansionCapabilityKeys.map((capability) => { + const row = currentCapabilities.get(capability); + return { + capability, + supportState: row?.support_state ?? null, + eventId: row?.id ?? null, + sourceId: row?.source_id ?? null, + declaredAt: row?.declared_at.toISOString() ?? null, + actor: row?.actor ?? null, + note: row?.note ?? null, + observed: observedByCapability[capability], + }; + }); + + const activeModelIds = models.map((model) => model.id); + const identityBlockers: string[] = []; + const identityEvidence: string[] = []; + if (models.length === 0) { + identityBlockers.push("active_model_missing"); + } + for (const model of models) { + if (!model.canonicalSourceId) { + identityBlockers.push( + "canonical_source_missing:" + model.canonicalSlug, + ); + } else { + identityEvidence.push(model.canonicalSourceId); + } + if (!model.sourcedDirectBindingId || !model.bindingSourceId) { + identityBlockers.push( + "sourced_direct_binding_missing:" + model.canonicalSlug, + ); + } else { + identityEvidence.push(model.sourcedDirectBindingId, model.bindingSourceId); + } + if (!model.endpointSourceId) { + identityBlockers.push( + "endpoint_source_missing:" + model.canonicalSlug, + ); + } else { + identityEvidence.push(model.endpointSourceId); + } + } + + const modelListDeclaration = + currentCapabilities.get("catalog_model_list"); + const catalogBlockers: string[] = []; + const catalogCaveats: string[] = []; + if (!catalogAnyObserved) { + catalogBlockers.push("catalog_collection_evidence_missing"); + } + if (!catalogModelListObserved) { + if (modelListDeclaration?.support_state === "unsupported") { + catalogCaveats.push("catalog_model_list_explicitly_unsupported"); + } else { + catalogBlockers.push("model_list_collection_missing"); + } + } + + const providerPolicies = testabilityResult.rows.filter( + (row) => row.subject_kind === "provider_policy", + ); + const providerWidePolicy = providerPolicies.find( + (row) => row.model_id === null, + ); + const policyCoveredModels = new Set( + providerPolicies + .filter((row) => row.model_id !== null) + .map((row) => row.model_id as string), + ); + const providerPolicyComplete = + Boolean(providerWidePolicy) || + (activeModelIds.length > 0 && + activeModelIds.every((id) => policyCoveredModels.has(id))); + const accessBlockers: string[] = []; + const accessCaveats: string[] = []; + if (!providerPolicyComplete) { + accessBlockers.push("provider_policy_evidence_missing"); + } + for (const observation of providerPolicies) { + if (observation.region_policy === "unknown") { + accessBlockers.push("region_policy_unknown:" + observation.id); + } + if ( + observation.access_state === "restricted" || + observation.access_state === "unavailable" + ) { + accessCaveats.push( + "provider_access_" + observation.access_state + ":" + observation.id, + ); + } else if (observation.access_state === "unknown") { + accessBlockers.push("provider_access_unknown:" + observation.id); + } + } + + const pricedPolicies = providerPolicies.filter( + (row) => + row.pricing_currency !== null && + ( + row.input_price_per_million !== null || + row.output_price_per_million !== null || + row.request_price !== null + ), + ); + const providerWidePricing = pricedPolicies.some( + (row) => row.model_id === null, + ); + const pricedModels = new Set( + pricedPolicies + .filter((row) => row.model_id !== null) + .map((row) => row.model_id as string), + ); + const pricingComplete = + providerWidePricing || + (activeModelIds.length > 0 && + activeModelIds.every((id) => pricedModels.has(id))); + const pricingBlockers = pricingComplete + ? [] + : ["pricing_evidence_missing"]; + + const runnerAccess = testabilityResult.rows.filter( + (row) => row.subject_kind === "runner_access", + ); + const runnableAccess = runnerAccess.filter( + (row) => + row.access_state === "available" && + row.account_tier !== null && + ( + row.service_assurance === "documented_default" || + row.service_assurance === "documented_variant" + ), + ); + const fleet = fleetResult.rows.map((row) => ({ + environmentId: row.environment_id, + slug: row.slug, + region: row.region, + accountTier: row.account_tier, + serviceTier: row.service_tier, + serviceAssurance: row.service_assurance, + capabilityEventId: row.capability_event_id, + })); + const qualifiedFleet = fleet.filter( + (environment) => + environment.region.trim().length > 0 && + environment.accountTier !== null && + ( + environment.serviceAssurance === "documented_default" || + environment.serviceAssurance === "documented_variant" + ), + ); + const runnerBlockers: string[] = []; + const runnerCaveats: string[] = []; + if (runnableAccess.length === 0) { + runnerBlockers.push("runner_access_context_missing"); + } + if (qualifiedFleet.length === 0) { + runnerBlockers.push("execution_environment_context_missing"); + } + if ( + runnerAccess.some( + (row) => + row.service_assurance === "operator_uncertain" || + row.service_assurance === "unknown", + ) + ) { + runnerCaveats.push("runner_service_assurance_uncertain"); + } + + const directRunIds = runEvidence.run_ids ?? []; + const directRunBlockers = + directRunIds.length > 0 ? [] : ["verified_direct_run_missing"]; + + const latestCalibrationRow = calibrationResult.rows[0]; + const latestCalibration = latestCalibrationRow + ? { + runId: latestCalibrationRow.run_id, + status: latestCalibrationRow.status, + repeatedAnomaly: latestCalibrationRow.repeated_anomaly, + completedAt: latestCalibrationRow.completed_at.toISOString(), + } + : null; + const calibrationBlockers = latestCalibration + ? [] + : ["calibration_evidence_missing"]; + const calibrationCaveats: string[] = []; + if (latestCalibration?.repeatedAnomaly) { + calibrationCaveats.push("calibration_repeated_anomaly"); + } else if (latestCalibration?.status === "anomaly") { + calibrationCaveats.push("calibration_anomaly_needs_replication"); + } else if (latestCalibration?.status === "unknown") { + calibrationCaveats.push("calibration_status_unknown"); + } + + const capabilityBlockers: string[] = []; + const capabilityCaveats: string[] = []; + const capabilityEvidence: string[] = []; + for (const capability of policy.requiredCapabilities) { + const current = capabilities.find( + (item) => item.capability === capability, + )!; + if (!current.eventId || !current.supportState) { + capabilityBlockers.push( + "capability_declaration_missing:" + capability, + ); + continue; + } + capabilityEvidence.push(current.eventId); + if (current.sourceId) capabilityEvidence.push(current.sourceId); + if (current.supportState === "unsupported") { + capabilityCaveats.push( + "capability_explicitly_unsupported:" + capability, + ); + } else if (!current.observed) { + capabilityBlockers.push( + "capability_supported_but_unobserved:" + capability, + ); + } + } + + const gates: ProviderExpansionGate[] = [ + gate({ + key: "identity_provenance", + complete: + !policy.requireIdentityProvenance || identityBlockers.length === 0, + summary: + models.length + + " active/preview model(s); sourced canonical identity + first-party binding required", + blockers: policy.requireIdentityProvenance ? identityBlockers : [], + evidenceIds: identityEvidence, + }), + gate({ + key: "catalog_collection", + complete: + !policy.requireCatalogCollection || catalogBlockers.length === 0, + limited: + policy.requireCatalogCollection && catalogCaveats.length > 0, + summary: + catalogResult.rows.length + + " enabled catalog source(s); current collection evidence required", + blockers: policy.requireCatalogCollection ? catalogBlockers : [], + caveats: catalogCaveats, + evidenceIds: catalogResult.rows.flatMap((row) => [ + row.source_id, + ...(row.latest_run_id ? [row.latest_run_id] : []), + ...(row.latest_source_record_id + ? [row.latest_source_record_id] + : []), + ]), + }), + gate({ + key: "access_region_evidence", + complete: + !policy.requireProviderPolicy || accessBlockers.length === 0, + limited: + policy.requireProviderPolicy && accessCaveats.length > 0, + summary: + providerPolicies.length + + " current provider-policy observation(s)", + blockers: policy.requireProviderPolicy ? accessBlockers : [], + caveats: accessCaveats, + evidenceIds: providerPolicies.flatMap((row) => [row.id, row.source_id]), + }), + gate({ + key: "pricing_evidence", + complete: + !policy.requirePricingEvidence || pricingBlockers.length === 0, + summary: + pricedPolicies.length + " current priced policy observation(s)", + blockers: policy.requirePricingEvidence ? pricingBlockers : [], + evidenceIds: pricedPolicies.flatMap((row) => [row.id, row.source_id]), + }), + gate({ + key: "runner_account_context", + complete: + !policy.requireRunnerContext || runnerBlockers.length === 0, + limited: + policy.requireRunnerContext && runnerCaveats.length > 0, + summary: + runnableAccess.length + + " usable runner-access observation(s); " + + qualifiedFleet.length + + " qualified fleet environment(s)", + blockers: policy.requireRunnerContext ? runnerBlockers : [], + caveats: runnerCaveats, + evidenceIds: [ + ...runnableAccess.flatMap((row) => [row.id, row.source_id]), + ...qualifiedFleet.flatMap((environment) => [ + environment.environmentId, + environment.capabilityEventId, + ]), + ], + }), + gate({ + key: "direct_execution_evidence", + complete: + !policy.requireDirectRun || directRunBlockers.length === 0, + summary: + directRunIds.length + + " sealed first-party E4/E5 completed Run(s)", + blockers: policy.requireDirectRun ? directRunBlockers : [], + evidenceIds: directRunIds, + }), + gate({ + key: "calibration_coverage", + complete: + !policy.requireCalibration || calibrationBlockers.length === 0, + limited: + policy.requireCalibration && calibrationCaveats.length > 0, + summary: latestCalibration + ? "latest calibration " + latestCalibration.status + : "no calibration assessment", + blockers: policy.requireCalibration ? calibrationBlockers : [], + caveats: calibrationCaveats, + evidenceIds: latestCalibration ? [latestCalibration.runId] : [], + }), + gate({ + key: "capability_contract", + complete: capabilityBlockers.length === 0, + limited: + capabilityBlockers.length === 0 && capabilityCaveats.length > 0, + summary: + policy.requiredCapabilities.length + + " required runtime capability declaration(s)", + blockers: capabilityBlockers, + caveats: capabilityCaveats, + evidenceIds: capabilityEvidence, + }), + ]; + + const blockers = [...new Set(gates.flatMap((item) => item.blockers))]; + const caveats = [...new Set(gates.flatMap((item) => item.caveats))]; + const status: ProviderExpansionStatus = + blockers.length > 0 + ? "incomplete" + : gates.some((item) => item.status === "limited") + ? "limited" + : "ready"; + + return { + generatedAt: clock.rows[0]!.generated_at.toISOString(), + provider: { + id: providerRow.id, + slug: providerRow.slug, + name: providerRow.name, + }, + policyVersion: policy.version, + status, + blockerCount: blockers.length, + caveatCount: caveats.length, + activeModelCount: models.length, + latestDirectRunAt: + runEvidence.latest_completed_at?.toISOString() ?? null, + latestCalibrationAt: latestCalibration?.completedAt ?? null, + policy, + blockers, + caveats, + gates, + capabilities, + models, + fleet, + latestCalibration, + }; + } +} diff --git a/packages/catalog-admin/test/bootstrap.integration.test.ts b/packages/catalog-admin/test/bootstrap.integration.test.ts index 1e472ca..04bbd0f 100644 --- a/packages/catalog-admin/test/bootstrap.integration.test.ts +++ b/packages/catalog-admin/test/bootstrap.integration.test.ts @@ -2410,6 +2410,32 @@ describe("production catalog bootstrap", () => { expect(target.providerId).toBe(first.providerId); expect(target.endpointBaseUrl).toBe("https://api.deepseek.com"); expect(target.promptBlob.sha256).toBe(first.promptSha256); + + const verification = new Pool({ connectionString: isolatedDatabaseUrl }); + try { + const capabilities = await verification.query<{ + capability: string; + support_state: string; + }>( + `SELECT capability, support_state + FROM modelapse.provider_capability_current + WHERE provider_id = $1 + ORDER BY capability`, + [first.providerId], + ); + expect(capabilities.rows).toEqual( + expect.arrayContaining([ + { capability: "returned_model_metadata", support_state: "supported" }, + { capability: "model_version_metadata", support_state: "unsupported" }, + { capability: "provider_request_id", support_state: "supported" }, + { capability: "provider_response_id", support_state: "supported" }, + { capability: "service_tier_metadata", support_state: "unsupported" }, + { capability: "token_usage", support_state: "supported" }, + ]), + ); + } finally { + await verification.end(); + } }); it("is idempotent and produces an executable sourced direct target", async () => { @@ -2464,6 +2490,27 @@ describe("production catalog bootstrap", () => { family_source_id: first.definitionSourceId, version_source_id: first.definitionSourceId, }); + + const capabilities = await verification.query<{ + capability: string; + support_state: string; + }>( + `SELECT capability, support_state + FROM modelapse.provider_capability_current + WHERE provider_id = $1 + ORDER BY capability`, + [first.providerId], + ); + expect(capabilities.rows).toEqual( + expect.arrayContaining([ + { capability: "returned_model_metadata", support_state: "supported" }, + { capability: "model_version_metadata", support_state: "unsupported" }, + { capability: "provider_request_id", support_state: "supported" }, + { capability: "provider_response_id", support_state: "supported" }, + { capability: "service_tier_metadata", support_state: "supported" }, + { capability: "token_usage", support_state: "supported" }, + ]), + ); } finally { await verification.end(); } diff --git a/packages/catalog-admin/test/provider-expansion.integration.test.ts b/packages/catalog-admin/test/provider-expansion.integration.test.ts new file mode 100644 index 0000000..8375131 --- /dev/null +++ b/packages/catalog-admin/test/provider-expansion.integration.test.ts @@ -0,0 +1,748 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { migrateDatabase } from "@modelapse/database"; +import { Pool } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { PgProviderExpansion } from "../src/index.js"; + +const ADMIN_DATABASE_URL = + process.env.DATABASE_URL ?? + "postgres://modelapse:modelapse@127.0.0.1:5432/modelapse"; + +function databaseUrl(databaseName: string): string { + const url = new URL(ADMIN_DATABASE_URL); + url.pathname = "/" + databaseName; + return url.toString(); +} + +describe("Provider Expansion Playbook", () => { + const adminPool = new Pool({ connectionString: ADMIN_DATABASE_URL }); + const databaseName = + "modelapse_expansion_" + randomUUID().replace(/-/g, "").slice(0, 12); + const isolatedDatabaseUrl = databaseUrl(databaseName); + let pool: Pool; + let expansion: PgProviderExpansion; + let providerId = ""; + let capabilitySourceId = ""; + + beforeAll(async () => { + await adminPool.query(`CREATE DATABASE "${databaseName}"`); + await migrateDatabase({ + connectionString: isolatedDatabaseUrl, + migrationsDirectory: fileURLToPath( + new URL("../../database/migrations/", import.meta.url), + ), + runnerBuild: "provider-expansion-integration", + }); + pool = new Pool({ connectionString: isolatedDatabaseUrl }); + expansion = PgProviderExpansion.connect(isolatedDatabaseUrl, { max: 2 }); + + const source = await pool.query<{ id: string }>( + `INSERT INTO modelapse.source_records + (source_type, url, title, content_sha256) + VALUES + ('provider_docs', 'https://expansion.example.test/docs', 'Provider docs', $1), + ('provider_catalog', 'https://expansion.example.test/models', 'Provider model list', $2), + ('operator_verification', NULL, 'Runner verification', $3), + ('modelapse_definition', 'https://github.com/imwarn/modelapse', 'Adapter definition', $4) + RETURNING id`, + [ + randomBytes(32).toString("hex"), + randomBytes(32).toString("hex"), + randomBytes(32).toString("hex"), + randomBytes(32).toString("hex"), + ], + ); + const [docsSource, catalogSource, runnerSource, capabilitySource] = + source.rows; + capabilitySourceId = capabilitySource!.id; + + const provider = await pool.query<{ id: string }>( + `INSERT INTO modelapse.providers (slug, name, homepage) + VALUES ('expansion-fixture', 'Expansion Fixture', 'https://expansion.example.test/') + RETURNING id`, + ); + providerId = provider.rows[0]!.id; + + const endpoint = await pool.query<{ id: string }>( + `INSERT INTO modelapse.provider_endpoints + (provider_id, path, base_url, hostname, valid_from, source_id) + VALUES ( + $1, + 'first_party_direct', + 'https://api.expansion.example.test', + 'api.expansion.example.test', + '2026-08-01T00:00:00Z', + $2 + ) + RETURNING id`, + [providerId, docsSource!.id], + ); + + const model = await pool.query<{ id: string }>( + `INSERT INTO modelapse.models + ( + provider_id, + canonical_slug, + marketing_name, + status, + canonical_source_id + ) + VALUES ($1, 'expansion-model', 'Expansion Model', 'active', $2) + RETURNING id`, + [providerId, docsSource!.id], + ); + const modelId = model.rows[0]!.id; + + await pool.query( + `INSERT INTO modelapse.model_execution_bindings + ( + model_id, + endpoint_id, + api_model_id, + valid_from, + source_id + ) + VALUES ($1, $2, 'expansion-model', '2026-08-01T00:00:00Z', $3)`, + [modelId, endpoint.rows[0]!.id, docsSource!.id], + ); + + const observer = await pool.query<{ id: string }>( + `INSERT INTO modelapse.catalog_observer_sources + ( + provider_id, + source_key, + source_kind, + url, + title, + parser, + interval_seconds, + enabled + ) + VALUES ( + $1, + 'models', + 'model_list', + 'https://api.expansion.example.test/models', + 'Expansion model list', + 'openai_models', + 3600, + true + ) + RETURNING id`, + [providerId], + ); + const collection = await pool.query<{ id: string }>( + `INSERT INTO modelapse.catalog_collection_runs + ( + observer_source_id, + status, + started_at, + completed_at, + item_count, + observations_emitted, + collector_build + ) + VALUES ( + $1, + 'succeeded', + '2026-09-01T00:00:00Z', + '2026-09-01T00:00:01Z', + 1, + 1, + 'provider-expansion-test' + ) + RETURNING id`, + [observer.rows[0]!.id], + ); + await pool.query( + `INSERT INTO modelapse.catalog_source_snapshots + ( + observer_source_id, + collection_run_id, + source_record_id, + retrieved_at, + content_sha256, + response_body + ) + VALUES ($1, $2, $3, '2026-09-01T00:00:01Z', $4, '{"data":[]}'::text)`, + [ + observer.rows[0]!.id, + collection.rows[0]!.id, + catalogSource!.id, + randomBytes(32).toString("hex"), + ], + ); + + const providerPolicy = await pool.query<{ id: string }>( + `INSERT INTO modelapse.provider_testability_observations + ( + provider_id, + execution_path, + subject_kind, + access_state, + registration_requirement, + billing_requirement, + region_policy, + allowed_regions, + service_assurance, + pricing_currency, + input_price_per_million, + output_price_per_million, + source_id, + observed_at, + actor + ) + VALUES ( + $1, + 'first_party_direct', + 'provider_policy', + 'available', + 'open_signup', + 'paid_account', + 'unrestricted', + ARRAY['US'], + 'documented_default', + 'USD', + 1.000000, + 2.000000, + $2, + '2026-09-01T00:10:00Z', + 'provider-expansion-test' + ) + RETURNING id`, + [providerId, docsSource!.id], + ); + + await pool.query( + `INSERT INTO modelapse.provider_testability_observations + ( + provider_id, + execution_path, + subject_kind, + access_state, + registration_requirement, + billing_requirement, + region_policy, + allowed_regions, + account_tier, + service_tier, + service_assurance, + source_id, + observed_at, + actor + ) + VALUES ( + $1, + 'first_party_direct', + 'runner_access', + 'available', + 'open_signup', + 'paid_account', + 'unrestricted', + ARRAY['US'], + 'paid-standard', + 'default', + 'documented_default', + $2, + '2026-09-01T00:11:00Z', + 'provider-expansion-test' + )`, + [providerId, runnerSource!.id], + ); + + const environment = await pool.query<{ id: string }>( + `INSERT INTO modelapse.execution_environments + ( + slug, + region, + account_tier, + service_tier, + service_assurance, + created_by + ) + VALUES ( + 'expansion-us-paid', + 'US', + 'paid-standard', + 'default', + 'documented_default', + 'provider-expansion-test' + ) + RETURNING id`, + ); + await pool.query( + `INSERT INTO modelapse.execution_environment_state_events + (environment_id, enabled, effective_at, actor) + VALUES ($1, true, '2026-09-01T00:12:00Z', 'provider-expansion-test')`, + [environment.rows[0]!.id], + ); + const fleetCapability = await pool.query<{ id: string }>( + `INSERT INTO modelapse.execution_environment_capability_events + ( + environment_id, + provider_id, + execution_path, + enabled, + selection_priority, + effective_at, + actor + ) + VALUES ( + $1, + $2, + 'first_party_direct', + true, + 10, + '2026-09-01T00:12:00Z', + 'provider-expansion-test' + ) + RETURNING id`, + [environment.rows[0]!.id, providerId], + ); + + const promptSha = randomBytes(32).toString("hex"); + await pool.query( + `INSERT INTO modelapse.blobs + (sha256, size_bytes, mime_type, object_key, visibility) + VALUES ($1, 1, 'text/plain', $2, 'public')`, + [promptSha, "provider-expansion/" + promptSha], + ); + + const family = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_families + (slug, name, origin, canonical_source_id) + VALUES ('provider-expansion-fixture', 'Provider Expansion Fixture', 'modelapse', $1) + RETURNING id`, + [docsSource!.id], + ); + const benchmarkVariant = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_variants + (family_id, slug, name, category, artifact_type) + VALUES ($1, 'benchmark', 'Benchmark', 'smoke', 'text') + RETURNING id`, + [family.rows[0]!.id], + ); + const benchmarkVersion = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_versions + (variant_id, version, status, definition_sha256, source_id) + VALUES ($1, '1.0.0', 'draft', $2, $3) + RETURNING id`, + [ + benchmarkVariant.rows[0]!.id, + randomBytes(32).toString("hex"), + docsSource!.id, + ], + ); + const benchmarkCase = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_cases + ( + test_version_id, + slug, + case_type, + visibility, + status, + prompt_blob_sha256 + ) + VALUES ($1, 'smoke', 'public', 'public', 'active', $2) + RETURNING id`, + [benchmarkVersion.rows[0]!.id, promptSha], + ); + await pool.query( + `UPDATE modelapse.test_versions + SET status = 'published', + published_at = '2026-09-01T00:13:00Z' + WHERE id = $1`, + [benchmarkVersion.rows[0]!.id], + ); + + const directRun = await pool.query<{ id: string }>( + `INSERT INTO modelapse.runs + ( + test_case_id, + model_id, + provider_id, + execution_path, + requested_model, + returned_model, + status, + runner_build, + started_at, + completed_at, + sealed_at + ) + VALUES ( + $1, + $2, + $3, + 'first_party_direct', + 'expansion-model', + 'expansion-model', + 'completed', + 'provider-expansion-test', + '2026-09-01T00:20:00Z', + '2026-09-01T00:20:01Z', + '2026-09-01T00:20:02Z' + ) + RETURNING id`, + [benchmarkCase.rows[0]!.id, modelId, providerId], + ); + const directRunId = directRun.rows[0]!.id; + await pool.query( + `INSERT INTO modelapse.run_execution_qualification_envelopes + ( + run_id, + selected_at, + execution_environment_id, + execution_capability_event_id, + execution_region, + provider_policy_observation_id, + account_tier, + service_tier, + requested_service_tier, + service_assurance, + caveats + ) + VALUES ( + $1, + '2026-09-01T00:19:59Z', + $2, + $3, + 'US', + $4, + 'paid-standard', + 'default', + 'default', + 'documented_default', + '{}'::text[] + )`, + [ + directRunId, + environment.rows[0]!.id, + fleetCapability.rows[0]!.id, + providerPolicy.rows[0]!.id, + ], + ); + await pool.query( + `INSERT INTO modelapse.run_execution_qualification_outcomes + (run_id, returned_service_tier, captured_at) + VALUES ($1, 'default', '2026-09-01T00:20:01Z')`, + [directRunId], + ); + await pool.query( + `INSERT INTO modelapse.provider_run_metadata + ( + run_id, + provider_request_id, + provider_response_id, + usage, + timing, + metadata + ) + VALUES ( + $1, + 'req-expansion', + 'resp-expansion', + '{"inputTokens":4,"outputTokens":1,"totalTokens":5}'::jsonb, + '{"durationMs":1}'::jsonb, + '{"serviceTier":"default"}'::jsonb + )`, + [directRunId], + ); + await pool.query( + `INSERT INTO modelapse.evidence_records + (run_id, level, execution_path, collector) + VALUES ($1, 'E4', 'first_party_direct', 'provider-expansion-test')`, + [directRunId], + ); + + const calibrationVariant = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_variants + (family_id, slug, name, category, artifact_type) + VALUES ($1, 'calibration', 'Calibration', 'calibration', 'text') + RETURNING id`, + [family.rows[0]!.id], + ); + const calibrationVersion = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_versions + (variant_id, version, status, definition_sha256, source_id) + VALUES ($1, '1.0.0', 'draft', $2, $3) + RETURNING id`, + [ + calibrationVariant.rows[0]!.id, + randomBytes(32).toString("hex"), + docsSource!.id, + ], + ); + const calibrationCase = await pool.query<{ id: string }>( + `INSERT INTO modelapse.test_cases + ( + test_version_id, + slug, + case_type, + visibility, + status, + prompt_blob_sha256, + metadata + ) + VALUES ( + $1, + 'service-health', + 'calibration', + 'public', + 'active', + $2, + '{"expected":"ok","assertion":"exact-text","leaderboardEligible":false}'::jsonb + ) + RETURNING id`, + [calibrationVersion.rows[0]!.id, promptSha], + ); + const evaluator = await pool.query<{ id: string }>( + `SELECT id + FROM modelapse.evaluators + WHERE slug = 'exact-text' + AND version = '1.0.0' + LIMIT 1`, + ); + await pool.query( + `INSERT INTO modelapse.test_version_evaluators + (test_version_id, evaluator_id) + VALUES ($1, $2)`, + [calibrationVersion.rows[0]!.id, evaluator.rows[0]!.id], + ); + await pool.query( + `UPDATE modelapse.test_versions + SET status = 'published', + published_at = '2026-09-01T00:21:00Z' + WHERE id = $1`, + [calibrationVersion.rows[0]!.id], + ); + + const calibrationRun = await pool.query<{ id: string }>( + `INSERT INTO modelapse.runs + ( + test_case_id, + model_id, + provider_id, + execution_path, + requested_model, + returned_model, + status, + runner_build, + started_at, + completed_at, + sealed_at + ) + VALUES ( + $1, + $2, + $3, + 'first_party_direct', + 'expansion-model', + 'expansion-model', + 'completed', + 'provider-expansion-test', + '2026-09-01T00:22:00Z', + '2026-09-01T00:22:01Z', + '2026-09-01T00:22:02Z' + ) + RETURNING id`, + [calibrationCase.rows[0]!.id, modelId, providerId], + ); + const calibrationRunId = calibrationRun.rows[0]!.id; + const evaluation = await pool.query<{ id: string }>( + `INSERT INTO modelapse.evaluations + ( + run_id, + evaluator_id, + status, + started_at, + completed_at + ) + VALUES ( + $1, + $2, + 'completed', + '2026-09-01T00:22:01Z', + '2026-09-01T00:22:01Z' + ) + RETURNING id`, + [calibrationRunId, evaluator.rows[0]!.id], + ); + await pool.query( + `INSERT INTO modelapse.metric_values + (evaluation_id, metric_key, numeric_value) + VALUES ($1, 'exact_match', 1)`, + [evaluation.rows[0]!.id], + ); + const calibrationPolicy = await pool.query<{ id: string }>( + `SELECT id + FROM modelapse.calibration_policies + WHERE version = 'calibration-v1'`, + ); + await pool.query( + `INSERT INTO modelapse.calibration_run_assessments + ( + run_id, + policy_id, + status, + exact_match, + anomaly_streak, + repeated_anomaly, + repeat_recommended, + assessed_at + ) + VALUES ( + $1, + $2, + 'pass', + true, + 0, + false, + false, + '2026-09-01T00:22:02Z' + )`, + [calibrationRunId, calibrationPolicy.rows[0]!.id], + ); + }); + + afterAll(async () => { + await expansion?.close(); + await pool?.end(); + await adminPool.query(`DROP DATABASE IF EXISTS "${databaseName}"`); + await adminPool.end(); + }); + + it("starts incomplete, then becomes limited only by an explicit unsupported capability", async () => { + const initial = await expansion.getProvider(providerId); + expect(initial).not.toBeNull(); + expect(initial).toMatchObject({ + policyVersion: "provider-expansion-v1", + status: "incomplete", + activeModelCount: 1, + latestCalibration: { + status: "pass", + repeatedAnomaly: false, + }, + }); + expect(initial!.blockers).toEqual( + expect.arrayContaining([ + "capability_declaration_missing:returned_model_metadata", + "capability_declaration_missing:model_version_metadata", + "capability_declaration_missing:provider_request_id", + "capability_declaration_missing:provider_response_id", + "capability_declaration_missing:service_tier_metadata", + "capability_declaration_missing:token_usage", + ]), + ); + + const declarations = [ + ["returned_model_metadata", "supported"], + ["model_version_metadata", "unsupported"], + ["provider_request_id", "supported"], + ["provider_response_id", "supported"], + ["service_tier_metadata", "supported"], + ["token_usage", "supported"], + ] as const; + + for (const [capability, supportState] of declarations) { + await expansion.recordCapability({ + providerId, + capability, + supportState, + sourceId: capabilitySourceId, + actor: "provider-expansion-test", + }); + } + + const limited = await expansion.getProvider(providerId); + expect(limited).not.toBeNull(); + expect(limited!.status).toBe("limited"); + expect(limited!.blockers).toEqual([]); + expect(limited!.caveats).toContain( + "capability_explicitly_unsupported:model_version_metadata", + ); + expect( + limited!.capabilities.find( + (item) => item.capability === "returned_model_metadata", + ), + ).toMatchObject({ + supportState: "supported", + observed: true, + }); + expect( + limited!.capabilities.find( + (item) => item.capability === "service_tier_metadata", + ), + ).toMatchObject({ + supportState: "supported", + observed: true, + }); + }); + + it("supports versioned playbook policies without rewriting old declarations", async () => { + const policy = await expansion.recordPolicy({ + version: "provider-expansion-fixture-v2", + requiredCapabilities: [ + "returned_model_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", + ], + requireIdentityProvenance: true, + requireCatalogCollection: true, + requireProviderPolicy: true, + requirePricingEvidence: true, + requireRunnerContext: true, + requireDirectRun: true, + requireCalibration: true, + actor: "provider-expansion-test", + note: "Fixture policy excludes model version capture.", + }); + + const ready = await expansion.getProvider(providerId, policy.version); + expect(ready).not.toBeNull(); + expect(ready).toMatchObject({ + policyVersion: "provider-expansion-fixture-v2", + status: "ready", + blockers: [], + caveats: [], + }); + + const listed = await expansion.listProviders(policy.version); + expect( + listed.find((item) => item.provider.id === providerId), + ).toMatchObject({ + status: "ready", + blockerCount: 0, + caveatCount: 0, + }); + + const capabilityEvent = await pool.query<{ id: string }>( + `SELECT id + FROM modelapse.provider_capability_events + WHERE provider_id = $1 + ORDER BY created_at + LIMIT 1`, + [providerId], + ); + await expect( + pool.query( + `UPDATE modelapse.provider_capability_events + SET note = 'rewritten' + WHERE id = $1`, + [capabilityEvent.rows[0]!.id], + ), + ).rejects.toThrow(/append-only/i); + + await expect( + pool.query( + `UPDATE modelapse.provider_expansion_policies + SET note = 'rewritten' + WHERE id = $1`, + [policy.id], + ), + ).rejects.toThrow(/append-only/i); + }); +}); diff --git a/packages/database/migrations/0023_provider_expansion_playbook.sql b/packages/database/migrations/0023_provider_expansion_playbook.sql new file mode 100644 index 0000000..a56749d --- /dev/null +++ b/packages/database/migrations/0023_provider_expansion_playbook.sql @@ -0,0 +1,108 @@ +BEGIN; + +SET search_path TO modelapse, public; + +CREATE TABLE provider_expansion_policies ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + version text NOT NULL UNIQUE + CHECK (version ~ '^[a-z0-9][a-z0-9._-]*$'), + required_capabilities text[] NOT NULL DEFAULT ARRAY[ + 'returned_model_metadata', + 'model_version_metadata', + 'provider_request_id', + 'provider_response_id', + 'service_tier_metadata', + 'token_usage' + ]::text[], + require_identity_provenance boolean NOT NULL DEFAULT true, + require_catalog_collection boolean NOT NULL DEFAULT true, + require_provider_policy boolean NOT NULL DEFAULT true, + require_pricing_evidence boolean NOT NULL DEFAULT true, + require_runner_context boolean NOT NULL DEFAULT true, + require_direct_run boolean NOT NULL DEFAULT true, + require_calibration boolean NOT NULL DEFAULT true, + actor text NOT NULL CHECK (length(btrim(actor)) > 0), + note text, + created_at timestamptz NOT NULL DEFAULT now(), + CHECK ( + required_capabilities <@ ARRAY[ + 'returned_model_metadata', + 'model_version_metadata', + 'provider_request_id', + 'provider_response_id', + 'service_tier_metadata', + 'token_usage', + 'catalog_model_list' + ]::text[] + ) +); + +CREATE TRIGGER provider_expansion_policies_append_only +BEFORE UPDATE OR DELETE ON provider_expansion_policies +FOR EACH ROW EXECUTE FUNCTION prevent_append_only_mutation(); + +INSERT INTO provider_expansion_policies + ( + version, + actor, + note + ) +VALUES ( + 'provider-expansion-v1', + 'archive-v0.24-migration', + 'Provider onboarding requires sourced identity, collection, access, pricing, runner context, direct execution, calibration, and explicit runtime capability declarations.' +) +ON CONFLICT (version) DO NOTHING; + +CREATE VIEW provider_expansion_policy_current AS +SELECT * +FROM provider_expansion_policies +ORDER BY created_at DESC, version DESC, id DESC +LIMIT 1; + +CREATE TABLE provider_capability_events ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + provider_id uuid NOT NULL REFERENCES providers(id), + capability text NOT NULL CHECK ( + capability IN ( + 'returned_model_metadata', + 'model_version_metadata', + 'provider_request_id', + 'provider_response_id', + 'service_tier_metadata', + 'token_usage', + 'catalog_model_list' + ) + ), + support_state text NOT NULL + CHECK (support_state IN ('supported', 'unsupported')), + source_id uuid REFERENCES source_records(id), + declared_at timestamptz NOT NULL DEFAULT now(), + actor text NOT NULL CHECK (length(btrim(actor)) > 0), + note text, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TRIGGER provider_capability_events_append_only +BEFORE UPDATE OR DELETE ON provider_capability_events +FOR EACH ROW EXECUTE FUNCTION prevent_append_only_mutation(); + +CREATE INDEX provider_capability_events_lookup_idx + ON provider_capability_events( + provider_id, + capability, + declared_at DESC, + id DESC + ); + +CREATE VIEW provider_capability_current AS +SELECT DISTINCT ON (provider_id, capability) + * +FROM provider_capability_events +ORDER BY + provider_id, + capability, + declared_at DESC, + id DESC; + +COMMIT; diff --git a/packages/evidence-transport/test/node.test.ts b/packages/evidence-transport/test/node.test.ts index 0854514..e9dc59e 100644 --- a/packages/evidence-transport/test/node.test.ts +++ b/packages/evidence-transport/test/node.test.ts @@ -24,6 +24,14 @@ describe("NodeEvidenceTransport", () => { providerSlug: "example", executionPath: "first_party_direct", allowedHosts: ["api.example.com"], + capabilities: { + returned_model_metadata: "unsupported", + model_version_metadata: "unsupported", + provider_request_id: "unsupported", + provider_response_id: "unsupported", + service_tier_metadata: "unsupported", + token_usage: "unsupported", + }, }, request: { url: "https://api.example.com/v1/test", diff --git a/packages/persistence/test/integration.test.ts b/packages/persistence/test/integration.test.ts index f000679..1d05684 100644 --- a/packages/persistence/test/integration.test.ts +++ b/packages/persistence/test/integration.test.ts @@ -410,6 +410,14 @@ describe("PostgreSQL Run persistence", () => { providerSlug: "integration-router", executionPath: "routed_provider", allowedHosts: ["router.fake.test"], + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }, }, prepare: (request) => ({ url: "https://router.fake.test/v1/messages", diff --git a/packages/provider-adapter/src/types.ts b/packages/provider-adapter/src/types.ts index 0660e48..cb40dcd 100644 --- a/packages/provider-adapter/src/types.ts +++ b/packages/provider-adapter/src/types.ts @@ -87,6 +87,20 @@ export interface NormalizedProviderResponse { readonly providerMetadata?: Readonly>; } +export const providerAdapterCapabilityKeys = [ + "returned_model_metadata", + "model_version_metadata", + "provider_request_id", + "provider_response_id", + "service_tier_metadata", + "token_usage", +] as const; + +export type ProviderAdapterCapabilityKey = + (typeof providerAdapterCapabilityKeys)[number]; + +export type ProviderAdapterCapabilitySupport = "supported" | "unsupported"; + export interface ProviderAdapterDescriptor { readonly id: string; readonly providerSlug: string; @@ -94,4 +108,11 @@ export interface ProviderAdapterDescriptor { /** Exact hostnames the controlled transport may contact for this adapter. */ readonly allowedHosts: readonly string[]; readonly apiVersion?: string; + /** + * Exhaustive response/metadata capture contract for the adapter. + * Unsupported capabilities must be declared explicitly rather than omitted. + */ + readonly capabilities: Readonly< + Record + >; } diff --git a/packages/provider-adapter/test/adapter.test.ts b/packages/provider-adapter/test/adapter.test.ts index 6b9db24..36daef9 100644 --- a/packages/provider-adapter/test/adapter.test.ts +++ b/packages/provider-adapter/test/adapter.test.ts @@ -1,11 +1,22 @@ import { describe, expect, it } from "vitest"; -import { assertPreparedRequestAllowed } from "../src/index.js"; +import { + assertPreparedRequestAllowed, + type ProviderAdapterDescriptor, +} from "../src/index.js"; -const descriptor = { +const descriptor: ProviderAdapterDescriptor = { id: "openai-direct", providerSlug: "openai", executionPath: "first_party_direct" as const, allowedHosts: ["api.openai.com"], + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "supported", + token_usage: "supported", + }, }; describe("provider endpoint policy", () => { diff --git a/packages/provider-anthropic/src/index.ts b/packages/provider-anthropic/src/index.ts index fc741b3..a1bdbf1 100644 --- a/packages/provider-anthropic/src/index.ts +++ b/packages/provider-anthropic/src/index.ts @@ -4,6 +4,7 @@ import type { NormalizedProviderResponse, PreparedHttpRequest, ProviderAdapter, + ProviderAdapterDescriptor, ProviderContentPart, } from "@modelapse/provider-adapter"; @@ -17,12 +18,20 @@ function textOnly(parts: readonly ProviderContentPart[]): string { } export class AnthropicMessagesAdapter implements ProviderAdapter { - readonly descriptor = { + readonly descriptor: ProviderAdapterDescriptor = { id: "anthropic-messages-direct-2023-06-01", providerSlug: "anthropic", executionPath: "first_party_direct" as const, allowedHosts: ["api.anthropic.com"], apiVersion: "2023-06-01", + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }, }; prepare(request: CanonicalModelRequest): PreparedHttpRequest { diff --git a/packages/provider-anthropic/test/adapter.test.ts b/packages/provider-anthropic/test/adapter.test.ts index 1dfb80d..15425a8 100644 --- a/packages/provider-anthropic/test/adapter.test.ts +++ b/packages/provider-anthropic/test/adapter.test.ts @@ -10,6 +10,17 @@ const request = { }; describe("AnthropicMessagesAdapter", () => { + it("declares supported and unsupported metadata capture explicitly", () => { + expect(new AnthropicMessagesAdapter().descriptor.capabilities).toEqual({ + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }); + }); + it("requires an explicit max output token budget", () => { expect(() => new AnthropicMessagesAdapter().prepare({ diff --git a/packages/provider-deepseek/src/index.ts b/packages/provider-deepseek/src/index.ts index 318b8a9..2516d09 100644 --- a/packages/provider-deepseek/src/index.ts +++ b/packages/provider-deepseek/src/index.ts @@ -4,6 +4,7 @@ import type { NormalizedProviderResponse, PreparedHttpRequest, ProviderAdapter, + ProviderAdapterDescriptor, ProviderContentPart, } from "@modelapse/provider-adapter"; @@ -44,12 +45,20 @@ function outputText(body: Record): string { } export class DeepSeekResponsesAdapter implements ProviderAdapter { - readonly descriptor = { + readonly descriptor: ProviderAdapterDescriptor = { id: "deepseek-responses-direct-v1", providerSlug: "deepseek", executionPath: "first_party_direct" as const, allowedHosts: ["api.deepseek.com"], apiVersion: "responses-v1", + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }, }; prepare(request: CanonicalModelRequest): PreparedHttpRequest { diff --git a/packages/provider-deepseek/test/adapter.test.ts b/packages/provider-deepseek/test/adapter.test.ts index 3ae2aa1..62bb086 100644 --- a/packages/provider-deepseek/test/adapter.test.ts +++ b/packages/provider-deepseek/test/adapter.test.ts @@ -2,6 +2,17 @@ import { describe, expect, it } from "vitest"; import { DeepSeekResponsesAdapter } from "../src/index.js"; describe("DeepSeek Responses adapter", () => { + it("declares supported and unsupported metadata capture explicitly", () => { + expect(new DeepSeekResponsesAdapter().descriptor.capabilities).toEqual({ + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }); + }); + it("prepares the first-party Responses API request without credentials", () => { const adapter = new DeepSeekResponsesAdapter(); const prepared = adapter.prepare({ diff --git a/packages/provider-openai/src/index.ts b/packages/provider-openai/src/index.ts index eb86cd7..7a7c2ed 100644 --- a/packages/provider-openai/src/index.ts +++ b/packages/provider-openai/src/index.ts @@ -4,6 +4,7 @@ import type { NormalizedProviderResponse, PreparedHttpRequest, ProviderAdapter, + ProviderAdapterDescriptor, ProviderContentPart, } from "@modelapse/provider-adapter"; @@ -37,12 +38,20 @@ function outputText(body: Record): string { } export class OpenAIResponsesAdapter implements ProviderAdapter { - readonly descriptor = { + readonly descriptor: ProviderAdapterDescriptor = { id: "openai-responses-direct-v1", providerSlug: "openai", executionPath: "first_party_direct" as const, allowedHosts: ["api.openai.com"], apiVersion: "responses-v1", + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "supported", + token_usage: "supported", + }, }; prepare(request: CanonicalModelRequest): PreparedHttpRequest { diff --git a/packages/provider-openai/test/adapter.test.ts b/packages/provider-openai/test/adapter.test.ts index 7ff465e..c072f6f 100644 --- a/packages/provider-openai/test/adapter.test.ts +++ b/packages/provider-openai/test/adapter.test.ts @@ -11,6 +11,17 @@ const request = { }; describe("OpenAIResponsesAdapter", () => { + it("declares supported and unsupported metadata capture explicitly", () => { + expect(new OpenAIResponsesAdapter().descriptor.capabilities).toEqual({ + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "supported", + token_usage: "supported", + }); + }); + it("prepares a direct Responses request without embedding credentials", () => { const prepared = new OpenAIResponsesAdapter().prepare(request); expect(prepared.url).toBe("https://api.openai.com/v1/responses"); diff --git a/packages/runner/test/runner.test.ts b/packages/runner/test/runner.test.ts index e8d13e5..ece10ff 100644 --- a/packages/runner/test/runner.test.ts +++ b/packages/runner/test/runner.test.ts @@ -16,6 +16,14 @@ const adapter: ProviderAdapter = { providerSlug: "fake", executionPath: "first_party_direct", allowedHosts: ["api.fake.test"], + capabilities: { + returned_model_metadata: "supported", + model_version_metadata: "unsupported", + provider_request_id: "supported", + provider_response_id: "supported", + service_tier_metadata: "unsupported", + token_usage: "supported", + }, }, prepare: () => ({ url: "https://api.fake.test/v1/messages",