From 8f618a34c38f1555be2dc88c3207d3afefd6ff35 Mon Sep 17 00:00:00 2001 From: Cedric Vidal Date: Wed, 4 Nov 2026 09:55:44 -0800 Subject: [PATCH] fix: isolate profile variation submissions Move profile fan-out ownership into the API so every profile resolves against fresh shared run intent. Keep Portal and CLI payloads free of profile-managed fields and document the cross-client ownership invariant. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 10 + apps/api/src/endpoints.test.ts | 154 +++++++++++++++ apps/api/src/routes/requests/index.ts | 87 ++++----- apps/cli/src/commands/run.test.ts | 85 +++++++++ apps/cli/src/commands/run.ts | 28 ++- apps/portal/src/pages/SubmitRun.tsx | 2 +- .../src/pages/SubmitRun.variations.test.tsx | 176 ++++++++++++++++++ docs/architecture/app-design.md | 7 +- docs/architecture/resources.md | 6 +- 9 files changed, 503 insertions(+), 52 deletions(-) create mode 100644 apps/portal/src/pages/SubmitRun.variations.test.tsx diff --git a/AGENTS.md b/AGENTS.md index 11ddac8bd..026564b22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,16 @@ config/ # Benchmark definitions (YAML) For package architecture and data models, see [docs/architecture/app-design.md](docs/architecture/app-design.md). +## Cross-client domain ownership + +The API owns domain validation, orchestration, fan-out, defaulting, and +resolution rules shared by Portal, CLI, and direct API consumers. Portal and CLI +are thin clients: they collect user intent and may provide early validation, but +must not duplicate or materialize API-owned business logic. When one user action +expands into multiple domain objects, each object is derived independently by +the API from the shared intent so client or sibling state cannot leak between +them. + ## Key Components ### API (`apps/api/`) diff --git a/apps/api/src/endpoints.test.ts b/apps/api/src/endpoints.test.ts index 9552f0696..7e9ed0800 100644 --- a/apps/api/src/endpoints.test.ts +++ b/apps/api/src/endpoints.test.ts @@ -2231,6 +2231,160 @@ describe("API Endpoints", () => { expect(res.body.error).toContain("missing a prompt"); expect(mocks.collection.insertMany).not.toHaveBeenCalled(); }); + + it("rejects shared resources in variation mode before resolving profiles", async () => { + const resourceRevision = { + _id: "resource-revision-1", + resourceId: "resource-1", + projectId: TEST_PROJECT_ID, + slug: "github-simulator", + revisionNumber: 1, + ref: "github-simulator@r1", + setup: { sh: "echo ready" }, + exports: [], + parameters: [{ name: "REPO", required: true }], + contentSha256: "sha", + createdAt: new Date(), + }; + (mocks.resourceStore.getBySlug as any).mockResolvedValue({ + _id: "resource-1", + projectId: TEST_PROJECT_ID, + slug: "github-simulator", + latestRevisionId: resourceRevision._id, + }); + (mocks.resourceRevisionStore.getLatest as any).mockResolvedValue(resourceRevision); + (mocks.profileVersionCollection.findOne as any).mockImplementation( + async (q: { profileId: string }) => ({ + _id: `pv-${q.profileId}`, + profileId: q.profileId, + version: 1, + workerType: "coder-acp-copilot", + model: q.profileId === "base-profile" ? "m1" : "m2", + mcpServers: [], + skillRevisions: [], + extensions: [], + resources: [{ + ref: "github-simulator", + params: { REPO: "microsoft/scope" }, + }], + }), + ); + + const res = await request(testServer()) + .post(`/api/v1/requests?projectId=${TEST_PROJECT_ID}`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + profileId: "base-profile", + profileVariations: ["var-profile"], + resources: [{ + ref: "github-simulator", + params: { REPO: "microsoft/scope" }, + }], + }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain("resources"); + expect(mocks.collection.insertMany).not.toHaveBeenCalled(); + }); + + it.each([ + ["model", "m1"], + ["reasoningEffort", "high"], + ["agentVersion", "v1"], + ["mcpServers", ["server-a"]], + ["skills", ["skill-a"]], + ["extensions", ["publisher.extension"]], + ])("rejects shared %s in variation mode", async (field, value) => { + const res = await request(testServer()) + .post(`/api/v1/requests?projectId=${TEST_PROJECT_ID}`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + profileId: "base-profile", + profileVariations: ["var-profile"], + [field]: value, + }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain(field); + expect(mocks.collection.insertMany).not.toHaveBeenCalled(); + }); + + it("rejects a shared worker query in variation mode", async () => { + const res = await request(testServer()) + .post(`/api/v1/requests?projectId=${TEST_PROJECT_ID}&worker=coder-acp-copilot`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + profileId: "base-profile", + profileVariations: ["var-profile"], + }); + + expect(res.status).toBe(400); + expect(res.body.error).toContain("worker"); + expect(mocks.collection.insertMany).not.toHaveBeenCalled(); + }); + + it("resolves each variation profile's resources without leaking base values", async () => { + const resourceRevision = { + _id: "resource-revision-1", + resourceId: "resource-1", + projectId: TEST_PROJECT_ID, + slug: "github-simulator", + revisionNumber: 1, + ref: "github-simulator@r1", + setup: { sh: "echo ready" }, + exports: [], + parameters: [{ name: "REPO", required: true }], + contentSha256: "sha", + createdAt: new Date(), + }; + (mocks.resourceStore.getBySlug as any).mockResolvedValue({ + _id: "resource-1", + projectId: TEST_PROJECT_ID, + slug: "github-simulator", + latestRevisionId: resourceRevision._id, + }); + (mocks.resourceRevisionStore.getLatest as any).mockResolvedValue(resourceRevision); + (mocks.profileVersionCollection.findOne as any).mockImplementation( + async (q: { profileId: string }) => ({ + _id: `pv-${q.profileId}`, + ref: `${q.profileId}@1`, + profileId: q.profileId, + version: 1, + workerType: "coder-acp-copilot", + model: q.profileId === "base-profile" ? "m1" : "m2", + mcpServers: [], + skillRevisions: [], + extensions: [], + resources: [{ + ref: "github-simulator", + params: { + REPO: q.profileId === "base-profile" + ? "baseline/repository" + : "comparison/repository", + }, + }], + }), + ); + + const res = await request(testServer()) + .post(`/api/v1/requests?projectId=${TEST_PROJECT_ID}`) + .send({ + scenario: { task: "Build something", criteria: ["works"] }, + profileId: "base-profile", + profileVariations: ["var-profile"], + }); + + expect(res.status).toBe(201); + const docs = (mocks.collection.insertMany as any).mock.calls[0][0]; + expect(docs).toHaveLength(2); + expect(docs.map((doc: { profileId: string; resources: Array<{ params: Record }> }) => ({ + profileId: doc.profileId, + repo: doc.resources[0].params.REPO, + }))).toEqual([ + { profileId: "base-profile", repo: "baseline/repository" }, + { profileId: "var-profile", repo: "comparison/repository" }, + ]); + }); }); // =================================================================== diff --git a/apps/api/src/routes/requests/index.ts b/apps/api/src/routes/requests/index.ts index 0d719ffbb..7c644c3fe 100644 --- a/apps/api/src/routes/requests/index.ts +++ b/apps/api/src/routes/requests/index.ts @@ -462,6 +462,44 @@ apiRoute(ctx.app, ctx.registry, { const { scenario: scenarioObj, persona: personaObj, maxIterations, personaInstructions, count = 1, model: requestedModel, reasoningEffort: requestedReasoningEffort, mcpServers: mcpServerSlugs, skills: skillSlugs, extensions: extensionIds, agentVersion: requestedAgentVersion, profileId: requestedProfileSpec, profileVariations, priority: requestedPriority, agentsMd: requestedAgentsMd, agentsMdParentIds: requestedAgentsMdParentIds, gates: requestedGates, codebase: codebaseSpec, codebaseRevisionId: requestedCodebaseRevisionId, resources: resourceSpecs } = req.body; let worker = req.query.worker as string | undefined; + const typedProfileVariations: string[] = Array.isArray(profileVariations) + ? profileVariations.filter((v: unknown): v is string => typeof v === "string") + : []; + const isVariationSubmit = typedProfileVariations.length > 0; + + if (isVariationSubmit) { + if (!requestedProfileSpec) { + res.status(400).json({ error: "profileId (the base profile) is required when profileVariations are provided" }); + return; + } + + // Profile fan-out starts from shared run intent only. Every controlled + // value must be resolved independently from each profile. + const requestControlledFields: Array<[string, unknown]> = [ + ["model", requestedModel], + ["reasoningEffort", requestedReasoningEffort], + ["agentVersion", requestedAgentVersion], + ["mcpServers", mcpServerSlugs], + ["skills", skillSlugs], + ["extensions", extensionIds], + ["resources", resourceSpecs], + ]; + const providedControlledFields = requestControlledFields.filter(([, value]) => value !== undefined); + if (providedControlledFields.length > 0) { + res.status(400).json({ + error: `In variation mode, request-level ${providedControlledFields.map(([field]) => field).join("/")} must be omitted; each profile supplies its own controlled configuration.`, + }); + return; + } + + if (worker !== undefined) { + res.status(400).json({ + error: "The ?worker query parameter is not allowed in variation mode; worker is derived from each variation profile.", + }); + return; + } + } + // AGENTS.md body + lineage (for any caller that wants to attach an // AGENTS.md instruction file to a run). const agentsMdText = @@ -520,34 +558,7 @@ apiRoute(ctx.app, ctx.registry, { label?: string; }; - const typedProfileVariations: string[] = Array.isArray(profileVariations) - ? profileVariations.filter((v: unknown): v is string => typeof v === "string") - : []; - const isVariationSubmit = typedProfileVariations.length > 0; - if (isVariationSubmit) { - if (!requestedProfileSpec) { - res.status(400).json({ error: "profileId (the base profile) is required when profileVariations are provided" }); - return; - } - - // In variation mode, controlled fields come from the variation profiles themselves. - if (requestedModel !== undefined || mcpServerSlugs !== undefined || skillSlugs !== undefined || extensionIds !== undefined) { - res.status(400).json({ - error: "In variation mode, model/mcpServers/skills/extensions must be set via the variation profile.", - }); - return; - } - - // In variation mode, the worker is derived per-variation from each profile's workerType. - // Reject the ?worker= query param so callers don't think it has any effect. - if (worker) { - res.status(400).json({ - error: "The ?worker query parameter is not allowed in variation mode; worker is derived from each variation profile.", - }); - return; - } - if (!scenarioObj || typeof scenarioObj !== "object" || !scenarioObj.task || typeof scenarioObj.task !== "string") { res.status(400).json({ error: "scenario.task is required and must be a string" }); return; @@ -569,7 +580,7 @@ apiRoute(ctx.app, ctx.registry, { } const baseParsed = (() => { - try { return parseProfileSpec(requestedProfileSpec); } + try { return parseProfileSpec(requestedProfileSpec!); } catch (err) { res.status(400).json({ error: (err as Error).message }); return null; } })(); if (!baseParsed) return; @@ -673,25 +684,17 @@ apiRoute(ctx.app, ctx.registry, { const effectiveMcpServers = variationProfileVersion.mcpServers ?? undefined; const effectiveSkills = variationProfileVersion.skillRevisions ?? undefined; const effectiveExtensions = variationProfileVersion.extensions ?? undefined; - const requestedVariationAgentVersion = variationProfileVersion.agentVersion ?? requestedAgentVersion; const targetCheck = await validateAgentTarget(ctx.agentCollection, { workerType: variationWorkerType, - requestedVersion: requestedVariationAgentVersion, + requestedVersion: variationProfileVersion.agentVersion, model: variationProfileVersion.model, requireModel: true, requirements: requestedAgentCapabilities({ - reasoningEffort: - variationProfileVersion.reasoningEffort ?? requestedReasoningEffort, + reasoningEffort: variationProfileVersion.reasoningEffort, mcpServers: effectiveMcpServers, skillRevisions: effectiveSkills, extensions: effectiveExtensions, - // Mirrors resolveResourceBindings' precedence (profile wins). Uses the - // specs rather than resolved bindings because the capability check only - // needs to know whether any resource was requested, and resolution - // happens after this point. - resources: - normalizeResourceBindingSpecs(variationProfileVersion.resources) - ?? requestedResourceSpecs, + resources: normalizeResourceBindingSpecs(variationProfileVersion.resources), }), strictCapabilities: ctx.strictAgentCapabilities, }); @@ -772,7 +775,7 @@ apiRoute(ctx.app, ctx.registry, { const resourceResult = await resolveResourceBindings( ctx, projectId, - requestedResourceSpecs, + undefined, normalizeResourceBindingSpecs(variationProfileVersion.resources), ); if (resourceResult.errors.length > 0) { @@ -833,8 +836,8 @@ apiRoute(ctx.app, ctx.registry, { createdAt: new Date(), priority: requestedPriority ?? 0, ...(r.model ? { model: r.model } : {}), - ...((r.profileVersion.reasoningEffort ?? requestedReasoningEffort) - ? { reasoningEffort: r.profileVersion.reasoningEffort ?? requestedReasoningEffort } + ...(r.profileVersion.reasoningEffort + ? { reasoningEffort: r.profileVersion.reasoningEffort } : {}), ...(maxIterations ? { maxIterations } : {}), ...(personaInstructions ? { personaInstructions } : {}), diff --git a/apps/cli/src/commands/run.test.ts b/apps/cli/src/commands/run.test.ts index b26334bab..8b5fa3bd7 100644 --- a/apps/cli/src/commands/run.test.ts +++ b/apps/cli/src/commands/run.test.ts @@ -3,6 +3,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { Command } from "commander"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { registerRunCommands } from "./run.js"; interface MockResponse { @@ -236,6 +239,88 @@ describe("run submit", () => { const { body } = captureSubmit(); expect(body).not.toHaveProperty("agentsMd"); }); + + it("submits only shared intent for profile variations", async () => { + mockFetchWith({ + ids: ["req-base", "req-variation"], + count: 2, + variationCount: 2, + status: "pending", + variations: [], + }); + const tempDir = mkdtempSync(join(tmpdir(), "scope-profile-variations-")); + const variationsPath = join(tempDir, "variations.json"); + writeFileSync(variationsPath, JSON.stringify(["comparison-profile@1"])); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + try { + const program = makeProgram(); + await program.parseAsync( + [ + "run", "submit", + "--message", "Compare profiles", + "--max-iterations", "1", + "--profile", "baseline-profile@1", + "--profile-variations-file", variationsPath, + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + ); + } finally { + logSpy.mockRestore(); + rmSync(tempDir, { recursive: true, force: true }); + } + + const { url, body } = captureSubmit(); + expect(url).toBe("http://localhost:3100/api/v1/requests?projectId=proj-test"); + expect(body).toEqual({ + scenario: { task: "Compare profiles", criteria: [] }, + maxIterations: 1, + profileId: "baseline-profile@1", + profileVariations: ["comparison-profile@1"], + }); + }); + + it("rejects profile-managed options before submitting variations", async () => { + mockFetchWith({}); + const tempDir = mkdtempSync(join(tmpdir(), "scope-profile-variations-")); + const variationsPath = join(tempDir, "variations.json"); + writeFileSync(variationsPath, JSON.stringify(["comparison-profile@1"])); + const errorLines: string[] = []; + const errSpy = vi.spyOn(console, "error").mockImplementation((...parts: unknown[]) => { + errorLines.push(parts.map(String).join(" ")); + }); + const exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`process.exit(${code}) called`); + }) as never); + + try { + const program = makeProgram(); + await expect(program.parseAsync( + [ + "run", "submit", + "--message", "Compare profiles", + "--profile", "baseline-profile@1", + "--profile-variations-file", variationsPath, + "--model", "gpt-5.6-sol", + "--resources", "github-simulator@r1", + "--no-stream", + "-u", "http://localhost:3100", + "--project", "proj-test", + ], + { from: "user" }, + )).rejects.toThrow("process.exit(1) called"); + } finally { + errSpy.mockRestore(); + exitSpy.mockRestore(); + rmSync(tempDir, { recursive: true, force: true }); + } + + expect(lastRequest).toBeUndefined(); + expect(errorLines.join("\n")).toContain("--model, --resources cannot be used with --profile-variations-file"); + }); }); describe("run retry", () => { diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index 4c2727699..e56ac6651 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -209,13 +209,29 @@ run body.profileVariations = parsed; } - // In variation mode the API derives the worker per-variation from each - // profile's workerType, and explicitly rejects `?worker=`. Skip the - // query param so the request isn't 400'd, and warn if --worker was - // explicitly passed (default values are silently ignored). const isVariationSubmit = Array.isArray(body.profileVariations) && body.profileVariations.length > 0; - if (isVariationSubmit && command.getOptionValueSource("worker") === "cli") { - console.warn(label("Warning:"), "--worker is ignored in variation mode; worker is derived per-variation from each profile's workerType."); + if (isVariationSubmit) { + // Keep the CLI a thin client, but fail before the network call when the + // user supplies fields that the API resolves independently per profile. + const incompatibleOptions = [ + command.getOptionValueSource("worker") === "cli" ? "--worker" : undefined, + model !== undefined ? "--model" : undefined, + reasoningEffort !== undefined ? "--reasoning-effort" : undefined, + mcpServerSlugs !== undefined ? "--mcp-servers" : undefined, + skillSlugs !== undefined ? "--skills" : undefined, + resourceSpecs !== undefined ? "--resources" : undefined, + resourceParamOverrides.length > 0 ? "--resource-param" : undefined, + extensionIds !== undefined ? "--extensions" : undefined, + agentVersion !== undefined ? "--agent-version" : undefined, + ].filter((flag): flag is string => flag !== undefined); + if (incompatibleOptions.length > 0) { + console.error( + errorText( + `Error: ${incompatibleOptions.join(", ")} cannot be used with --profile-variations-file; each profile supplies its own controlled configuration.`, + ), + ); + process.exit(1); + } } if (!isVariationSubmit && !profileId && !worker) { console.error( diff --git a/apps/portal/src/pages/SubmitRun.tsx b/apps/portal/src/pages/SubmitRun.tsx index 54beb26ed..3fc762efa 100644 --- a/apps/portal/src/pages/SubmitRun.tsx +++ b/apps/portal/src/pages/SubmitRun.tsx @@ -699,7 +699,7 @@ export function SubmitRun() { ...(inVariationMode ? {} : { ...(selectedSkills.length > 0 ? { skills: selectedSkills } : {}) }), ...(inVariationMode ? {} : { ...(selectedExtensions.length > 0 ? { extensions: selectedExtensions } : {}) }), ...(selectedCodebaseSpec ? { codebase: selectedCodebaseSpec } : {}), - ...(selectedResourceSpecs.length > 0 ? { resources: selectedResourceSpecs } : {}), + ...(inVariationMode ? {} : { ...(selectedResourceSpecs.length > 0 ? { resources: selectedResourceSpecs } : {}) }), ...(inVariationMode ? {} : { ...(selectedAgentVersion ? { agentVersion: selectedAgentVersion } : {}) }), ...(inVariationMode ? { diff --git a/apps/portal/src/pages/SubmitRun.variations.test.tsx b/apps/portal/src/pages/SubmitRun.variations.test.tsx new file mode 100644 index 000000000..531eeee75 --- /dev/null +++ b/apps/portal/src/pages/SubmitRun.variations.test.tsx @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// @vitest-environment happy-dom +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { MemoryRouter } from "react-router-dom"; +import type { + CodingAgent, + ProfileWithVersion, + ResourceDocument, + ResourceRevisionDocument, +} from "@/types"; + +vi.mock("@/hooks/useVisibleGates", () => ({ + useVisibleGates: () => ["select"], +})); + +vi.mock("@/lib/api", () => ({ + api: { + getVersion: vi.fn(async () => ({ strictAgentCapabilities: true })), + listAgents: vi.fn(), + listProfiles: vi.fn(), + listProfileVersions: vi.fn(), + listMcpServers: vi.fn(async () => []), + listRuns: vi.fn(async () => ({ data: [] })), + listModels: vi.fn(async () => []), + listCriteria: vi.fn(async () => []), + listTaskPrompts: vi.fn(async () => ({ data: [] })), + listResources: vi.fn(), + listResourceRevisions: vi.fn(), + submitRun: vi.fn(async () => ({ ids: ["run-base", "run-variation"] })), + }, +})); + +import { api } from "@/lib/api"; +import { SubmitRun } from "./SubmitRun"; + +const createdAt = "2026-01-01T00:00:00.000Z"; +const resourceRef = "github-simulator@r1"; + +const agent: CodingAgent = { + _id: "coder-acp-copilot", + name: "GitHub Copilot CLI", + supportedModels: ["gpt-5.6-sol"], + defaultModel: "gpt-5.6-sol", + available: true, + capabilities: { + supportsReasoningEffort: true, + supportsMcpServers: false, + supportsSkills: false, + supportsExtensions: false, + }, + versions: [{ + agentVersion: "1.0", + workerVersion: "1.0", + components: {}, + gitCommit: "abc", + buildTime: createdAt, + imageTag: "test", + queueName: "test-queue", + status: "active", + createdAt, + }], + createdAt, +}; + +function profile(id: string, name: string, repo: string): ProfileWithVersion { + return { + _id: id, + name, + latestVersion: 1, + createdAt, + version: { + _id: `${id}-v1`, + profileId: id, + version: 1, + workerType: agent._id, + model: "gpt-5.6-sol", + agentVersion: "1.0", + resources: [{ ref: resourceRef, params: { REPO: repo } }], + createdAt, + }, + }; +} + +const profiles = [ + profile("baseline", "Baseline profile", "baseline/repository"), + profile("comparison", "Comparison profile", "comparison/repository"), +]; + +const resource: ResourceDocument = { + _id: "resource-1", + projectId: "project-1", + slug: "github-simulator", + name: "GitHub simulator", + revisionCounter: 1, + latestRevisionId: "resource-revision-1", + latestRevisionNumber: 1, + createdAt, +}; + +const resourceRevision: ResourceRevisionDocument = { + _id: "resource-revision-1", + resourceId: resource._id, + projectId: resource.projectId, + slug: resource.slug, + revisionNumber: 1, + ref: resourceRef, + setup: { sh: "echo ready" }, + exports: [], + parameters: [{ name: "REPO", required: true }], + contentSha256: "sha", + createdAt, +}; + +let queryClient: QueryClient; + +beforeEach(() => { + vi.mocked(api.listAgents).mockResolvedValue([agent]); + vi.mocked(api.listProfiles).mockResolvedValue(profiles); + vi.mocked(api.listProfileVersions).mockImplementation(async (profileId) => [ + profiles.find((candidate) => candidate._id === profileId)!.version, + ]); + vi.mocked(api.listResources).mockResolvedValue([resource]); + vi.mocked(api.listResourceRevisions).mockResolvedValue([resourceRevision]); + queryClient = new QueryClient({ + defaultOptions: { queries: { retry: false }, mutations: { retry: false } }, + }); +}); + +afterEach(() => { + cleanup(); + queryClient.clear(); + vi.clearAllMocks(); +}); + +describe("SubmitRun profile variation payload", () => { + it("does not copy baseline profile-managed configuration into the shared request", async () => { + const user = userEvent.setup(); + render( + + + + + , + ); + + const baseProfileInput = await screen.findByPlaceholderText("Search existing profiles…"); + fireEvent.change(baseProfileInput, { target: { value: "Baseline profile" } }); + await user.click(screen.getByRole("listitem")); + await user.click(screen.getByRole("button", { name: "Add" })); + + fireEvent.change(screen.getByLabelText("Task *"), { target: { value: "Build a demo" } }); + fireEvent.change(screen.getByLabelText("Max iterations"), { target: { value: "1" } }); + await user.click(screen.getByRole("button", { name: /Submit 10 Runs/ })); + + await waitFor(() => expect(api.submitRun).toHaveBeenCalledOnce()); + const payload = vi.mocked(api.submitRun).mock.calls[0][0]; + + expect(payload).toEqual(expect.objectContaining({ + profileId: "baseline@1", + profileVariations: ["comparison@1"], + })); + expect(payload).not.toHaveProperty("worker"); + expect(payload).not.toHaveProperty("model"); + expect(payload).not.toHaveProperty("reasoningEffort"); + expect(payload).not.toHaveProperty("agentVersion"); + expect(payload).not.toHaveProperty("mcpServers"); + expect(payload).not.toHaveProperty("skills"); + expect(payload).not.toHaveProperty("extensions"); + expect(payload).not.toHaveProperty("resources"); + }); +}); diff --git a/docs/architecture/app-design.md b/docs/architecture/app-design.md index f1a243262..b06d69f4f 100644 --- a/docs/architecture/app-design.md +++ b/docs/architecture/app-design.md @@ -467,9 +467,14 @@ Profile fan-out mode is also supported for comparative runs: 1. User selects a base `profileId` (optionally pinned via `profileId@version`) and a `profileVariations[]` array of profile spec strings to compare against (e.g. `["def456", "ghi789@2"]`) 2. API parses each spec, validates every variation upfront, then expands one submit call into multiple requests under one shared `submissionId` -3. Each expanded request resolves configuration from its variation profile; unpinned specs resolve to `latestVersion` +3. The base is the first comparison member, not an inheritance source. The API starts from the same shared run intent for every member and resolves worker, model, reasoning effort, agent version, MCP servers, skills, extensions, and resources independently from that member's profile; unpinned specs resolve to `latestVersion` 4. Expanded requests persist `profileId` and `profileVersionId` on each `RequestDocument` for indexing and traceability +Request-level profile-managed fields are rejected in fan-out mode. Portal and +CLI submit only shared intent (scenario, gates, codebase, count, priority, and +AGENTS.md); they do not materialize baseline profile values into the request. +This keeps API, CLI, and Portal submissions behaviorally identical. + ## Runs List Query API `GET /api/v1/requests` powers the Portal **Runs** list and `scope run list`. All diff --git a/docs/architecture/resources.md b/docs/architecture/resources.md index a0fcd826e..47f6a9690 100644 --- a/docs/architecture/resources.md +++ b/docs/architecture/resources.md @@ -136,8 +136,10 @@ refs and any preset parameters. A run may still fill parameters left unset by th profile by sending a matching resource entry at the same position, but a different value for a profile-pinned parameter is reported through the same profile conflict response used for worker/model/MCP/skills/extensions. Grouped -profile-variation submissions apply the same positional rule per variation: -`profileVersion.resources ?? requestedResources`. +profile-variation submissions are intentionally stricter: request-level +`resources` are rejected, and every profile must independently provide complete +resource bindings. The API resolves each profile against a fresh run request so +the base profile's bindings cannot leak into another comparison member. The worker then runs the lifecycle in a fixed order. **Each of these orderings is load-bearing — moving one reintroduces a specific bug:**