feat(evalboard): add a Scribe tab, reading the Autopilot suite's own blob container

evalboard/.gitignore

@@ -4,6 +4,10 @@ out
 *.tsbuildinfo
 .env*.local
 runs-remote/
+# Per-source blob caches (runs-remote-scribe/, …). Each source gets a sibling of
+# the base cache dir — see runsDirFor in lib/sources.ts — and the bare
+# `runs-remote/` rule above does not match those.
+runs-remote-*/
 runs-local/
 coverage/
 

evalboard/README.md

@@ -42,6 +42,9 @@ show up in the index — empty shells and the `latest` symlink are filtered out.
 - `/watchlist` — what needs attention, ranked over the recent-runs window: tasks
   and skills scored on failures, regressions and turn-budget pressure
   (`lib/watchlist.ts`).
+- `/scribe` — the Autopilot (aria/Composer) suite, run by `coder_eval_uipath`'s
+  `UiPath.Autopilot.Eval.Manual` pipeline. This is the one surface that reads a
+  **different blob container** (`aria-runs`, not `runs`) — see *Sources* below.
 - `/runs/latest` — shortcut that redirects to the newest run id.
 - `/runs/<run-id>` — run summary (pass rate, cost, duration) + one row per task.
   A "Download run (.zip)" button bundles the whole run folder.
@@ -56,12 +59,53 @@ show up in the index — empty shells and the `latest` symlink are filtered out.
 `task_results[].task_id` (e.g., `skill-flow-calculator`) and equals the
 subdir name under `<run-id>/default/`.
 
+## Sources
+
+A **source** is one blob container of runs, surfaced as its own tab
+(`lib/sources.ts`). One deployment serves all of them — the container is a
+runtime dimension threaded through the data layer as a trailing
+`source: Source = DEFAULT_SOURCE` parameter, not a build-time env var:
+
+| Source | Container | Surface |
+|--------|-----------|---------|
+| `skills` (default) | `runs` | Everything not listed below |
+| `scribe` | `aria-runs` | `/scribe` |
+
+Non-default sources are selected by a `?src=<id>` query param, which every
+run-scoped page and API route reads. An absent or unrecognised `src` resolves to
+the default source (`sourceById` coerces rather than throwing, so a stray param
+in a shared link degrades to the skills dashboard instead of an error page).
+
+Two invariants worth preserving if you add a source:
+
+- **Run ids are only unique within a container.** Every suite names runs
+  `YYYY-MM-DD_HH-MM-SS`, so the same id routinely exists in two containers. This
+  is why each source gets its own cache dir (`runsDirFor`), why `lib/blob.ts`
+  scopes its in-flight dedupe keys by container, and why `unstable_cache` keys
+  in `lib/overview.ts` and `lib/trends.ts` all carry `source.id`. Drop any one of
+  those and one source starts serving another's data for a colliding id — with no
+  error.
+- **A run whose id is not date-shaped is invisible to the windowed views.**
+  `getRunListing`, `loadRecentRunsInner`, `getOverview` and
+  `listRunIdsInWindow` filter on `parseRunIdDate`, so such runs surface only in
+  the ad-hoc section. A new source's page therefore needs its OWN
+  `getAdhocRunListing` section, or ad-hoc uploads to that container land
+  nowhere reachable.
+- **Local mode is per-source too.** `listRunIds` resolves
+  `runsDirFor(RUNS_DIR, source)` when `EVALBOARD_LOCAL_RUNS_DIR` is set, so
+  `/scribe` reads `<local>-scribe`. Listing off the bare local dir instead —
+  which is what shipped first — returns the *default* source's ids for every
+  source while the readers resolve under the sibling, so the listing and the
+  reads disagree about which container they describe.
+  `lib/__tests__/source-isolation.test.ts` pins both halves; it's the only test
+  that exercises the reader layer, where the invariant above actually lives.
+
 ## Conventions
 
-- `/api/file?run=<id>&path=<relpath>` serves `.flow`, `.uipx`, etc. with
-  path-traversal guard (`resolveSafePath`).
-- `/api/download?run=<id>[&task=<id>]` streams a zip of a task folder (with
-  `task`) or the whole run (without). Files are gathered by `collectTaskFiles`
+- `/api/file?run=<id>&path=<relpath>[&src=<source>]` serves `.flow`, `.uipx`,
+  etc. with path-traversal guard (`resolveSafePath`).
+- `/api/download?run=<id>[&task=<id>][&src=<source>]` streams a zip of a task
+  folder (with `task`) or the whole run (without). Files are gathered by `collectTaskFiles`
   / `collectRunFiles`, which reuse the `walkArtifacts` noise filter, and zipped
   by `lib/zip.ts` (a dependency-free DEFLATE writer).
 - Pass rows render green (`bg-green-50 text-green-700`), failures render red

evalboard/app/_lib/__tests__/source-param.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, test } from "vitest";
+import { SCRIBE_SOURCE, SKILLS_SOURCE } from "@/lib/sources";
+import { scalarParam, withSource } from "../source-param";
+
+describe("scalarParam", () => {
+    test("passes a scalar through and unwraps a repeated param", () => {
+        expect(scalarParam("scribe")).toBe("scribe");
+        expect(scalarParam(["scribe", "skills"])).toBe("scribe");
+    });
+
+    test("absent stays absent (sourceById then picks the default)", () => {
+        expect(scalarParam(undefined)).toBeUndefined();
+        expect(scalarParam([])).toBeUndefined();
+    });
+});
+
+describe("withSource", () => {
+    test("omits the param for the default source so old URLs are unchanged", () => {
+        expect(withSource("/runs/r1", SKILLS_SOURCE.id)).toBe("/runs/r1");
+        expect(withSource("/runs/r1", undefined)).toBe("/runs/r1");
+        expect(withSource("/runs/r1?r=2", SKILLS_SOURCE.id)).toBe("/runs/r1?r=2");
+    });
+
+    test("appends for a non-default source, respecting an existing query", () => {
+        expect(withSource("/runs/r1", SCRIBE_SOURCE.id)).toBe(
+            "/runs/r1?src=scribe",
+        );
+        expect(withSource("/runs/r1/t?r=2", SCRIBE_SOURCE.id)).toBe(
+            "/runs/r1/t?r=2&src=scribe",
+        );
+    });
+
+    test("keeps the param before a fragment, not inside it", () => {
+        // "#section?src=scribe" is fragment TEXT to a browser, not a query — the
+        // link would silently fall back to the default source and render a
+        // different container's run under the same id.
+        expect(withSource("/runs/r1#section", SCRIBE_SOURCE.id)).toBe(
+            "/runs/r1?src=scribe#section",
+        );
+        expect(withSource("/runs/r1?r=2#section", SCRIBE_SOURCE.id)).toBe(
+            "/runs/r1?r=2&src=scribe#section",
+        );
+        // The default source still returns the href byte-identical.
+        expect(withSource("/runs/r1#section", SKILLS_SOURCE.id)).toBe(
+            "/runs/r1#section",
+        );
+    });
+});

evalboard/app/_lib/source-param.ts

@@ -0,0 +1,35 @@
+import { DEFAULT_SOURCE } from "@/lib/sources";
+
+// Every run-scoped surface carries its source in this query param. Run ids are
+// only unique WITHIN a blob container (both suites name runs
+// `YYYY-MM-DD_HH-MM-SS`), so a link that drops it doesn't 404 — it silently
+// renders a DIFFERENT run's data under the default source.
+export const SRC_PARAM = "src";
+
+/** Normalize a repeated query param to the scalar the readers expect. */
+export function scalarParam(
+    raw: string | string[] | undefined,
+): string | undefined {
+    return Array.isArray(raw) ? raw[0] : raw;
+}
+
+/**
+ * Append the source token to an internal href.
+ *
+ * The default source is omitted so every pre-existing URL stays byte-identical
+ * (and shareable links don't grow a param that means "the default").
+ */
+export function withSource(href: string, sourceId?: string | null): string {
+    if (!sourceId || sourceId === DEFAULT_SOURCE.id) return href;
+    // Split the fragment off first: appending to "/runs/r1#section" would yield
+    // "#section?src=…", which a browser reads as fragment TEXT, not a query —
+    // so the link would silently fall back to the default source and render a
+    // different container's run. Latent today (no caller passes a fragment) but
+    // the failure is invisible, which is exactly the class of bug the source
+    // param exists to close.
+    const hash = href.indexOf("#");
+    const base = hash === -1 ? href : href.slice(0, hash);
+    const frag = hash === -1 ? "" : href.slice(hash);
+    const sep = base.includes("?") ? "&" : "?";
+    return `${base}${sep}${SRC_PARAM}=${encodeURIComponent(sourceId)}${frag}`;
+}

evalboard/app/api/download/route.ts

@@ -1,26 +1,29 @@
 import { promises as fs } from "node:fs";
 import { NextResponse } from "next/server";
 import { collectRunFiles, collectTaskFiles } from "@/lib/runs";
+import { sourceById } from "@/lib/sources";
 import { createZip, type ZipEntry } from "@/lib/zip";
 
 export const dynamic = "force-dynamic";
 
 // Bundle a task folder, or a whole run, into a zip download.
 //   ?run=<id>&task=<id>  → just that task's folder (default/<taskId>/)
 //   ?run=<id>            → the entire run folder (run.json + every task dir)
-// minus the usual scaffolding noise. In blob mode the collect* helpers fetch
-// the needed blobs first, so this mirrors what the page would load.
+// minus the usual scaffolding noise, from the container named by ?src (the
+// skills nightly when absent). In blob mode the collect* helpers fetch the
+// needed blobs first, so this mirrors what the page would load.
 export async function GET(req: Request) {
     const url = new URL(req.url);
     const runId = url.searchParams.get("run");
     const taskId = url.searchParams.get("task");
+    const source = sourceById(url.searchParams.get("src"));
     if (!runId) {
         return new NextResponse("missing run", { status: 400 });
     }
 
     const files = taskId
-        ? await collectTaskFiles(runId, taskId)
-        : await collectRunFiles(runId);
+        ? await collectTaskFiles(runId, taskId, source)
+        : await collectRunFiles(runId, source);
     if (!files) {
         return new NextResponse("not found", { status: 404 });
     }

evalboard/app/api/file/route.ts

@@ -2,18 +2,23 @@ import { promises as fs } from "node:fs";
 import path from "node:path";
 import { NextResponse } from "next/server";
 import { resolveSafePath } from "@/lib/runs";
+import { sourceById } from "@/lib/sources";
 
 export const dynamic = "force-dynamic";
 
 export async function GET(req: Request) {
     const url = new URL(req.url);
     const runId = url.searchParams.get("run");
     const relPath = url.searchParams.get("path");
+    // Artifact paths are relative to the SOURCE's cache dir (readTaskDetail's
+    // artifactPrefix), so without ?src a Scribe artifact resolves under the
+    // skills cache and 404s — or, worse, hits a same-named file there.
+    const source = sourceById(url.searchParams.get("src"));
     if (!runId || !relPath) {
         return new NextResponse("missing run or path", { status: 400 });
     }
 
-    const abs = await resolveSafePath(runId, relPath);
+    const abs = await resolveSafePath(runId, relPath, source);
     if (!abs) {
         return new NextResponse("forbidden", { status: 403 });
     }

evalboard/app/api/refresh/__tests__/route.test.ts

@@ -9,6 +9,7 @@ import {
     test,
     vi,
 } from "vitest";
+import { SCRIBE_SOURCE, runsDirFor } from "@/lib/sources";
 
 // RUNS_DIR and LOCAL_RUNS_DIR are module-level consts read from env at import
 // time, so each scenario sets env, resets the module registry, then imports a
@@ -18,12 +19,14 @@ async function loadPost() {
     return (await import("../route")).POST;
 }
 
-function post(runId?: string): Request {
+function post(runId?: string, src?: string): Request {
     const base = "http://test/api/refresh";
     const url =
         runId === undefined
             ? base
-            : `${base}?run=${encodeURIComponent(runId)}`;
+            : `${base}?run=${encodeURIComponent(runId)}${
+                  src ? `&src=${encodeURIComponent(src)}` : ""
+              }`;
     return new Request(url, { method: "POST" });
 }
 
@@ -113,5 +116,56 @@ describe("POST /api/refresh", () => {
             expect(res.status).toBe(204);
             await expect(fs.access(runDir)).rejects.toThrow();
         });
+
+        // Run ids collide across sources (both suites name runs
+        // YYYY-MM-DD_HH-MM-SS), so the source decides WHICH cached copy is
+        // evicted. Getting this wrong deletes a run nobody asked about and
+        // leaves the requested one stale forever.
+        describe("per-source cache dir", () => {
+            const RUN = "2026-06-01_04-04-22";
+            let scribeCache: string;
+            let skillsRun: string;
+            let scribeRun: string;
+
+            beforeEach(async () => {
+                scribeCache = runsDirFor(cache, SCRIBE_SOURCE);
+                skillsRun = path.join(cache, RUN);
+                scribeRun = path.join(scribeCache, RUN);
+                for (const d of [skillsRun, scribeRun]) {
+                    await fs.mkdir(d, { recursive: true });
+                    await fs.writeFile(path.join(d, "meta.json"), "{}\n");
+                }
+            });
+            afterEach(async () => {
+                await fs.rm(scribeCache, { recursive: true, force: true });
+            });
+
+            test("src=scribe evicts the scribe copy and leaves skills alone", async () => {
+                const POST = await loadPost();
+                const res = await POST(post(RUN, "scribe"));
+
+                expect(res.status).toBe(204);
+                await expect(fs.access(scribeRun)).rejects.toThrow();
+                await expect(fs.access(skillsRun)).resolves.toBeUndefined();
+            });
+
+            test("no src evicts the skills copy and leaves scribe alone", async () => {
+                const POST = await loadPost();
+                const res = await POST(post(RUN));
+
+                expect(res.status).toBe(204);
+                await expect(fs.access(skillsRun)).rejects.toThrow();
+                await expect(fs.access(scribeRun)).resolves.toBeUndefined();
+            });
+
+            test("unknown src falls back to the default source", async () => {
+                const POST = await loadPost();
+                const res = await POST(post(RUN, "nope"));
+
+                expect(res.status).toBe(204);
+                await expect(fs.access(skillsRun)).rejects.toThrow();
+                await expect(fs.access(scribeRun)).resolves.toBeUndefined();
+            });
+        });
     });
 });

evalboard/app/api/refresh/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from "next/server";
 import { LOCAL_RUNS_DIR } from "@/lib/blob";
 import { RUNS_DIR, clearRunCacheDir } from "@/lib/runs";
+import { runsDirFor, sourceById } from "@/lib/sources";
 
 export const dynamic = "force-dynamic";
 
@@ -11,7 +12,7 @@ export const dynamic = "force-dynamic";
 // stay stale forever. This is the manual escape hatch. Single-run only by
 // design — orphan cleanup for deleted/renamed runs is out of scope.
 //
-//   POST /api/refresh?run=<run-id>
+//   POST /api/refresh?run=<run-id>[&src=<source-id>]
 export async function POST(req: Request) {
     // Local mode points RUNS_DIR at the real coder_eval runs dir (the source of
     // truth, not a cache); deleting from it would destroy run data.
@@ -21,14 +22,19 @@ export async function POST(req: Request) {
             { status: 400 },
         );
     }
-    const runId = new URL(req.url).searchParams.get("run");
+    const params = new URL(req.url).searchParams;
+    const runId = params.get("run");
     if (!runId) {
         return NextResponse.json({ error: "missing run" }, { status: 400 });
     }
+    // Each source caches into its own subtree, and a run id can exist in more
+    // than one; clearing the base dir would evict a DIFFERENT run and leave the
+    // requested one stale forever.
+    const source = sourceById(params.get("src"));
     // clearRunCacheDir is the single id validator (rejects separators, `.`,
     // `..`); a false return means the id was unsafe, never "not found" — an
     // uncached run deletes to a harmless no-op.
-    if (!(await clearRunCacheDir(RUNS_DIR, runId))) {
+    if (!(await clearRunCacheDir(runsDirFor(RUNS_DIR, source), runId))) {
         return NextResponse.json({ error: "invalid run" }, { status: 400 });
     }
     // Re-download is lazy on next render. The run page is force-dynamic and