feat(portal): specialize inline Observation authoring for New Run

apps/portal/src/components/CreateCriterionDialog.stories.tsx

@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { useState } from "react";
+import type { Meta, StoryObj } from "@storybook/react-vite";
+import { expect, fn, userEvent, waitFor, within } from "storybook/test";
+import { http, HttpResponse } from "msw";
+import { setSelectedProjectIdHolder } from "@/lib/project-scope";
+import { CreateCriterionDialog } from "./CreateCriterionDialog";
+
+const meta = {
+  title: "Components/CreateCriterionDialog",
+  component: CreateCriterionDialog,
+  tags: ["ai-generated", "needs-work"],
+  args: { open: true, onOpenChange: fn(), onCreated: fn() },
+  decorators: [(Story) => {
+    setSelectedProjectIdHolder("demo-project");
+    return <Story />;
+  }],
+  render: (args) => {
+    const [open, setOpen] = useState(args.open);
+    return <CreateCriterionDialog {...args} open={open} onOpenChange={setOpen} />;
+  },
+  parameters: {
+    msw: {
+      handlers: [
+        http.get("*/api/v1/criteria", () => HttpResponse.json([])),
+        http.post("*/api/v1/criteria/generate-prompt", () => HttpResponse.json({
+          prompt: "Record whether the codebase contains unit tests.",
+          suggestedId: "has_unit_tests",
+          suggestedParents: [],
+          suggestedChildren: [],
+          suggestedSubject: "run",
+        })),
+        http.post("*/api/v1/criteria", () => HttpResponse.json({
+          id: "has_unit_tests", prompt: "Record whether the codebase contains unit tests.", createdAt: "",
+        })),
+      ],
+    },
+  },
+} satisfies Meta<typeof CreateCriterionDialog>;
+
+export default meta;
+type Story = StoryObj<typeof meta>;
+
+export const GenericCriteria: Story = {
+  play: async ({ canvasElement, args }) => {
+    const page = within(canvasElement.ownerDocument.body);
+    await waitFor(() => expect(page.getByRole("heading", { name: "New Criteria" })).toBeVisible());
+    await expect(page.getByLabelText("Criteria kind")).toBeVisible();
+    await userEvent.type(page.getByLabelText(/What behavior do you want to track/), "Has unit tests");
+    await userEvent.click(page.getByRole("button", { name: "Continue" }));
+    await expect(await page.findByText("Criteria Prompt (AI Generated)")).toBeVisible();
+    await expect(page.getByText("Gate compatibility")).toBeVisible();
+    await userEvent.click(page.getByRole("button", { name: "Create" }));
+    await waitFor(() => expect(args.onCreated).toHaveBeenCalledWith("has_unit_tests"));
+  },
+};
+
+export const ObservationCreation: Story = {
+  args: { mode: "observation" },
+  play: async ({ canvasElement, args }) => {
+    const page = within(canvasElement.ownerDocument.body);
+    await waitFor(() => expect(page.getByRole("heading", { name: "New Observation" })).toBeVisible());
+    await expect(page.queryByLabelText("Criteria kind")).not.toBeInTheDocument();
+    await expect(page.getByLabelText("Evaluation subject")).toHaveTextContent("Whole run");
+    await expect(page.getByText("Observation dimension")).toBeVisible();
+    await userEvent.type(page.getByLabelText(/What behavior do you want to observe/), "Has unit tests");
+    await userEvent.click(page.getByRole("button", { name: "Continue" }));
+    await expect(await page.findByText("Observation Prompt (AI Generated)")).toBeVisible();
+    await expect(page.getAllByPlaceholderText("Search observations…")).toHaveLength(2);
+    await expect(page.queryByText(/No gate-compatible criteria/)).not.toBeInTheDocument();
+    await userEvent.click(page.getByRole("button", { name: "Create Observation" }));
+    await waitFor(() => expect(args.onCreated).toHaveBeenCalledWith("has_unit_tests"));
+  },
+};
+
+export const LegacyObservationDefault: Story = {
+  args: { defaultKind: "observation" },
+  play: async ({ canvasElement }) => {
+    const page = within(canvasElement.ownerDocument.body);
+    await waitFor(() => expect(page.getByRole("heading", { name: "New Criteria" })).toBeVisible());
+    await expect(page.getByLabelText("Criteria kind")).toHaveTextContent("Observation");
+    await expect(page.getByLabelText("Evaluation subject")).toBeVisible();
+  },
+};

apps/portal/src/components/CreateCriterionDialog.test.tsx

@@ -0,0 +1,139 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+// @vitest-environment happy-dom
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { ComponentProps } from "react";
+import { CreateCriterionDialog } from "./CreateCriterionDialog";
+import { api } from "@/lib/api";
+
+vi.mock("@/lib/api", () => ({
+  api: {
+    listCriteria: vi.fn(),
+    generateCriteriaPrompt: vi.fn(),
+    createCriterion: vi.fn(),
+  },
+}));
+
+beforeEach(() => {
+  vi.mocked(api.listCriteria).mockResolvedValue([]);
+  vi.mocked(api.generateCriteriaPrompt).mockResolvedValue({
+    prompt: "Record whether the codebase contains unit tests.",
+    suggestedId: "ai_suggested_id",
+    suggestedParents: [],
+    suggestedChildren: [],
+  });
+  vi.mocked(api.createCriterion).mockImplementation(async (input) => ({
+    ...input,
+    createdAt: "",
+  }));
+});
+
+afterEach(() => {
+  cleanup();
+  vi.resetAllMocks();
+});
+
+function renderDialog(props: Partial<ComponentProps<typeof CreateCriterionDialog>> = {}) {
+  const queryClient = new QueryClient({
+    defaultOptions: {
+      queries: { retry: false },
+      mutations: { retry: false },
+    },
+  });
+  const onCreated = vi.fn();
+  const onOpenChange = vi.fn();
+
+  render(
+    <QueryClientProvider client={queryClient}>
+      <CreateCriterionDialog
+        open
+        onOpenChange={onOpenChange}
+        onCreated={onCreated}
+        {...props}
+      />
+    </QueryClientProvider>,
+  );
+
+  return { onCreated, onOpenChange };
+}
+
+describe("CreateCriterionDialog modes", () => {
+  it("keeps generic Criteria creation and locked gates unchanged by default", async () => {
+    renderDialog({ defaultGates: ["build"], lockedGates: ["build"] });
+
+    expect(screen.getByRole("heading", { name: "New Criteria" })).toBeTruthy();
+    expect(screen.getByLabelText("Criteria kind")).toBeTruthy();
+    expect(screen.getByText(/Gate criteria steer the agent/)).toBeTruthy();
+    expect(screen.getByText(/Criteria ID/)).toBeTruthy();
+
+    fireEvent.change(screen.getByLabelText(/What behavior do you want to track/), {
+      target: { value: "Has unit tests" },
+    });
+    await waitFor(() => expect(screen.getByRole("button", { name: /Continue/ }).hasAttribute("disabled")).toBe(false));
+    fireEvent.click(screen.getByRole("button", { name: /Continue/ }));
+    await screen.findByText("Criteria Prompt (AI Generated)");
+    fireEvent.click(screen.getByRole("button", { name: "Create" }));
+
+    await waitFor(() => expect(vi.mocked(api.createCriterion).mock.calls[0]?.[0]).toEqual({
+      id: "has_unit_tests",
+      prompt: "Record whether the codebase contains unit tests.",
+      dependsOn: undefined,
+      kind: "gate",
+      gates: ["build"],
+    }));
+  });
+
+  it("retains defaultKind without opting into specialized Observation copy", () => {
+    renderDialog({ defaultKind: "observation" });
+
+    expect(screen.getByRole("heading", { name: "New Criteria" })).toBeTruthy();
+    expect(screen.getByLabelText("Criteria kind").textContent).toContain("Observation");
+    expect(screen.getByLabelText("Evaluation subject")).toBeTruthy();
+  });
+
+  it("fixes the Observation kind and creates a run-subject observation without gates", async () => {
+    const { onCreated, onOpenChange } = renderDialog({
+      mode: "observation",
+      defaultKind: "gate",
+      defaultGates: ["build"],
+    });
+
+    expect(screen.getByRole("heading", { name: "New Observation" })).toBeTruthy();
+    expect(screen.getByText(/Observation ID/)).toBeTruthy();
+    expect(screen.getByText("Observation dimension")).toBeTruthy();
+    expect(screen.getByLabelText("Evaluation subject").textContent).toContain("Whole run");
+    expect(screen.queryByLabelText("Criteria kind")).toBeNull();
+    expect(screen.queryByText(/Gate criteria steer the agent/)).toBeNull();
+    expect(screen.queryByText("Gate compatibility")).toBeNull();
+
+    fireEvent.change(screen.getByLabelText(/What behavior do you want to observe/), {
+      target: { value: "Has unit tests" },
+    });
+    await waitFor(() => expect(screen.getByRole("button", { name: /Continue/ }).hasAttribute("disabled")).toBe(false));
+    fireEvent.click(screen.getByRole("button", { name: /Continue/ }));
+
+    await screen.findByText("Observation Prompt (AI Generated)");
+    expect(screen.getByRole("heading", { name: "Review & Create Observation" })).toBeTruthy();
+    expect(screen.getAllByPlaceholderText("Search observations…")).toHaveLength(2);
+    expect(screen.queryByText(/Criteria Prompt/)).toBeNull();
+    expect(screen.queryByText(/observation criteria/)).toBeNull();
+
+    fireEvent.click(screen.getByRole("button", { name: "Create Observation" }));
+
+    await waitFor(() => {
+      expect(vi.mocked(api.createCriterion).mock.calls[0]?.[0]).toEqual({
+        dependsOn: undefined,
+        id: "has_unit_tests",
+        prompt: "Record whether the codebase contains unit tests.",
+        kind: "observation",
+        subject: "run",
+      });
+      expect(onCreated).toHaveBeenCalledWith("has_unit_tests");
+      expect(onOpenChange).toHaveBeenCalledWith(false);
+    });
+    expect(api.generateCriteriaPrompt).toHaveBeenCalledWith("Has unit tests", undefined, undefined);
+  });
+});

apps/portal/src/components/CreateCriterionDialog.tsx

@@ -28,13 +28,25 @@ interface CreateCriterionDialogProps {
   lockedGates?: GateId[];
   /** Pre-populated criteria kind (defaults to gate) */
   defaultKind?: CriterionKind;
+  /** Specialized creation experience for inline observation entry points. */
+  mode?: "criteria" | "observation";
 }
 
-export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGates, lockedGates, defaultKind }: CreateCriterionDialogProps) {
+export function CreateCriterionDialog({
+  open,
+  onOpenChange,
+  onCreated,
+  defaultGates,
+  lockedGates,
+  defaultKind,
+  mode = "criteria",
+}: CreateCriterionDialogProps) {
+  const observationMode = mode === "observation";
   const wizard = useCriteriaWizard({
     initialGates: defaultGates,
     lockedGates,
-    initialKind: defaultKind,
+    initialKind: observationMode ? "observation" : defaultKind,
+    entityLabel: observationMode ? "Observation" : "Criterion",
     onSuccess: (id) => {
       onCreated(id);
       handleOpenChange(false);
@@ -65,14 +77,20 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa
         {wizard.step === 1 ? (
           <>
             <DialogHeader>
-              <DialogTitle>New Criteria</DialogTitle>
+              <DialogTitle>{observationMode ? "New Observation" : "New Criteria"}</DialogTitle>
               <DialogDescription>
-                Describe the behavior to evaluate
+                {observationMode
+                  ? "Describe the behavior to observe"
+                  : "Describe the behavior to evaluate"}
               </DialogDescription>
             </DialogHeader>
 
             <div className="py-2">
-              <CriteriaWizardStep1 wizard={wizard} idPrefix="dialog-" />
+              <CriteriaWizardStep1
+                wizard={wizard}
+                idPrefix="dialog-"
+                observationMode={observationMode}
+              />
             </div>
 
             <DialogFooter>
@@ -90,14 +108,16 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa
         ) : (
           <>
             <DialogHeader>
-              <DialogTitle>Review & Create</DialogTitle>
+              <DialogTitle>
+                {observationMode ? "Review & Create Observation" : "Review & Create"}
+              </DialogTitle>
               <DialogDescription>
                 {wizard.behavior}
               </DialogDescription>
             </DialogHeader>
 
             <div className="py-2">
-              <CriteriaWizardStep2 wizard={wizard} />
+              <CriteriaWizardStep2 wizard={wizard} observationMode={observationMode} />
             </div>
 
             <DialogFooter className="flex-row justify-between sm:justify-between">
@@ -120,7 +140,7 @@ export function CreateCriterionDialog({ open, onOpenChange, onCreated, defaultGa
                 ) : (
                   <Sparkles className="h-4 w-4" />
                 )}
-                Create
+                {observationMode ? "Create Observation" : "Create"}
               </Button>
             </DialogFooter>
           </>

apps/portal/src/components/CriteriaPicker.stories.tsx

@@ -0,0 +1,57 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { useState } from "react";
+import type { Meta, StoryObj } from "@storybook/react-vite";
+import { expect, userEvent, within } from "storybook/test";
+import { http, HttpResponse } from "msw";
+import { setSelectedProjectIdHolder } from "@/lib/project-scope";
+import { CriteriaPicker } from "./CriteriaPicker";
+
+const meta = {
+  title: "Components/CriteriaPicker",
+  component: CriteriaPicker,
+  tags: ["ai-generated", "needs-work"],
+  args: { selected: [], onChange: () => undefined },
+  decorators: [(Story) => {
+    setSelectedProjectIdHolder("demo-project");
+    return <div className="max-w-xl p-6"><Story /></div>;
+  }],
+  render: (args) => {
+    const [selected, setSelected] = useState<string[]>([]);
+    return <CriteriaPicker {...args} selected={selected} onChange={setSelected} />;
+  },
+  parameters: {
+    msw: {
+      handlers: [http.get("*/api/v1/criteria", () => HttpResponse.json([
+        { id: "gate_requirement", prompt: "The project builds.", kind: "gate", createdAt: "" },
+        { id: "has_tests", prompt: "Record whether tests exist.", kind: "observation", createdAt: "" },
+      ]))],
+    },
+  },
+} satisfies Meta<typeof CriteriaPicker>;
+
+export default meta;
+type Story = StoryObj<typeof meta>;
+
+export const GenericCriteria: Story = {
+  play: async ({ canvas, canvasElement }) => {
+    const input = await canvas.findByPlaceholderText("Type to search criteria…");
+    await userEvent.type(input, "missing");
+    await expect(await within(canvasElement.ownerDocument.body).findByText("No matching criteria")).toBeVisible();
+  },
+};
+
+export const ObservationDependencies: Story = {
+  args: { kind: "observation", entityLabel: "observations" },
+  play: async ({ canvas, canvasElement }) => {
+    const page = within(canvasElement.ownerDocument.body);
+    const input = await canvas.findByPlaceholderText("Search observations…");
+    await userEvent.click(input);
+    await expect(page.queryByRole("button", { name: /gate_requirement/ })).not.toBeInTheDocument();
+    await userEvent.click(await page.findByRole("button", { name: /has_tests/ }));
+    await expect(canvas.getByText("has_tests")).toBeVisible();
+    await userEvent.type(input, "missing");
+    await expect(await page.findByText("No matching observations")).toBeVisible();
+  },
+};

apps/portal/src/components/CriteriaPicker.test.tsx

@@ -9,8 +9,9 @@ import { CriteriaPicker } from "./CriteriaPicker";
 import type { CriteriaDocument } from "@/types";
 
 const CRITERIA: CriteriaDocument[] = [
-  { id: "crit_a", prompt: "a" } as CriteriaDocument,
-  { id: "crit_b", prompt: "b" } as CriteriaDocument,
+  { id: "crit_a", prompt: "a", createdAt: "" },
+  { id: "crit_b", prompt: "b", createdAt: "" },
+  { id: "observation_a", prompt: "record a", kind: "observation", createdAt: "" },
 ];
 
 vi.mock("@/lib/api", () => ({
@@ -60,4 +61,22 @@ describe("CriteriaPicker dropdown", () => {
     fireEvent.click(option);
     expect(selected).toEqual(["crit_a"]);
   });
+
+  it("uses specialized copy without admitting gate criteria to observation suggestions", async () => {
+    const onChange = vi.fn();
+    const qc = new QueryClient({ defaultOptions: { queries: { retry: false } } });
+    render(
+      <QueryClientProvider client={qc}>
+        <CriteriaPicker selected={[]} onChange={onChange} kind="observation" entityLabel="observations" />
+      </QueryClientProvider>,
+    );
+    const input = await screen.findByPlaceholderText("Search observations…");
+    fireEvent.focus(input);
+    expect(screen.queryByRole("button", { name: /crit_a/ })).toBeNull();
+    fireEvent.click(screen.getByRole("button", { name: /observation_a/ }));
+    expect(onChange).toHaveBeenCalledWith(["observation_a"]);
+
+    fireEvent.change(input, { target: { value: "missing" } });
+    expect(screen.getByText("No matching observations")).toBeTruthy();
+  });
 });