From 1e62db8a07a821cd07f8a585039875fd72defe65 Mon Sep 17 00:00:00 2001 From: mohamed zwanski Date: Tue, 27 Oct 2026 19:18:43 +0100 Subject: [PATCH] Allow a separate model for testing agents (#146) Add --testing-model to autonomous_agent_demo.py and parallel_orchestrator.py, a global testing_model setting (Claude provider only), API/schema support, and a Testing Agent Model selector in the settings UI. Co-Authored-By: Claude Opus 5.5 --- autonomous_agent_demo.py | 11 +++ parallel_orchestrator.py | 22 ++++- registry.py | 21 ++++- server/routers/agent.py | 2 + server/routers/settings.py | 6 ++ server/schemas.py | 24 +++++ server/services/process_manager.py | 11 +++ test_testing_model.py | 111 +++++++++++++++++++++++ ui/src/components/SettingsModal.tsx | 38 ++++++++ ui/src/components/views/SettingsView.tsx | 38 ++++++++ ui/src/hooks/useProjects.ts | 1 + ui/src/lib/types.ts | 2 + 12 files changed, 282 insertions(+), 5 deletions(-) create mode 100644 test_testing_model.py diff --git a/autonomous_agent_demo.py b/autonomous_agent_demo.py index 787cdf6e2..f9ee62630 100644 --- a/autonomous_agent_demo.py +++ b/autonomous_agent_demo.py @@ -72,6 +72,9 @@ def parse_args() -> argparse.Namespace: # Configure testing agent ratio (2 testing agents per coding agent) python autonomous_agent_demo.py --project-dir my-app --testing-ratio 2 + # Use Opus for coding agents and Sonnet for testing agents + python autonomous_agent_demo.py --project-dir my-app --model claude-opus-4-7 --testing-model claude-sonnet-4-6 + # Disable testing agents (similar to YOLO but with verification) python autonomous_agent_demo.py --project-dir my-app --testing-ratio 0 @@ -102,6 +105,13 @@ def parse_args() -> argparse.Namespace: help=f"Claude model to use (default: {DEFAULT_MODEL})", ) + parser.add_argument( + "--testing-model", + type=str, + default=None, + help="Claude model to use for regression testing agents (default: same as --model)", + ) + parser.add_argument( "--yolo", action="store_true", @@ -332,6 +342,7 @@ def main() -> None: testing_agent_ratio=args.testing_ratio, testing_batch_size=args.testing_batch_size, batch_size=args.batch_size, + testing_model=args.testing_model, ) ) except KeyboardInterrupt: diff --git a/parallel_orchestrator.py b/parallel_orchestrator.py index e39ef7995..2fe18b31f 100644 --- a/parallel_orchestrator.py +++ b/parallel_orchestrator.py @@ -155,6 +155,7 @@ def __init__( testing_agent_ratio: int = 1, testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE, batch_size: int = 3, + testing_model: str | None = None, on_output: Callable[[int, str], None] | None = None, on_status: Callable[[int, str], None] | None = None, ): @@ -164,18 +165,21 @@ def __init__( project_dir: Path to the project directory max_concurrency: Maximum number of concurrent coding agents (1-5). Also caps testing agents at the same limit. - model: Claude model to use (or None for default) + model: Claude model to use for coding/initializer agents (or None for default) yolo_mode: Whether to run in YOLO mode (skip testing agents entirely) testing_agent_ratio: Number of regression testing agents to maintain (0-3). 0 = disabled, 1-3 = maintain that many testing agents running independently. testing_batch_size: Number of features to include per testing session (1-15). Each testing agent receives this many features to regression test. + testing_model: Claude model to use for testing agents + (or None to use the same model as coding agents) on_output: Callback for agent output (feature_id, line) on_status: Callback for agent status changes (feature_id, status) """ self.project_dir = project_dir self.max_concurrency = min(max(max_concurrency, 1), MAX_PARALLEL_AGENTS) self.model = model + self.testing_model = testing_model or model self.yolo_mode = yolo_mode self.testing_agent_ratio = min(max(testing_agent_ratio, 0), 3) # Clamp 0-3 self.testing_batch_size = min(max(testing_batch_size, 1), 15) # Clamp 1-15 @@ -1008,8 +1012,8 @@ def _spawn_testing_agent(self) -> tuple[bool, str]: "--agent-type", "testing", "--testing-feature-ids", batch_str, ] - if self.model: - cmd.extend(["--model", self.model]) + if self.testing_model: + cmd.extend(["--model", self.testing_model]) try: # CREATE_NO_WINDOW on Windows prevents console window pop-ups @@ -1721,6 +1725,7 @@ async def run_parallel_orchestrator( testing_agent_ratio: int = 1, testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE, batch_size: int = 3, + testing_model: str | None = None, ) -> None: """Run the unified orchestrator. @@ -1732,6 +1737,7 @@ async def run_parallel_orchestrator( testing_agent_ratio: Number of regression agents to maintain (0-3) testing_batch_size: Number of features per testing batch (1-5) batch_size: Max features per coding agent batch (1-3) + testing_model: Claude model for testing agents (None = same as model) """ print(f"[ORCHESTRATOR] run_parallel_orchestrator called with max_concurrency={max_concurrency}", flush=True) orchestrator = ParallelOrchestrator( @@ -1742,6 +1748,7 @@ async def run_parallel_orchestrator( testing_agent_ratio=testing_agent_ratio, testing_batch_size=testing_batch_size, batch_size=batch_size, + testing_model=testing_model, ) # Set up cleanup to run on exit (handles normal exit, exceptions) @@ -1807,7 +1814,13 @@ def main(): "--model", type=str, default=DEFAULT_MODEL, - help=f"Claude model to use (default: {DEFAULT_MODEL})", + help=f"Claude model to use for coding agents (default: {DEFAULT_MODEL})", + ) + parser.add_argument( + "--testing-model", + type=str, + default=None, + help="Claude model to use for testing agents (default: same as --model)", ) parser.add_argument( "--yolo", @@ -1861,6 +1874,7 @@ def main(): testing_agent_ratio=args.testing_agent_ratio, testing_batch_size=args.testing_batch_size, batch_size=args.batch_size, + testing_model=args.testing_model, )) except KeyboardInterrupt: print("\n\nInterrupted by user", flush=True) diff --git a/registry.py b/registry.py index 6255d431d..eb744ba7c 100644 --- a/registry.py +++ b/registry.py @@ -704,6 +704,25 @@ def get_effort_setting() -> EffortLevel: return DEFAULT_EFFORT +def get_testing_model_setting() -> str | None: + """ + Read the global testing-agent model override. + + Returns None (meaning "use the same model as coding agents") when unset, + when the stored value is not a known Claude model, or when an alternative + API provider is active (those providers pin every model tier to a single + ``api_model``, so a separate testing model would have no effect). + """ + if (get_setting("api_provider", "claude") or "claude") != "claude": + return None + value = get_setting("testing_model") + if value in LEGACY_MODEL_MAP: + value = LEGACY_MODEL_MAP[value] + if value in VALID_MODELS: + return value + return None + + def set_setting(key: str, value: str) -> None: """ Set a setting value (creates or updates). @@ -747,7 +766,7 @@ def get_all_settings() -> dict[str, str]: # Auto-migrate legacy model IDs migrated = False - for key in ("model", "api_model"): + for key in ("model", "api_model", "testing_model"): old_id = result.get(key) if old_id and old_id in LEGACY_MODEL_MAP: new_id = LEGACY_MODEL_MAP[old_id] diff --git a/server/routers/agent.py b/server/routers/agent.py index d27d803cd..196a3357a 100644 --- a/server/routers/agent.py +++ b/server/routers/agent.py @@ -110,6 +110,7 @@ async def start_agent( testing_batch_size = default_testing_batch_size # Always run headless - the embedded browser view panel replaces desktop windows + # testing_model=None lets the process manager fall back to global settings success, message = await manager.start( yolo_mode=yolo_mode, model=model, @@ -118,6 +119,7 @@ async def start_agent( playwright_headless=True, batch_size=batch_size, testing_batch_size=testing_batch_size, + testing_model=request.testing_model, ) # Notify scheduler of manual start (to prevent auto-stop during scheduled window) diff --git a/server/routers/settings.py b/server/routers/settings.py index 9ff04acde..ad2155e36 100644 --- a/server/routers/settings.py +++ b/server/routers/settings.py @@ -28,6 +28,7 @@ get_all_settings, get_effort_setting, get_setting, + get_testing_model_setting, set_setting, ) @@ -111,6 +112,7 @@ async def get_settings(): return SettingsResponse( yolo_mode=_parse_yolo_mode(all_settings.get("yolo_mode")), model=all_settings.get("model", DEFAULT_MODEL), + testing_model=get_testing_model_setting(), glm_mode=glm_mode, ollama_mode=ollama_mode, testing_agent_ratio=_parse_int(all_settings.get("testing_agent_ratio"), 1), @@ -134,6 +136,9 @@ async def update_settings(update: SettingsUpdate): if update.model is not None: set_setting("model", update.model) + if update.testing_model is not None: + set_setting("testing_model", update.testing_model) + if update.testing_agent_ratio is not None: set_setting("testing_agent_ratio", str(update.testing_agent_ratio)) @@ -183,6 +188,7 @@ async def update_settings(update: SettingsUpdate): return SettingsResponse( yolo_mode=_parse_yolo_mode(all_settings.get("yolo_mode")), model=all_settings.get("model", DEFAULT_MODEL), + testing_model=get_testing_model_setting(), glm_mode=glm_mode, ollama_mode=ollama_mode, testing_agent_ratio=_parse_int(all_settings.get("testing_agent_ratio"), 1), diff --git a/server/schemas.py b/server/schemas.py index d54f00aa9..dfdb9f82f 100644 --- a/server/schemas.py +++ b/server/schemas.py @@ -221,6 +221,7 @@ class AgentStartRequest(BaseModel): """Request schema for starting the agent.""" yolo_mode: bool | None = None # None means use global settings model: str | None = None # None means use global settings + testing_model: str | None = None # Model for testing agents; None means use global settings parallel_mode: bool | None = None # DEPRECATED: Use max_concurrency instead max_concurrency: int | None = None # Max concurrent coding agents (1-5) testing_agent_ratio: int | None = None # Regression testing agents (0-3) @@ -236,6 +237,17 @@ def validate_model(cls, v: str | None) -> str | None: raise ValueError(f"Invalid model. Must be one of: {VALID_MODELS}") return v + @field_validator('testing_model') + @classmethod + def validate_testing_model(cls, v: str | None) -> str | None: + """Validate testing model is in the allowed list (Claude) or allow any model for alternative providers.""" + if v is not None and v not in VALID_MODELS: + from registry import get_all_settings + settings = get_all_settings() + if settings.get("api_provider", "claude") == "claude": + raise ValueError(f"Invalid testing model. Must be one of: {VALID_MODELS}") + return v + @field_validator('max_concurrency') @classmethod def validate_concurrency(cls, v: int | None) -> int | None: @@ -478,6 +490,7 @@ class SettingsResponse(BaseModel): """Response schema for global settings.""" yolo_mode: bool = False model: str = DEFAULT_MODEL + testing_model: str | None = None # None means testing agents use `model` glm_mode: bool = False # True when api_provider is "glm" ollama_mode: bool = False # True when api_provider is "ollama" testing_agent_ratio: int = 1 # Regression testing agents (0-3) @@ -501,6 +514,7 @@ class SettingsUpdate(BaseModel): """Request schema for updating global settings.""" yolo_mode: bool | None = None model: str | None = None + testing_model: str | None = None # "" clears the override (use coding model) testing_agent_ratio: int | None = None # 0-3 playwright_headless: bool | None = None batch_size: int | None = None # Features per agent batch (1-15) @@ -536,6 +550,16 @@ def validate_model(cls, v: str | None, info) -> str | None: raise ValueError(f"Invalid model. Must be one of: {VALID_MODELS}") return v + @field_validator('testing_model') + @classmethod + def validate_testing_model(cls, v: str | None) -> str | None: + if v is not None and v != "": + if v in LEGACY_MODEL_MAP: + v = LEGACY_MODEL_MAP[v] + if v not in VALID_MODELS: + raise ValueError(f"Invalid testing model. Must be one of: {VALID_MODELS}") + return v + @field_validator('testing_agent_ratio') @classmethod def validate_testing_ratio(cls, v: int | None) -> int | None: diff --git a/server/services/process_manager.py b/server/services/process_manager.py index 0a7b8feea..fc6b39ea1 100644 --- a/server/services/process_manager.py +++ b/server/services/process_manager.py @@ -377,6 +377,7 @@ async def start( batch_size: int = 3, testing_batch_size: int = 3, auto_improve: bool = False, + testing_model: str | None = None, ) -> tuple[bool, str]: """ Start the agent as a subprocess. @@ -392,6 +393,9 @@ async def start( execution (concurrency=1, testing_ratio=0) and passes --auto-improve to the subprocess so it runs exactly one improvement session and exits. + testing_model: Model for regression testing agents. None falls back + to the global ``testing_model`` setting, and if that is unset + testing agents use the same model as coding agents. Returns: Tuple of (success, message) @@ -441,6 +445,13 @@ async def start( if model: cmd.extend(["--model", model]) + # Add --testing-model flag when testing agents should use a different model + if testing_model is None: + from registry import get_testing_model_setting + testing_model = get_testing_model_setting() + if testing_model and testing_model != model: + cmd.extend(["--testing-model", testing_model]) + # Add --yolo flag if YOLO mode is enabled if yolo_mode: cmd.append("--yolo") diff --git a/test_testing_model.py b/test_testing_model.py new file mode 100644 index 000000000..de16f09ba --- /dev/null +++ b/test_testing_model.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +""" +Testing Model Tests +=================== + +Verifies that coding and testing agents can be launched with different +models (issue #146). + +Run with: python -m pytest test_testing_model.py -v +""" + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from parallel_orchestrator import ParallelOrchestrator + + +def _model_arg(cmd: list[str]) -> str | None: + """Return the value following --model in a subprocess command, if any.""" + if "--model" not in cmd: + return None + return cmd[cmd.index("--model") + 1] + + +class TestOrchestratorTestingModel(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.project_dir = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + def _make(self, **kwargs) -> ParallelOrchestrator: + orch = ParallelOrchestrator(project_dir=self.project_dir, max_concurrency=2, **kwargs) + orch._get_test_batch = MagicMock(return_value=[1, 2]) # type: ignore[method-assign] + return orch + + def _spawn_cmds(self, orch: ParallelOrchestrator) -> tuple[list[str], list[str]]: + """Spawn one testing and one coding agent, returning their commands.""" + fake_proc = MagicMock() + fake_proc.pid = 4242 + fake_proc.stdout = iter(()) + with patch("parallel_orchestrator.subprocess.Popen", return_value=fake_proc) as popen, \ + patch("parallel_orchestrator.threading.Thread"): + ok, msg = orch._spawn_testing_agent() + self.assertTrue(ok, msg) + orch._spawn_coding_agent(7) + testing_cmd = popen.call_args_list[0].args[0] + coding_cmd = popen.call_args_list[1].args[0] + return testing_cmd, coding_cmd + + def test_testing_model_defaults_to_coding_model(self): + orch = self._make(model="claude-opus-4-7") + self.assertEqual(orch.testing_model, "claude-opus-4-7") + testing_cmd, coding_cmd = self._spawn_cmds(orch) + self.assertEqual(_model_arg(testing_cmd), "claude-opus-4-7") + self.assertEqual(_model_arg(coding_cmd), "claude-opus-4-7") + + def test_separate_testing_model(self): + orch = self._make(model="claude-opus-4-7", testing_model="claude-sonnet-4-6") + testing_cmd, coding_cmd = self._spawn_cmds(orch) + self.assertEqual(_model_arg(testing_cmd), "claude-sonnet-4-6") + self.assertEqual(_model_arg(coding_cmd), "claude-opus-4-7") + + +class TestTestingModelSetting(unittest.TestCase): + def _resolve(self, settings: dict[str, str]) -> str | None: + import registry + + with patch.object(registry, "get_setting", side_effect=lambda k, d=None: settings.get(k, d)): + return registry.get_testing_model_setting() + + def test_unset_returns_none(self): + self.assertIsNone(self._resolve({})) + + def test_empty_string_means_same_as_coding(self): + self.assertIsNone(self._resolve({"testing_model": ""})) + + def test_valid_model_returned(self): + self.assertEqual(self._resolve({"testing_model": "claude-sonnet-4-6"}), "claude-sonnet-4-6") + + def test_legacy_model_migrated(self): + self.assertEqual(self._resolve({"testing_model": "claude-sonnet-4-5"}), "claude-sonnet-4-6") + + def test_unknown_model_ignored(self): + self.assertIsNone(self._resolve({"testing_model": "not-a-model"})) + + def test_ignored_for_alternative_provider(self): + self.assertIsNone(self._resolve({"testing_model": "claude-sonnet-4-6", "api_provider": "glm"})) + + +class TestSettingsSchemas(unittest.TestCase): + def test_settings_update_accepts_clear_and_valid(self): + from server.schemas import SettingsUpdate + + self.assertEqual(SettingsUpdate(testing_model="").testing_model, "") + self.assertEqual(SettingsUpdate(testing_model="claude-sonnet-4-6").testing_model, "claude-sonnet-4-6") + + def test_settings_update_rejects_unknown_model(self): + from pydantic import ValidationError + + from server.schemas import SettingsUpdate + + with self.assertRaises(ValidationError): + SettingsUpdate(testing_model="gpt-4") + + +if __name__ == "__main__": + unittest.main() diff --git a/ui/src/components/SettingsModal.tsx b/ui/src/components/SettingsModal.tsx index 5605f4e63..671c65858 100644 --- a/ui/src/components/SettingsModal.tsx +++ b/ui/src/components/SettingsModal.tsx @@ -52,6 +52,12 @@ export function SettingsModal({ isOpen, onClose }: SettingsModalProps) { } } + const handleTestingModelChange = (modelId: string) => { + if (!updateSettings.isPending) { + updateSettings.mutate({ testing_model: modelId }) + } + } + const handleTestingRatioChange = (ratio: number) => { if (!updateSettings.isPending) { updateSettings.mutate({ testing_agent_ratio: ratio }) @@ -351,6 +357,11 @@ export function SettingsModal({ isOpen, onClose }: SettingsModalProps) { {/* Model Selection */}
+ {!isAlternativeProvider && ( +

+ Used by the initializer and coding agents +

+ )} {models.length > 0 && (
{models.map((model) => ( @@ -392,6 +403,33 @@ export function SettingsModal({ isOpen, onClose }: SettingsModalProps) { )}
+ {/* Testing Agent Model (Claude only - alternative providers pin a single model) */} + {!isAlternativeProvider && models.length > 0 && ( +
+ +

+ Model used by regression testing agents +

+
+ {[{ id: '', name: 'Same as coding' }, ...models].map((model) => ( + + ))} +
+
+ )} + {/* Reasoning Effort */}
diff --git a/ui/src/components/views/SettingsView.tsx b/ui/src/components/views/SettingsView.tsx index f22ec0344..e15c5d9fd 100644 --- a/ui/src/components/views/SettingsView.tsx +++ b/ui/src/components/views/SettingsView.tsx @@ -50,6 +50,12 @@ export function SettingsView() { } } + const handleTestingModelChange = (modelId: string) => { + if (!updateSettings.isPending) { + updateSettings.mutate({ testing_model: modelId }) + } + } + const handleTestingRatioChange = (ratio: number) => { if (!updateSettings.isPending) { updateSettings.mutate({ testing_agent_ratio: ratio }) @@ -360,6 +366,11 @@ export function SettingsView() { {/* Model Selection */}
+ {!isAlternativeProvider && ( +

+ Used by the initializer and coding agents +

+ )} {models.length > 0 && (
{models.map((model) => ( @@ -400,6 +411,33 @@ export function SettingsView() {
)}
+ + {/* Testing Agent Model (Claude only - alternative providers pin a single model) */} + {!isAlternativeProvider && models.length > 0 && ( +
+ +

+ Model used by regression testing agents +

+
+ {[{ id: '', name: 'Same as coding' }, ...models].map((model) => ( + + ))} +
+
+ )} diff --git a/ui/src/hooks/useProjects.ts b/ui/src/hooks/useProjects.ts index 88da9c0c4..3e7a87c7d 100644 --- a/ui/src/hooks/useProjects.ts +++ b/ui/src/hooks/useProjects.ts @@ -297,6 +297,7 @@ const DEFAULT_MODELS: ModelsResponse = { const DEFAULT_SETTINGS: Settings = { yolo_mode: false, model: 'claude-opus-4-7', + testing_model: null, glm_mode: false, ollama_mode: false, testing_agent_ratio: 1, diff --git a/ui/src/lib/types.ts b/ui/src/lib/types.ts index b2ff61aa2..6b0cdfbb1 100644 --- a/ui/src/lib/types.ts +++ b/ui/src/lib/types.ts @@ -647,6 +647,7 @@ export type EffortLevel = 'low' | 'medium' | 'high' | 'xhigh' | 'max' export interface Settings { yolo_mode: boolean model: string + testing_model: string | null // Model for testing agents (null = same as coding model) glm_mode: boolean ollama_mode: boolean testing_agent_ratio: number // Regression testing agents (0-3) @@ -663,6 +664,7 @@ export interface Settings { export interface SettingsUpdate { yolo_mode?: boolean model?: string + testing_model?: string // '' clears the override testing_agent_ratio?: number batch_size?: number testing_batch_size?: number