Allow a separate model for testing agents

autonomous_agent_demo.py

@@ -72,6 +72,9 @@ def parse_args() -> argparse.Namespace:
   # Configure testing agent ratio (2 testing agents per coding agent)
   python autonomous_agent_demo.py --project-dir my-app --testing-ratio 2
 
+  # Use Opus for coding agents and Sonnet for testing agents
+  python autonomous_agent_demo.py --project-dir my-app --model claude-opus-4-7 --testing-model claude-sonnet-4-6
+
   # Disable testing agents (similar to YOLO but with verification)
   python autonomous_agent_demo.py --project-dir my-app --testing-ratio 0
 
@@ -102,6 +105,13 @@ def parse_args() -> argparse.Namespace:
         help=f"Claude model to use (default: {DEFAULT_MODEL})",
     )
 
+    parser.add_argument(
+        "--testing-model",
+        type=str,
+        default=None,
+        help="Claude model to use for regression testing agents (default: same as --model)",
+    )
+
     parser.add_argument(
         "--yolo",
         action="store_true",
@@ -332,6 +342,7 @@ def main() -> None:
                     testing_agent_ratio=args.testing_ratio,
                     testing_batch_size=args.testing_batch_size,
                     batch_size=args.batch_size,
+                    testing_model=args.testing_model,
                 )
             )
     except KeyboardInterrupt:

parallel_orchestrator.py

@@ -155,6 +155,7 @@ def __init__(
         testing_agent_ratio: int = 1,
         testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE,
         batch_size: int = 3,
+        testing_model: str | None = None,
         on_output: Callable[[int, str], None] | None = None,
         on_status: Callable[[int, str], None] | None = None,
     ):
@@ -164,18 +165,21 @@ def __init__(
             project_dir: Path to the project directory
             max_concurrency: Maximum number of concurrent coding agents (1-5).
                 Also caps testing agents at the same limit.
-            model: Claude model to use (or None for default)
+            model: Claude model to use for coding/initializer agents (or None for default)
             yolo_mode: Whether to run in YOLO mode (skip testing agents entirely)
             testing_agent_ratio: Number of regression testing agents to maintain (0-3).
                 0 = disabled, 1-3 = maintain that many testing agents running independently.
             testing_batch_size: Number of features to include per testing session (1-15).
                 Each testing agent receives this many features to regression test.
+            testing_model: Claude model to use for testing agents
+                (or None to use the same model as coding agents)
             on_output: Callback for agent output (feature_id, line)
             on_status: Callback for agent status changes (feature_id, status)
         """
         self.project_dir = project_dir
         self.max_concurrency = min(max(max_concurrency, 1), MAX_PARALLEL_AGENTS)
         self.model = model
+        self.testing_model = testing_model or model
         self.yolo_mode = yolo_mode
         self.testing_agent_ratio = min(max(testing_agent_ratio, 0), 3)  # Clamp 0-3
         self.testing_batch_size = min(max(testing_batch_size, 1), 15)  # Clamp 1-15
@@ -1008,8 +1012,8 @@ def _spawn_testing_agent(self) -> tuple[bool, str]:
                 "--agent-type", "testing",
                 "--testing-feature-ids", batch_str,
             ]
-            if self.model:
-                cmd.extend(["--model", self.model])
+            if self.testing_model:
+                cmd.extend(["--model", self.testing_model])
 
             try:
                 # CREATE_NO_WINDOW on Windows prevents console window pop-ups
@@ -1721,6 +1725,7 @@ async def run_parallel_orchestrator(
     testing_agent_ratio: int = 1,
     testing_batch_size: int = DEFAULT_TESTING_BATCH_SIZE,
     batch_size: int = 3,
+    testing_model: str | None = None,
 ) -> None:
     """Run the unified orchestrator.
 
@@ -1732,6 +1737,7 @@ async def run_parallel_orchestrator(
         testing_agent_ratio: Number of regression agents to maintain (0-3)
         testing_batch_size: Number of features per testing batch (1-5)
         batch_size: Max features per coding agent batch (1-3)
+        testing_model: Claude model for testing agents (None = same as model)
     """
     print(f"[ORCHESTRATOR] run_parallel_orchestrator called with max_concurrency={max_concurrency}", flush=True)
     orchestrator = ParallelOrchestrator(
@@ -1742,6 +1748,7 @@ async def run_parallel_orchestrator(
         testing_agent_ratio=testing_agent_ratio,
         testing_batch_size=testing_batch_size,
         batch_size=batch_size,
+        testing_model=testing_model,
     )
 
     # Set up cleanup to run on exit (handles normal exit, exceptions)
@@ -1807,7 +1814,13 @@ def main():
         "--model",
         type=str,
         default=DEFAULT_MODEL,
-        help=f"Claude model to use (default: {DEFAULT_MODEL})",
+        help=f"Claude model to use for coding agents (default: {DEFAULT_MODEL})",
+    )
+    parser.add_argument(
+        "--testing-model",
+        type=str,
+        default=None,
+        help="Claude model to use for testing agents (default: same as --model)",
     )
     parser.add_argument(
         "--yolo",
@@ -1861,6 +1874,7 @@ def main():
             testing_agent_ratio=args.testing_agent_ratio,
             testing_batch_size=args.testing_batch_size,
             batch_size=args.batch_size,
+            testing_model=args.testing_model,
         ))
     except KeyboardInterrupt:
         print("\n\nInterrupted by user", flush=True)

registry.py

@@ -704,6 +704,25 @@ def get_effort_setting() -> EffortLevel:
     return DEFAULT_EFFORT
 
 
+def get_testing_model_setting() -> str | None:
+    """
+    Read the global testing-agent model override.
+
+    Returns None (meaning "use the same model as coding agents") when unset,
+    when the stored value is not a known Claude model, or when an alternative
+    API provider is active (those providers pin every model tier to a single
+    ``api_model``, so a separate testing model would have no effect).
+    """
+    if (get_setting("api_provider", "claude") or "claude") != "claude":
+        return None
+    value = get_setting("testing_model")
+    if value in LEGACY_MODEL_MAP:
+        value = LEGACY_MODEL_MAP[value]
+    if value in VALID_MODELS:
+        return value
+    return None
+
+
 def set_setting(key: str, value: str) -> None:
     """
     Set a setting value (creates or updates).
@@ -747,7 +766,7 @@ def get_all_settings() -> dict[str, str]:
 
             # Auto-migrate legacy model IDs
             migrated = False
-            for key in ("model", "api_model"):
+            for key in ("model", "api_model", "testing_model"):
                 old_id = result.get(key)
                 if old_id and old_id in LEGACY_MODEL_MAP:
                     new_id = LEGACY_MODEL_MAP[old_id]

server/routers/agent.py

@@ -110,6 +110,7 @@ async def start_agent(
     testing_batch_size = default_testing_batch_size
 
     # Always run headless - the embedded browser view panel replaces desktop windows
+    # testing_model=None lets the process manager fall back to global settings
     success, message = await manager.start(
         yolo_mode=yolo_mode,
         model=model,
@@ -118,6 +119,7 @@ async def start_agent(
         playwright_headless=True,
         batch_size=batch_size,
         testing_batch_size=testing_batch_size,
+        testing_model=request.testing_model,
     )
 
     # Notify scheduler of manual start (to prevent auto-stop during scheduled window)

server/routers/settings.py

@@ -28,6 +28,7 @@
     get_all_settings,
     get_effort_setting,
     get_setting,
+    get_testing_model_setting,
     set_setting,
 )
 
@@ -111,6 +112,7 @@ async def get_settings():
     return SettingsResponse(
         yolo_mode=_parse_yolo_mode(all_settings.get("yolo_mode")),
         model=all_settings.get("model", DEFAULT_MODEL),
+        testing_model=get_testing_model_setting(),
         glm_mode=glm_mode,
         ollama_mode=ollama_mode,
         testing_agent_ratio=_parse_int(all_settings.get("testing_agent_ratio"), 1),
@@ -134,6 +136,9 @@ async def update_settings(update: SettingsUpdate):
     if update.model is not None:
         set_setting("model", update.model)
 
+    if update.testing_model is not None:
+        set_setting("testing_model", update.testing_model)
+
     if update.testing_agent_ratio is not None:
         set_setting("testing_agent_ratio", str(update.testing_agent_ratio))
 
@@ -183,6 +188,7 @@ async def update_settings(update: SettingsUpdate):
     return SettingsResponse(
         yolo_mode=_parse_yolo_mode(all_settings.get("yolo_mode")),
         model=all_settings.get("model", DEFAULT_MODEL),
+        testing_model=get_testing_model_setting(),
         glm_mode=glm_mode,
         ollama_mode=ollama_mode,
         testing_agent_ratio=_parse_int(all_settings.get("testing_agent_ratio"), 1),

server/schemas.py

@@ -221,6 +221,7 @@ class AgentStartRequest(BaseModel):
     """Request schema for starting the agent."""
     yolo_mode: bool | None = None  # None means use global settings
     model: str | None = None  # None means use global settings
+    testing_model: str | None = None  # Model for testing agents; None means use global settings
     parallel_mode: bool | None = None  # DEPRECATED: Use max_concurrency instead
     max_concurrency: int | None = None  # Max concurrent coding agents (1-5)
     testing_agent_ratio: int | None = None  # Regression testing agents (0-3)
@@ -236,6 +237,17 @@ def validate_model(cls, v: str | None) -> str | None:
                 raise ValueError(f"Invalid model. Must be one of: {VALID_MODELS}")
         return v
 
+    @field_validator('testing_model')
+    @classmethod
+    def validate_testing_model(cls, v: str | None) -> str | None:
+        """Validate testing model is in the allowed list (Claude) or allow any model for alternative providers."""
+        if v is not None and v not in VALID_MODELS:
+            from registry import get_all_settings
+            settings = get_all_settings()
+            if settings.get("api_provider", "claude") == "claude":
+                raise ValueError(f"Invalid testing model. Must be one of: {VALID_MODELS}")
+        return v
+
     @field_validator('max_concurrency')
     @classmethod
     def validate_concurrency(cls, v: int | None) -> int | None:
@@ -478,6 +490,7 @@ class SettingsResponse(BaseModel):
     """Response schema for global settings."""
     yolo_mode: bool = False
     model: str = DEFAULT_MODEL
+    testing_model: str | None = None  # None means testing agents use `model`
     glm_mode: bool = False  # True when api_provider is "glm"
     ollama_mode: bool = False  # True when api_provider is "ollama"
     testing_agent_ratio: int = 1  # Regression testing agents (0-3)
@@ -501,6 +514,7 @@ class SettingsUpdate(BaseModel):
     """Request schema for updating global settings."""
     yolo_mode: bool | None = None
     model: str | None = None
+    testing_model: str | None = None  # "" clears the override (use coding model)
     testing_agent_ratio: int | None = None  # 0-3
     playwright_headless: bool | None = None
     batch_size: int | None = None  # Features per agent batch (1-15)
@@ -536,6 +550,16 @@ def validate_model(cls, v: str | None, info) -> str | None:
                 raise ValueError(f"Invalid model. Must be one of: {VALID_MODELS}")
         return v
 
+    @field_validator('testing_model')
+    @classmethod
+    def validate_testing_model(cls, v: str | None) -> str | None:
+        if v is not None and v != "":
+            if v in LEGACY_MODEL_MAP:
+                v = LEGACY_MODEL_MAP[v]
+            if v not in VALID_MODELS:
+                raise ValueError(f"Invalid testing model. Must be one of: {VALID_MODELS}")
+        return v
+
     @field_validator('testing_agent_ratio')
     @classmethod
     def validate_testing_ratio(cls, v: int | None) -> int | None:

server/services/process_manager.py

@@ -377,6 +377,7 @@ async def start(
         batch_size: int = 3,
         testing_batch_size: int = 3,
         auto_improve: bool = False,
+        testing_model: str | None = None,
     ) -> tuple[bool, str]:
         """
         Start the agent as a subprocess.
@@ -392,6 +393,9 @@ async def start(
                 execution (concurrency=1, testing_ratio=0) and passes
                 --auto-improve to the subprocess so it runs exactly one
                 improvement session and exits.
+            testing_model: Model for regression testing agents. None falls back
+                to the global ``testing_model`` setting, and if that is unset
+                testing agents use the same model as coding agents.
 
         Returns:
             Tuple of (success, message)
@@ -441,6 +445,13 @@ async def start(
         if model:
             cmd.extend(["--model", model])
 
+        # Add --testing-model flag when testing agents should use a different model
+        if testing_model is None:
+            from registry import get_testing_model_setting
+            testing_model = get_testing_model_setting()
+        if testing_model and testing_model != model:
+            cmd.extend(["--testing-model", testing_model])
+
         # Add --yolo flag if YOLO mode is enabled
         if yolo_mode:
             cmd.append("--yolo")