Coverage for src/ai_jury/scaffold.py: 100%
190 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-30 06:29 +0000
1"""Scaffold a ``jury.toml`` from agent selections (issue #107).
3Backs the ``jury init`` command: instead of hand-editing TOML, a user (or a
4script) picks agents/rounds/chair and this renders a valid config. The cloud
5agent templates reuse the **secure-by-default** entries from
6:data:`config.DEFAULT_CONFIG` (issue #100) so generated configs are safe; a
7``local`` template targets an OpenAI-compatible server (Ollama by default).
9Pure and deterministic: building the config dict and rendering it to TOML are
10side-effect-free, so they are fully unit-testable; the CLI layer owns prompting,
11availability detection, and writing the file.
12"""
14from __future__ import annotations
16import math
17from urllib.parse import urlsplit
19from .config import AGY_AGENT, DEFAULT_CONFIG, GENERIC_CLI_VENDORS, adapter_key
21#: The one list of model ids the shipped samples name, keyed by the kind of seat
22#: (#870). `jury init` writes the hosted and local ones below; the README, the
23#: site's integration cards and "Build your jury" demo, `examples/jury.toml` and
24#: the docs name the same ids, and ``tests/test_sample_configs.py`` holds every
25#: one of them to this table — so a stale id is changed here and the test lists
26#: each copy that still disagrees. Each was checked against the vendor's public
27#: model list on 2026-09-28 (no API key used): Anthropic's models overview,
28#: OpenAI's models page, Google's Gemini API models page, xAI's models page,
29#: DeepSeek's pricing page, Groq's models page, Moonshot's pricing page,
30#: OpenRouter's public ``/api/v1/models``, Together's serverless model list and
31#: Ollama's library tags.
32SAMPLE_MODELS: dict[str, str] = {
33 "anthropic": "claude-opus-5-5",
34 "openai": "gpt-6-sol",
35 "google": "gemini-3.8-flash",
36 "xai": "grok-4.7",
37 "deepseek": "deepseek-v4-pro",
38 "groq": "llama-3.3-70b-versatile",
39 "moonshot": "kimi-k3",
40 "together": "meta-llama/Llama-3.3-70B-Instruct-Turbo",
41 "openrouter": "anthropic/claude-opus-5.5",
42 "local": "qwen2.5-coder:7b",
43}
45_LOCAL_TEMPLATE = {
46 "name": "qwen",
47 "vendor": "local",
48 "model": SAMPLE_MODELS["local"],
49 "endpoint": "http://localhost:11434/v1",
50}
52# Hosted-API templates (issue #430): no `command`/`endpoint` — see
53# adapters._HostedApiAdapter. `model` is left for the user to fill in (a
54# hardcoded model id here would go stale as vendors deprecate/rename models;
55# `validate_config` already warns when it's missing).
56_ANTHROPIC_API_TEMPLATE = {"name": "claude-api", "vendor": "anthropic-api", "model": ""}
57_OPENAI_API_TEMPLATE = {"name": "codex-api", "vendor": "openai-api", "model": ""}
58_GOOGLE_API_TEMPLATE = {"name": "gemini-api", "vendor": "google-api", "model": ""}
59_OPENROUTER_TEMPLATE = {
60 "name": "openrouter",
61 "vendor": "openai-compatible",
62 "endpoint": "https://openrouter.ai/api/v1",
63 "api_key_env": "OPENROUTER_API_KEY",
64 "model": SAMPLE_MODELS["openrouter"],
65}
66_DEEPSEEK_TEMPLATE = {
67 "name": "deepseek",
68 "vendor": "openai-compatible",
69 "endpoint": "https://api.deepseek.com/v1",
70 "api_key_env": "DEEPSEEK_API_KEY",
71 "model": SAMPLE_MODELS["deepseek"],
72}
73_GROQ_TEMPLATE = {
74 "name": "groq",
75 "vendor": "openai-compatible",
76 "endpoint": "https://api.groq.com/openai/v1",
77 "api_key_env": "GROQ_API_KEY",
78 "model": SAMPLE_MODELS["groq"],
79}
80#: aider with the flags that ask it not to edit (#859). `--read-only` is not an aider
81#: option (its options reference has `--read FILE`), so the template it replaces
82#: wrote a seat aider refuses to start. Ask mode "never make[s] changes",
83#: `--dry-run` stops it applying edits or committing, and the rest turn off its
84#: commits, shell-command suggestions, post-edit lint run, URL fetches and
85#: `.gitignore` edit. `--message` is last: with `prompt_mode = "arg"` the prompt
86#: is appended after it. What no flag controls: aider reads `.aider.conf.yml` and
87#: `.env` from the checkout it runs in, and those can turn on its test, lint or
88#: load commands, which it runs before the message (``--test``/``--lint`` have no
89#: ``--no-`` form, and ``--config``/``--env-file`` add a file rather than replace
90#: the ones it finds). jury adds or checks no sandbox for an arbitrary CLI, so the
91#: seat is written under :data:`UNSANDBOXED_LABEL` and :data:`UNSANDBOXED_HINT`,
92#: and the least-privilege audit still warns about it.
93_GENERIC_CLI_TEMPLATE = {
94 "name": "aider",
95 "vendor": "cli",
96 "command": "aider",
97 "prompt_mode": "arg",
98 "extra_args": [
99 "--chat-mode",
100 "ask",
101 "--dry-run",
102 "--no-auto-commits",
103 "--no-dirty-commits",
104 "--no-suggest-shell-commands",
105 "--no-auto-lint",
106 "--no-detect-urls",
107 "--no-gitignore",
108 "--message",
109 ],
110}
112#: Written above every bring-your-own CLI seat `jury init` scaffolds (#859): jury
113#: adds or checks no sandbox for one, so it runs with whatever its own flags
114#: allow. The same words label the seat on the site and in the docs.
115UNSANDBOXED_LABEL = "unsandboxed \u2014 runs with your permissions"
117#: Written under the label: the CLI runs in the checkout jury starts in and obeys
118#: that checkout's own config (aider's `.aider.conf.yml`/`.env`, Cursor's
119#: `.cursor/` hooks), which no flag in the seat turns off.
120UNSANDBOXED_HINT = (
121 "# It reads its own config from the checkout it runs in, which can run commands;",
122 "# seat it only on checkouts you trust. See docs/configuration.md.",
123)
126def _from_default(name: str) -> dict | None:
127 for a in [*DEFAULT_CONFIG.get("agent", []), AGY_AGENT]:
128 if a.get("name") == name:
129 return dict(a)
130 return None
133#: Templates `jury init` writes only when they are named: never picked by
134#: "detected", "all", or an interactive default. agy cannot be confined for a
135#: reviewer of untrusted diffs (see ``config.AGY_AGENT``), so seating it is the
136#: operator's explicit decision, and `jury init` says so when it writes one.
137OPT_IN_AGENTS: tuple[str, ...] = ("agy",)
140def implicit_choices(names) -> list[str]:
141 """*names* without the opt-in-only agents: what a default may pick."""
142 return [n for n in names if n not in OPT_IN_AGENTS]
145def agent_templates() -> dict[str, dict]:
146 """Built-in agent templates keyed by short name (a fresh copy each call)."""
147 templates: dict[str, dict] = {}
148 for name in ("claude", "codex", "agy"):
149 tmpl = _from_default(name)
150 if tmpl is not None:
151 templates[name] = tmpl
152 templates["qwen"] = dict(_LOCAL_TEMPLATE)
153 templates["claude-api"] = dict(_ANTHROPIC_API_TEMPLATE)
154 templates["codex-api"] = dict(_OPENAI_API_TEMPLATE)
155 templates["gemini-api"] = dict(_GOOGLE_API_TEMPLATE)
156 templates["openrouter"] = dict(_OPENROUTER_TEMPLATE)
157 templates["deepseek"] = dict(_DEEPSEEK_TEMPLATE)
158 templates["groq"] = dict(_GROQ_TEMPLATE)
159 templates["aider"] = dict(_GENERIC_CLI_TEMPLATE)
160 return templates
163_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "[::1]"})
166def agents_needing_remote_opt_in() -> tuple[str, ...]:
167 """Templates whose endpoint the config validator refuses without an opt-in.
169 `config` accepts a loopback endpoint and refuses every other host unless
170 ``JURY_ALLOW_REMOTE_ENDPOINT`` is set — a deliberate default-closed posture,
171 since a config-supplied URL is otherwise a request-forgery primitive. Three
172 hosted templates point at real vendors, so a preset that silently includes
173 them produces a config `jury init` then refuses to write.
175 Derived from the templates rather than listed, so a new hosted template is
176 covered the day it lands.
177 """
178 remote = []
179 for name, template in agent_templates().items():
180 endpoint = template.get("endpoint")
181 if not endpoint:
182 continue
183 host = urlsplit(endpoint).hostname or ""
184 if host.lower() not in _LOOPBACK_HOSTS:
185 remote.append(name)
186 return tuple(remote)
189#: Every agent `jury init` can scaffold, in the order it offers them.
190#:
191#: Derived from :func:`agent_templates` rather than listed, because a second
192#: hand-written copy of the same set is what #589 asked to be fixed and #590
193#: did not: four templates — ``openrouter``, ``deepseek``, ``groq``, ``aider`` —
194#: shipped without ever reaching this tuple, so ``jury init --list-agents``, the
195#: wizard, and ``--preset all`` could not see them, while the error message for
196#: an unknown agent named them. The CLI told users to choose from four options
197#: it never offered.
198#:
199#: ``agent_templates`` reads only module constants, so this costs no I/O at
200#: import and is deterministic.
201#:
202#: Read by ``cli._init_available``, ``cli._init_interactive``, ``cli._init_wizard``
203#: and ``cli._run_init`` — every path through which ``jury init`` offers, detects
204#: or defaults an agent. Nothing in *this* module reads it, which is the whole of
205#: what CodeQL's intra-module ``py/unused-global-variable`` sees (#696): the
206#: readers are real, and deleting this tuple would take ``--list-agents``, the
207#: wizard and ``--preset all`` with it.
208KNOWN_AGENTS: tuple[str, ...] = tuple(agent_templates())
210# Substrings that hint a local model is code-oriented (preferred for reviews).
211_CODER_HINTS: tuple[str, ...] = ("coder", "code", "deepseek", "qwen")
214def pick_default_model(models: list[str]) -> str | None:
215 """Choose a sensible default from discovered local models (issue #109).
217 Prefers a code-oriented model (name contains 'coder'/'code'/etc.), else the
218 first listed; returns None for an empty list.
219 """
220 if not models:
221 return None
222 for m in models:
223 low = m.lower()
224 if any(h in low for h in _CODER_HINTS):
225 return m
226 return models[0]
229#: Where the zero-config local fallback looks for a model (#863).
230LOCAL_FALLBACK_ENDPOINT = "http://localhost:11434/v1"
233def zero_config_local_seat(
234 config_named, mock, config_file_present, any_seat_available, list_models
235):
236 """The local seat a run with no config reviews with, or ``None`` (#863).
238 The one decision a run's fallback and ``jury --doctor`` share, so the doctor
239 predicts the panel the run forms rather than a copy of the rule that can
240 drift. It fires only in the "fresh user" case: no ``--config``, not
241 ``--mock``, no ``./jury.toml``, no configured seat available, and a local
242 server that lists a model. The I/O is injected and called lazily, in that
243 order: ``any_seat_available()`` only when the flags allow a fallback, and
244 ``list_models()`` only when no seat can run, so neither path asks the local
245 server when it would not use the answer. A probe that raises reads as "do
246 not fall back": availability probing must never crash a run.
247 """
248 from .config import AgentSpec
250 if config_named or mock or config_file_present:
251 return None
252 try:
253 if any_seat_available():
254 return None
255 except Exception: # noqa: BLE001 - availability probing must never crash a run
256 return None
257 model = pick_default_model(list_models())
258 if not model:
259 return None
260 return AgentSpec(name="local", vendor="local", model=model, endpoint=LOCAL_FALLBACK_ENDPOINT)
263def seat_local_agents(
264 agents: list[str], models: list[str] | None
265) -> tuple[list[str], str | None, list[str]]:
266 """Fill the local seats in *agents* from the models a server lists (issue #864).
268 Plain ``jury init`` wrote the local template's model whatever the server had,
269 then warned about its own output. *models* is ``adapters.local_model_listing``'s
270 answer: a list when the server answered, ``None`` when the listing failed.
271 Returns ``(seated, model, left_out)``:
273 * the server lists models → every agent stays seated and *model* is the one
274 :func:`pick_default_model` prefers;
275 * the server lists none, other seats present → the local seats move to
276 *left_out*, for the caller to write commented out, so the file names no model
277 nobody has pulled and its hash is that of the panel that actually runs;
278 * the server lists none, local seats only → they stay seated on the template's
279 model. A config with no seat is invalid, and the pull hint `jury init` prints
280 for an empty server names that same model, so pulling it completes the file;
281 * the listing failed (refused, timed out, an error status, an endpoint the SSRF
282 gate refuses) → every agent stays seated on the template's model, as before
283 #864. No evidence is not evidence of a fault — the rule
284 ``doctor._local_model_gap`` follows (#849): a server that is down says
285 nothing about what is pulled on it.
287 Pure: the caller lists the models, and only when there is a local seat.
288 """
289 templates = agent_templates()
290 local = [a for a in agents if templates.get(a, {}).get("vendor") == "local"]
291 model = pick_default_model(models or [])
292 if models is None or model is not None or not local or set(local) == set(agents):
293 return list(agents), model, []
294 return [a for a in agents if a not in local], None, list(dict.fromkeys(local))
297# Named setup presets (issue: easier config). Each gives default agents +
298# settings for a common intent; explicit flags / detected agents override the
299# `agents` value ("detected" = the agents available right now, "all" = every
300# known agent). Resolved by the CLI, which knows availability. Neither
301# "detected" nor "all" includes an OPT_IN_AGENTS entry: a preset is a default,
302# and agy is only ever seated by name.
303PRESETS: dict[str, dict] = {
304 "offline": {"agents": ["qwen"], "rounds": 1, "verify": False},
305 "fast": {"agents": "detected", "rounds": 1, "verify": False},
306 "balanced": {"agents": "detected", "rounds": 2, "verify": True, "early_stop": True},
307 "thorough": {"agents": "all", "rounds": 2, "verify": True},
308}
311def build_config(
312 agents: list[str],
313 *,
314 rounds: int = 2,
315 chair: str | None = None,
316 verify: bool = True,
317 early_stop: bool | None = None,
318 local_model: str | None = None,
319 local_endpoint: str | None = None,
320 decision: str | None = None,
321 auto_depth: bool | None = None,
322 context_mode: str | None = None,
323 redact_secrets: bool | None = None,
324 ci_fail_on: list[str] | None = None,
325 effort: str | None = None,
326) -> dict:
327 """Build a jury config dict from selected agent names.
329 Raises ``ValueError`` on an unknown agent name or an empty selection. The
330 chair defaults to the first selected agent. Local agents pick up the
331 optional model/endpoint overrides.
333 The optional ``decision``/``auto_depth``/``context_mode``/``redact_secrets``/
334 ``ci_fail_on`` knobs (used by ``jury init --wizard``) are written ONLY when
335 not ``None`` — callers that omit them produce byte-identical output to before,
336 keeping the scaffolded file free of redundant built-in defaults.
338 ``effort`` (issue #662) is written onto each selected agent whose vendor can
339 act on it; agents whose vendor has no effort control are left alone rather
340 than scaffolded with a setting that would only warn at run time.
341 """
342 templates = agent_templates()
343 chosen: list[dict] = []
344 seen: set[str] = set()
345 for name in agents:
346 if name in seen:
347 continue
348 tmpl = templates.get(name)
349 if tmpl is None:
350 raise ValueError(f"unknown agent '{name}'; choose from {', '.join(templates.keys())}")
351 entry = dict(tmpl)
352 # EXEMPT from `normalise_vendor` (issue #701, round 3): `entry` is a copy
353 # of one of this module's OWN templates, whose vendor strings are literals
354 # written here in normalised form. There is no operator spelling to
355 # normalise — the value is not yet configuration, it is what this function
356 # is about to write out as configuration.
357 if entry.get("vendor") == "local":
358 if local_model:
359 entry["model"] = local_model
360 if local_endpoint:
361 entry["endpoint"] = local_endpoint
362 if effort and _effort_supported(entry.get("vendor", "")):
363 entry["effort"] = effort
364 chosen.append(entry)
365 seen.add(name)
367 if not chosen:
368 raise ValueError("select at least one agent")
370 if chair is None:
371 chair = chosen[0]["name"]
373 jury: dict = {"rounds": int(rounds), "chair": chair, "verify": bool(verify)}
374 if early_stop:
375 jury["early_stop"] = True
376 if auto_depth is not None:
377 jury["auto_depth"] = bool(auto_depth)
378 if decision is not None:
379 jury["decision"] = decision
380 if context_mode is not None or redact_secrets is not None:
381 context: dict = {}
382 if context_mode is not None:
383 context["mode"] = context_mode
384 if redact_secrets is not None:
385 context["redact_secrets"] = bool(redact_secrets)
386 jury["context"] = context
387 if ci_fail_on is not None:
388 jury["ci"] = {"fail_on": list(ci_fail_on)}
389 return {"jury": jury, "agent": chosen}
392def _effort_supported(vendor: str) -> bool:
393 """Whether *vendor* has an effort control (see ``adapters.effort_args``).
395 Imported lazily so this module keeps its light import graph; ``adapters``
396 is the single owner of the vendor -> effort mapping.
397 """
398 from .adapters import effort_supported
400 return effort_supported(vendor)
403def _scalar(value) -> str:
404 if isinstance(value, bool):
405 return "true" if value else "false"
406 if isinstance(value, int):
407 return str(value)
408 if isinstance(value, float) and math.isfinite(value):
409 # `repr` is a valid TOML float for every finite value ("1.0", "0.7",
410 # "1e-05"). A non-finite one is refused below: no config key accepts it.
411 return repr(value)
412 if isinstance(value, str):
413 escaped = value.replace("\\", "\\\\").replace('"', '\\"')
414 return f'"{escaped}"'
415 raise TypeError(f"cannot render TOML scalar of type {type(value).__name__}")
418def _render_value(value) -> str:
419 if isinstance(value, list):
420 return "[" + ", ".join(_scalar(v) for v in value) + "]"
421 return _scalar(value)
424# Stable key order for agent tables so output is deterministic and readable.
425_AGENT_KEY_ORDER = (
426 "name",
427 "vendor",
428 "command",
429 "endpoint",
430 "model",
431 "effort",
432 "temperature",
433 "extra_args",
434 # Written when a template sets it (#859): the aider seat's argv ends in
435 # `--message`, which only works when the prompt is appended after it.
436 "prompt_mode",
437)
439#: Commented hint written under every effort-capable agent that has no explicit
440#: level, so the setting is discoverable from the generated file itself.
441_EFFORT_HINT = '# effort = "medium" # low | medium | high'
443#: Commented hint written under every local seat with no explicit temperature.
444#: Commented, not set: the default model does not need it, and a written value
445#: would split every generated config's hash for nothing. It is here so the
446#: operator who picks a model that loops at the greedy default finds the knob.
447_TEMPERATURE_HINT = "# temperature = 1.0 # default 0; set 1.0 for models that loop at 0 (gpt-oss)"
450#: Written under every scaffolded ``[jury.ci]`` (issue #682). Commented out,
451#: because the shipped default already IS 2 — the hint exists so a reader
452#: discovers the knob and its opt-out here rather than only after a run exits 3.
453#:
454#: The section it lives under is emitted UNCONDITIONALLY (issue #692). It used to
455#: be written only when a caller passed ``ci_fail_on``, which no preset and no
456#: plain ``jury init`` ever does — so the hint the module defines never reached a
457#: generated file, and `--preset thorough` (three or four vendors, and therefore
458#: the config most likely to exit 3 on a one-CLI machine) shipped with nothing
459#: about the guard in it at all.
460_MIN_VENDORS_HINT = (
461 "# Distinct vendors that must have contributed a review before the run can",
462 "# stand as cross-vendor consensus (exit 3 otherwise). Defaults to 2 and only",
463 "# applies when 2+ vendors are enabled HERE — including when one of their CLIs",
464 "# is not installed; set 0 (or pass --no-min-vendors) to accept a panel that",
465 "# collapsed to one vendor, or --strict to fail at startup on a missing CLI.",
466 "# min_vendors = 2",
467)
470#: Written above a local seat `jury init` left out because its server listed no
471#: model (issue #864). The block under it is the seat, commented, with the
472#: template's model — the one the pull command below fetches.
473_NO_LOCAL_MODEL_HINT = (
474 "# The local server listed no model when `jury init` ran, so this seat is",
475 "# left out rather than named after a model nobody has pulled. Pull one (for",
476 f"# Ollama: `ollama pull {_LOCAL_TEMPLATE['model']}`) and uncomment the block below,",
477 "# or rerun `jury init` with `--local-model <id>`.",
478)
481def render_toml(config: dict, *, commented_agents: list[dict] | tuple = ()) -> str:
482 """Render a jury config dict to ``jury.toml`` text (minimal, typed).
484 Handles exactly the value types this config uses (str/int/bool/list[str]).
485 Empty/None values are omitted so a local agent (no ``command``/``extra_args``)
486 stays clean. *commented_agents* are written after the panel as commented-out
487 ``[[agent]]`` blocks under a hint — the local seats :func:`seat_local_agents`
488 left out because the server listed no model.
489 """
490 lines = [
491 "# Generated by `jury init`. Edit freely — see docs/configuration.md",
492 "# for the full schema (rounds, ci gate, context policy, diff handling).",
493 "",
494 "[jury]",
495 ]
496 jury = config["jury"]
497 # Scalar [jury] keys in a stable, readable order. ``decision``/``auto_depth``
498 # are emitted here only when present (the wizard sets them on a non-default).
499 for key in ("rounds", "chair", "verify", "decision", "auto_depth", "early_stop", "max_rounds"):
500 if key in jury:
501 lines.append(f"{key} = {_render_value(jury[key])}")
502 lines.append("")
504 # Optional nested tables, written only when the wizard captured a non-default.
505 context = jury.get("context")
506 if context:
507 lines.append("[jury.context]")
508 for key in ("mode", "redact_secrets"):
509 if key in context:
510 lines.append(f"{key} = {_render_value(context[key])}")
511 lines.append("")
512 # `[jury.ci]` is always written, with the cross-vendor hint under it (#692).
513 # `fail_on` still appears only when a caller chose one, so the file keeps
514 # stating no redundant defaults; an otherwise empty section is comments only
515 # and parses to `{}`, which is exactly what the run resolves today.
516 ci = jury.get("ci") or {}
517 lines.append("[jury.ci]")
518 if "fail_on" in ci:
519 lines.append(f"fail_on = {_render_value(ci['fail_on'])}")
520 lines.extend(_MIN_VENDORS_HINT)
521 lines.append("")
523 for agent in config["agent"]:
524 if adapter_key(agent.get("vendor", ""), agent.get("adapter")) in GENERIC_CLI_VENDORS:
525 lines.append(f"# {UNSANDBOXED_LABEL}")
526 lines.extend(UNSANDBOXED_HINT)
527 lines.append("[[agent]]")
528 lines.extend(_agent_keys(agent))
529 # Only hint at `effort` where the vendor can actually act on it; a hint
530 # under the `claude`/`codex` CLI blocks would invite a setting that only
531 # ever produces an "effort unsupported" warning.
532 if not agent.get("effort") and _effort_supported(agent.get("vendor", "")):
533 lines.append(_EFFORT_HINT)
534 # Only local seats send a temperature, and which adapter a seat runs
535 # through decides that, not its vendor, the same rule validation uses.
536 if agent.get("temperature") is None and (
537 adapter_key(agent.get("vendor", ""), agent.get("adapter")) == "local"
538 ):
539 lines.append(_TEMPERATURE_HINT)
540 lines.append("")
542 # A seat left out is still written, commented, so uncommenting it is the whole
543 # fix once its model exists (#864). Comments parse to nothing, so the config
544 # and its hash are those of the seats above.
545 for agent in commented_agents:
546 lines.extend(_NO_LOCAL_MODEL_HINT)
547 lines.append("# [[agent]]")
548 lines.extend(f"# {line}" for line in _agent_keys(agent))
549 lines.append("")
551 return "\n".join(lines).rstrip() + "\n"
554def _agent_keys(agent: dict) -> list[str]:
555 """The ``key = value`` lines of one ``[[agent]]`` table, in a stable order."""
556 lines = []
557 for key in _AGENT_KEY_ORDER:
558 value = agent.get(key)
559 if value in (None, "", []):
560 continue
561 lines.append(f"{key} = {_render_value(value)}")
562 return lines