Coverage for src/ai_jury/orchestrator.py: 99%
587 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
1"""Jury orchestration: review -> debate -> synthesis.
3The orchestrator owns the round structure and prompt assembly; adapters only run
4their CLI. Rounds run agents concurrently (thread pool) because each call is an
5independent, IO-bound subprocess.
6"""
8from __future__ import annotations
10import random
11import shutil
12import string
13import time
14from concurrent.futures import ThreadPoolExecutor
15from dataclasses import dataclass, field, replace
17from . import convergence, injection, largediff, panel, prompts, routing
18from .adapters import RETRYABLE_ERROR_CODES, Adapter, AgentResult, make_adapter
19from .config import JuryConfig, agy_opt_in_hint
20from .consensus import FindingGroup, demote_local_only_groups, group_findings
21from .diffprofile import profile_diff
22from .findings import (
23 Finding,
24 Verdict,
25 emitted_findings_block,
26 parse_findings,
27 parse_verdicts,
28)
29from .policy import ReviewPolicy, render_policy_section
30from .privilege import audit_privilege
31from .redaction import redact
34class RunBudget:
35 """Wall-clock budget for one jury run (issue #30).
37 Tracks the elapsed time since construction and derives the timeout to pass a
38 single agent call from the optional total-run and per-phase budgets. ``None``
39 for either budget means uncapped; when both are unset ``call_timeout``
40 returns ``None`` so adapters fall back to their own per-agent timeout and
41 behaviour is identical to having no budget at all.
42 """
44 def __init__(self, total_timeout: int | None, phase_timeout: int | None):
45 self.total = total_timeout
46 self.phase = phase_timeout
47 self._start = time.monotonic()
49 def elapsed(self) -> float:
50 return time.monotonic() - self._start
52 def remaining(self) -> float | None:
53 if self.total is None:
54 return None
55 return max(0.0, self.total - self.elapsed())
57 def expired(self) -> bool:
58 return self.total is not None and self.elapsed() >= self.total
60 def call_timeout(self) -> int | None:
61 """Per-call timeout: the min of the phase budget and remaining total.
63 The agent's own per-agent timeout is applied by the adapter (it takes the
64 min with this value), so it is not needed here. Returns ``None`` when
65 neither budget caps the call, leaving the adapter to use its configured
66 per-agent timeout.
67 """
68 caps: list[float] = []
69 if self.phase is not None:
70 caps.append(float(self.phase))
71 remaining = self.remaining()
72 if remaining is not None:
73 caps.append(remaining)
74 if not caps:
75 return None
76 return max(1, int(min(caps)))
79def _run_with_retry(
80 adapter: Adapter,
81 prompt: str,
82 phase: str,
83 budget: RunBudget,
84 retries: int,
85 log,
86) -> AgentResult:
87 """Run one agent for one phase, retrying transient failures (issue #30).
89 Retries only failures whose typed error code is in
90 ``RETRYABLE_ERROR_CODES`` (timeout/rate-limit/spawn), up to ``retries`` extra
91 attempts. A deterministic failure (auth, missing CLI, empty output, generic
92 nonzero exit) is returned immediately. The returned result's ``attempts``
93 records how many tries were made. Retrying stops early when the run budget is
94 exhausted so a retry never overruns the total timeout.
96 The result is also stamped with the model id the adapter sent (issue #709).
97 This is one of exactly two places a seat is invoked — the other is ``jury
98 run-agent`` — so stamping here covers every phase of every run, and the
99 ballot quotes :attr:`AgentResult.model` instead of deriving a second answer
100 from the spec.
101 """
102 max_attempts = max(1, retries + 1)
103 result = adapter.run(prompt, phase=phase, timeout=budget.call_timeout())
104 attempts = 1
105 while (
106 not result.ok
107 and result.error_code in RETRYABLE_ERROR_CODES
108 and attempts < max_attempts
109 and not budget.expired()
110 ):
111 log(f"{adapter.name}: {phase} attempt {attempts} failed ({result.error_code}); retrying")
112 result = adapter.run(prompt, phase=phase, timeout=budget.call_timeout())
113 attempts += 1
114 result.attempts = attempts
115 result.model = adapter.resolved_model()
116 return result
119def _order_by_agents(results: list[AgentResult], order: list[str]) -> list[AgentResult]:
120 """Reorder phase results into the configured/enabled agent order.
122 Round phases run agents concurrently (ThreadPoolExecutor.map), so the order
123 in which results arrive is not guaranteed across runs. The report and all
124 downstream consumers must NOT depend on thread-completion order, so we sort
125 every phase's results by each agent's index in ``order`` (the stable
126 enabled-agent list). Agents not present in ``order`` (should not happen)
127 sort to the end, preserving their relative arrival order as a stable
128 tiebreak so the sort is total and deterministic.
129 """
130 index = {name: i for i, name in enumerate(order)}
131 fallback = len(order)
132 return sorted(results, key=lambda r: index.get(r.agent, fallback))
135@dataclass
136class JuryOutcome:
137 reviews: list[AgentResult]
138 debate: list[AgentResult]
139 synthesis: AgentResult | None
140 chair: str
141 findings: list[Finding] = field(default_factory=list)
142 warnings: list[str] = field(default_factory=list)
143 groups: list[FindingGroup] = field(default_factory=list)
144 verify: AgentResult | None = None
145 verdicts: list[Verdict] = field(default_factory=list)
146 context_mode: str = "diff-only"
147 redact_secrets: bool = True
148 redaction_count: int = 0
149 injection_hits: list = field(default_factory=list)
150 # Execution/partial-result signals (issue #30): agents skipped because their
151 # CLI was unavailable (name, reason), and whether the run budget was
152 # exhausted before all phases completed.
153 skipped: list = field(default_factory=list)
154 budget_exhausted: bool = False
155 # Adaptive-rounds signals (issue #40): rounds actually executed and a short
156 # human-readable reason for why the debate ran / stopped.
157 rounds_executed: int = 1
158 stop_reason: str = ""
159 # Set when this outcome was served from the local result cache (issue #33),
160 # so the report/metadata can mark it as cached rather than freshly computed.
161 from_cache: bool = False
162 # The paths and symbols in the change the panel was shown (issue #710), so a
163 # ballot's ``Checked:`` line can be resolved against something instead of
164 # being accepted on shape alone. ``None`` means "this outcome was not built
165 # from a diff" — a hand-assembled outcome, a caller that never had one — and
166 # the scope rule then falls back to the structural test it applied before.
167 changed: largediff.ChangeIndex | None = None
168 # What tiered routing decided (#714): the plan's dict, or the standard record.
169 routing: dict = field(default_factory=lambda: routing.standard_plan([]).as_dict())
170 # Files the large-diff planner kept out of what the panel saw, as
171 # ``(path, reason)`` pairs (#978); a binary one is named to the reviewers
172 # and in the report so a changed golden image is not invisible.
173 excluded_files: list = field(default_factory=list)
176def _run_phase(
177 adapters: list[Adapter],
178 prompt_for: dict[str, str],
179 phase: str,
180 parallel: bool,
181 *,
182 budget: RunBudget,
183 retries: int,
184 log,
185) -> list[AgentResult]:
186 def task(a: Adapter) -> AgentResult:
187 return _run_with_retry(a, prompt_for[a.name], phase, budget, retries, log)
189 if parallel and len(adapters) > 1:
190 with ThreadPoolExecutor(max_workers=len(adapters)) as pool:
191 return list(pool.map(task, adapters))
192 return [task(a) for a in adapters]
195def _others(reviews: list[AgentResult], me: str) -> str:
196 """Identity-labeled peer reviews (legacy path; ``anonymize_debate = false``).
198 Renders each *other* reviewer's round-1 output with its real agent/vendor
199 identity in the stable enabled-agent order. This is the pre-#37 behaviour and
200 leaks both identity and position; the anonymizing path below is the default.
201 """
202 chunks = [
203 f"### {r.agent} ({r.vendor})\n{r.output}"
204 for r in reviews
205 if r.agent != me and r.ok and r.output
206 ]
207 return "\n\n".join(chunks) if chunks else "_(no other reviews available)_"
210def _anon_label(i: int) -> str:
211 """Stable anonymous reviewer label: 0->'A', 1->'B', ... 26->'AA'."""
212 letters = string.ascii_uppercase
213 label = ""
214 i += 1
215 while i > 0:
216 i, rem = divmod(i - 1, 26)
217 label = letters[rem] + label
218 return label
221def _anonymize_peers(
222 reviews: list[AgentResult], me: str, rng: random.Random
223) -> tuple[str, dict[str, str]]:
224 """Chatham House peer view for a debater (#37).
226 Returns ``(prompt_text, label_to_agent)`` where the prompt text renders each
227 *other* successful reviewer's round-1 output under an anonymous
228 ``### Reviewer A`` / ``### Reviewer B`` heading — NO vendor or agent name.
229 The debater's OWN review is excluded (it is passed separately as
230 ``own_review``). Presentation order is shuffled DETERMINISTICALLY using the
231 shared run RNG so neither identity nor position is a stable signal; the same
232 seed yields the same order, different seeds may differ.
234 ``label_to_agent`` keeps the anonymous-label -> real-agent mapping internal so
235 callers can still recover authorship (the report attributes by real name).
236 """
237 peers = [r for r in reviews if r.agent != me and r.ok and r.output]
238 if not peers:
239 return "_(no other reviews available)_", {}
240 # Deterministic per-debater shuffle from the shared run RNG. We shuffle a
241 # copy so the caller's review list (used elsewhere) is untouched.
242 order = list(peers)
243 rng.shuffle(order)
244 chunks: list[str] = []
245 label_to_agent: dict[str, str] = {}
246 for i, r in enumerate(order):
247 label = f"Reviewer {_anon_label(i)}"
248 label_to_agent[label] = r.agent
249 chunks.append(f"### {label}\n{r.output}")
250 return "\n\n".join(chunks), label_to_agent
253def _debate_round(
254 debaters: list[Adapter],
255 reviews: list[AgentResult],
256 diff: str,
257 config: JuryConfig,
258 run_rng: random.Random,
259 agent_order: list[str],
260 prior: list[AgentResult],
261 budget: RunBudget,
262 retries: int,
263 log,
264 round_no: int,
265 template: str = prompts.DEBATE,
266) -> list[AgentResult]:
267 """Run one debate round and return its results in stable agent order.
269 ``prior`` holds the previous round's debate outputs (empty for the first
270 debate round); when present they are appended to each debater's prompt as a
271 "prior debate" addendum so later rounds in an adaptive run (issue #40) build
272 on, rather than repeat, earlier cross-examination. The peer-review anonymizing
273 path (#37) is preserved unchanged.
274 """
275 log(f"round {round_no}: {len(debaters)} agents cross-examining")
276 own = {r.agent: r.output for r in reviews if r.ok}
277 # Prior-round debate output quotes attacker-controlled diff text, so it is
278 # untrusted: neutralize sentinels (issue #316/L-1) before it is fenced and
279 # appended below, matching every other peer-output slot.
280 prior_txt = prompts.neutralize_sentinels(
281 "\n\n".join(f"### {r.agent}\n{r.output}" for r in prior if r.ok and r.output)
282 )
283 debate_prompt: dict[str, str] = {}
284 for a in debaters:
285 if config.anonymize_debate:
286 # Per-debater deterministic shuffle: derive a child RNG from the
287 # shared run RNG so each debater gets an independent but reproducible
288 # peer ordering (same seed -> same order).
289 peer_rng = random.Random(run_rng.random())
290 other_reviews, _label_map = _anonymize_peers(reviews, a.name, peer_rng)
291 else:
292 other_reviews = _others(reviews, a.name)
293 text = template.format(
294 name=a.name,
295 diff=prompts.neutralize_sentinels(diff),
296 own_review=prompts.neutralize_sentinels(
297 own.get(a.name, "_(your review was unavailable)_")
298 ),
299 other_reviews=prompts.neutralize_sentinels(other_reviews),
300 notice=prompts._UNTRUSTED_NOTICE,
301 )
302 if prior_txt:
303 text += (
304 "\n\n=== PRIOR DEBATE (earlier round) ===\n"
305 "Build on this; do not just repeat it. Only keep a DISPUTE or "
306 "MISSED item if it is still unresolved.\n\n"
307 "<<<UNTRUSTED_REVIEW\n" + prior_txt + "\nUNTRUSTED_REVIEW>>>\n"
308 )
309 debate_prompt[a.name] = text
310 results = _run_phase(
311 debaters,
312 debate_prompt,
313 "debate",
314 config.parallel,
315 budget=budget,
316 retries=retries,
317 log=log,
318 )
319 # Same stable-ordering guarantee as round 1: independent of thread-pool
320 # completion order.
321 return _order_by_agents(results, agent_order)
324def _skip_reason(spec) -> str:
325 """Why an unavailable seat was skipped, worded for its transport (#831).
327 The one hardcoded ``CLI not found ({command})`` rendered ``CLI not found ()`` for a
328 commandless local or hosted-API seat, whose ``command`` is empty. Name what is actually
329 missing instead — a CLI on PATH, a reachable endpoint, or an API key — with the value
330 passed through :func:`redact` so an endpoint's credentials never reach the log.
331 """
332 command = getattr(spec, "command", "") or ""
333 if command:
334 return f"CLI not found ({redact(command)[0]})"
335 endpoint = getattr(spec, "endpoint", "") or ""
336 if endpoint:
337 return f"endpoint not reachable ({redact(endpoint)[0]})"
338 vendor = redact(getattr(spec, "vendor", "") or "hosted-API")[0]
339 return f"{vendor} seat unavailable (no API key, or the API is unreachable)"
342def run_jury(
343 config: JuryConfig,
344 diff: str,
345 *,
346 context: str = "",
347 hints: str = "",
348 mock: bool = False,
349 strict: bool = False,
350 seed: int | None = None,
351 policy: ReviewPolicy | None = None,
352 log=lambda _msg: None,
353 budget: RunBudget | None = None,
354 on_event=None,
355 mode: str = "code",
356 risk: str | None = None,
357 excluded_files=(),
358) -> JuryOutcome:
359 # Jury mode (issue #221): "code" (default) reviews a diff with the code-review
360 # rubric; "issue" reviews a GitHub issue's prose for completeness/clarity.
361 # Only the prompt TEMPLATES differ — the round structure, consensus, voting,
362 # verification, ordering, and determinism are identical. ``tmpl`` selects the
363 # four phase templates; each is threaded into the phase that uses it so the
364 # call sites are otherwise unchanged.
365 tmpl = prompts.for_mode(mode)
366 # Live play-by-play hook (issue #210): an optional callback fired after each
367 # phase result is produced — ``on_event(kind, result, round_no=None)`` with
368 # kind in {"review", "debate", "verify", "synthesis"}. It lets a caller stream
369 # the deliberation as it happens (CLI ``--live``) without the orchestrator
370 # doing any I/O itself. Fired in stable per-phase order (not thread-completion
371 # order) so the event sequence is deterministic. Defaults to a no-op.
372 emit = on_event or (lambda *_a, **_k: None)
373 # Repository review policy (optional, #8): maintainer-authored, TRUSTED
374 # content rendered into each REVIEW prompt in a clearly separated section.
375 # When ``policy`` is None a sentinel placeholder is used, so the prompt is
376 # unchanged except for that section. The policy is distinct from the
377 # agent-runtime ``config`` and never enters the untrusted diff/context fences.
378 policy_section = render_policy_section(policy)
379 # Run reproducibility: a single shared RNG seeds every randomized
380 # orchestration decision (future: anonymized-rebuttal order, rotating
381 # chair, tie-breaks). The seed comes from the explicit ``seed`` argument if
382 # given, else from ``config.seed``. We construct a dedicated
383 # ``random.Random`` instance rather than touching the global ``random``
384 # module so seeding a jury run never perturbs unrelated global state.
385 # When the seed is None the RNG is unseeded (still deterministic
386 # orchestration; randomness, if any, is just not reproducible run-to-run).
387 # LLM output itself is never made deterministic by this — only the
388 # orchestration around it. ``run_rng`` is the shared run RNG: pass it to
389 # any feature that needs reproducible randomness instead of using ``random``.
390 run_seed = seed if seed is not None else config.seed
391 run_rng = random.Random(run_seed) # shared run RNG (see docstring)
393 # Run budget (issue #30): a single wall-clock budget threaded through every
394 # phase. Defaults (both None) leave behaviour identical to no budget, with
395 # each agent bounded only by its own per-agent timeout. ``retries`` is the
396 # number of extra attempts for transient (retryable) failures. A caller may
397 # pass a SHARED budget so ``total_timeout`` spans a whole chunked review
398 # rather than resetting per chunk (issue #31 / review finding).
399 if budget is None:
400 budget = RunBudget(config.total_timeout, config.phase_timeout)
401 retries = config.retries
403 # Context policy: diff-only sends only the diff; expanded includes context.
404 ctx_cfg = getattr(config, "context", None)
405 context_mode = getattr(ctx_cfg, "mode", "diff-only") if ctx_cfg else "diff-only"
406 redact_on = getattr(ctx_cfg, "redact_secrets", True) if ctx_cfg else True
407 if context_mode == "diff-only":
408 context = ""
409 redaction_count = 0
410 if redact_on:
411 diff, _n1 = redact(diff)
412 context, _n2 = redact(context)
413 redaction_count = _n1 + _n2
414 if redaction_count:
415 log(f"redacted {redaction_count} secret(s) before sending to agents")
417 # Static-analysis pre-pass block (#523), joined into the Round 1 prompt AFTER
418 # the context-mode filter above (#715). It is not user context: it is produced
419 # locally by this run's linters, so the "diff-only" mode — the default — must
420 # not discard it. Carrying it inside ``context`` did exactly that, and
421 # `hints = true` reached no reviewer under any default configuration.
422 review_context = "\n\n".join(part for part in (context, hints) if part.strip())
424 # Prompt-injection heuristic (OWASP LLM01): scan untrusted diff/context for
425 # patterns that try to override instructions, then SURFACE them as a synthetic
426 # finding/warning. We never act on them; the CI gate is derived from
427 # structured consensus (see ci.evaluate_ci), so an injected "APPROVE"
428 # cannot flip the verdict.
429 # Index the change AFTER redaction, so the index describes the same bytes the
430 # panel is shown and never carries a secret the diff no longer has (#710).
431 changed = largediff.change_index(diff)
433 injection_hits = injection.scan_inputs(diff, context)
434 injection_findings: list[Finding] = []
435 if injection_hits:
436 log(f"prompt-injection heuristic: {len(injection_hits)} suspicious pattern(s) flagged")
437 syn = injection.hits_to_finding(injection_hits)
438 if syn is not None: 438 ↛ 443line 438 didn't jump to line 443 because the condition on line 438 was always true
439 injection_findings.append(syn)
441 # Least-privilege audit: warn when a configured agent could perform
442 # write/tool actions while reviewing attacker-controlled content.
443 privilege_warnings = audit_privilege(config.enabled_agents)
444 for w in privilege_warnings:
445 log(f"least-privilege warning: {w}")
446 if strict and privilege_warnings:
447 raise RuntimeError(
448 "least-privilege check failed (--strict): " + "; ".join(privilege_warnings)
449 )
451 specs = config.enabled_agents
452 adapters = [make_adapter(s, mock=mock) for s in specs]
454 # Filter to available agents (unless strict, where a missing CLI is fatal).
455 # Skipped agents are recorded (name, reason) so the report can state exactly
456 # which agents never ran — part of the partial-result policy (issue #30).
457 usable: list[Adapter] = []
458 skipped: list[tuple[str, str]] = []
459 for a in adapters:
460 if a.available():
461 usable.append(a)
462 elif strict:
463 raise RuntimeError(f"agent '{a.name}' CLI not available: {a.spec.command}")
464 else:
465 reason = _skip_reason(a.spec)
466 log(f"skipping '{a.name}': {reason}")
467 skipped.append((a.name, reason))
468 if not usable:
469 message = (
470 "no usable agents — install an agent CLI (claude / codex), run a local "
471 "model, or set a hosted-API key (e.g. ANTHROPIC_API_KEY with `jury init --agents "
472 "claude-api`) — or use --mock"
473 )
474 # An agy-only machine: the one CLI it has is opt-in, so say why it was
475 # not used rather than only telling it to install another.
476 hint = agy_opt_in_hint((s.adapter_key for s in specs), shutil.which)
477 if hint:
478 message += f". Note: {hint}."
479 raise RuntimeError(message)
481 usable_names = [a.name for a in usable]
482 # Tiered routing (#714): a pure plan over the enabled bench, the usable
483 # names and the diff's risk band decides who sits in round 1, who anchors
484 # and who is benched; the standard mode records a plan too so the report
485 # always says what happened. Benched seats stay available for escalation.
486 if config.routing == routing.MODE_TIERED:
487 plan = routing.plan_panel(
488 specs,
489 usable_names,
490 # The band of the WHOLE change when the caller has one. A chunked
491 # review calls this once per chunk (#31), and a chunk of a large or
492 # security-touching diff can look routine on its own — routing the
493 # panel off it would bench the frontier seats on exactly the change
494 # they were kept for. `review_diff` profiles the filtered diff once
495 # and threads the answer through every chunk.
496 risk if risk is not None else profile_diff(diff).risk,
497 chair=config.chair,
498 min_vendors=int(getattr(config.ci, "min_vendors", 0) or 0),
499 min_reviews=int(getattr(config.ci, "min_reviews", 0) or 0),
500 )
501 else:
502 plan = routing.standard_plan(usable_names)
503 log(routing.describe(plan))
504 seated = set(plan.panel)
505 round1 = [a for a in usable if a.name in seated]
506 benched = [a for a in usable if a.name in set(plan.benched)]
508 # State the number a downstream consumer will actually receive, BEFORE the
509 # panel is paid for (#699). "3 agents reviewing" is not that number: it is one
510 # review per agent that answers *and names what it read*, and the chair's
511 # synthesis record rides along beside them without being one. What is knowable
512 # here is only the ceiling — the line says "at most" — because a seat that
513 # returns nothing, or returns prose naming nothing checkable, abstains and is
514 # not a review (#700, round 2). When a consumer's minimum is configured and
515 # the bench cannot reach even the ceiling, this is a shortfall the run should
516 # name here rather than at the consumer, an hour and three CLI invocations
517 # later.
518 log(panel.describe(len(round1), available=len(usable)))
519 too_small = panel.shortfall(len(round1), config.ci.min_reviews, stage="before the panel runs")
520 if too_small:
521 raise RuntimeError(too_small)
523 # Round 1: independent reviews.
524 log(f"round 1: {len(round1)} agents reviewing")
525 review_prompt = {
526 a.name: tmpl["review"].format(
527 name=a.name,
528 context=prompts.neutralize_sentinels(review_context or "_(none)_"),
529 diff=prompts.neutralize_sentinels(diff),
530 policy=policy_section,
531 notice=prompts._UNTRUSTED_NOTICE,
532 excluded=prompts.binary_files_section(
533 [p for p, why in excluded_files if why == largediff.EXCLUDE_BINARY]
534 ),
535 )
536 for a in round1
537 }
538 reviews = _run_phase(
539 round1,
540 review_prompt,
541 "review",
542 config.parallel,
543 budget=budget,
544 retries=retries,
545 log=log,
546 )
547 # Stable ordering: the thread pool can return results in any completion
548 # order. Reorder to the enabled-agent order so the report (and every
549 # downstream consumer) is independent of which thread finished first.
550 agent_order = [a.name for a in round1]
551 reviews = _order_by_agents(reviews, agent_order)
553 # Parse structured findings from each successful review and aggregate them.
554 # Seed with the synthetic injection finding/warnings so they surface in the
555 # report and outcome.warnings without ever influencing agent behaviour.
556 all_findings: list[Finding] = list(injection_findings)
557 all_warnings: list[str] = injection.hits_to_warnings(injection_hits)
558 all_warnings.extend(privilege_warnings)
559 for r in reviews:
560 if not r.ok:
561 continue
562 found, warns = parse_findings(r.output, r.agent)
563 r.findings = found
564 r.warnings = warns
565 r.structured = emitted_findings_block(r.output)
566 all_findings.extend(found)
567 all_warnings.extend(warns)
568 if mode == "issue":
569 for f in all_findings:
570 f.origin = "unclear"
572 # Stream round-1 reviews as they're now finalized (stable order).
573 for r in reviews:
574 emit("review", r)
576 # Deterministic consensus grouping across reviewers.
577 groups = group_findings(all_findings, len(reviews))
579 # Names of agents whose round-1 review succeeded — the chair resolver uses
580 # this to (optionally) prefer a non-reviewer chair (#38).
581 reviewer_names = [r.agent for r in reviews if r.ok]
583 # Resolve the chair ONCE for the whole run so verify and synthesis use the
584 # SAME chair. ``chair = "rotate"`` and prefer-non-reviewer both consume the
585 # shared run RNG / reviewer info, so resolving once (rather than recomputing
586 # per phase) is what keeps a rotating chair stable within a run (#38).
587 # Escalation (#714): a critical or major finding after round 1 brings the
588 # benched frontier seats into the debate and draws the chair from the
589 # frontier seats. Decided once, here, and recorded on the plan — including
590 # what it can actually do in THIS run: a single-round run has no debate to
591 # join, and ``--auto`` sets exactly that on the routine band that benched
592 # the seats, so the record names the chair-only case rather than promising
593 # a debate that will not happen (review round 1).
594 round1_debaters = [a for a in round1 if any(r.agent == a.name and r.ok for r in reviews)]
595 if benched:
596 plan.escalated, plan.escalation_reason = routing.should_escalate(groups)
597 if plan.escalated:
598 log(f"tiered routing: escalating — {plan.escalation_reason}")
599 chair_pool = [a.name for a in round1]
600 if plan.escalated:
601 chair_pool = routing.frontier_names(specs, usable_names) or chair_pool
602 agent_order = agent_order + [a.name for a in benched]
603 chair_name = resolve_chair(config, chair_pool, reviewer_names, run_rng)
605 # Round 2+: debate. Only agents whose round-1 review succeeded participate.
606 # Two modes (issue #40):
607 # - fixed (early_stop = false): honour ``rounds`` exactly — run one debate
608 # round iff rounds >= 2. Reproducible fixed-N behaviour for benchmarking.
609 # - adaptive (early_stop = true): skip the debate when round-1 reviewers
610 # already agree, otherwise run debate up to ``max_rounds`` rounds and stop
611 # as soon as a round resolves all disputes.
612 debate: list[AgentResult] = []
613 rounds_executed = 1
614 stop_reason = ""
615 budget_exhausted = False
616 debaters = round1_debaters
617 if plan.escalated:
618 # Benched frontier seats cross-examine: they receive every round-1
619 # review as "the others" and answer in the debate format.
620 debaters = debaters + benched
621 can_debate = len(debaters) >= 2
623 if config.early_stop:
624 max_rounds = config.effective_max_rounds
625 if not can_debate:
626 stop_reason = "stopped after round 1: need >=2 successful reviews to debate"
627 log(stop_reason)
628 elif max_rounds < 2:
629 stop_reason = "stopped after round 1: max_rounds < 2"
630 log(stop_reason)
631 else:
632 converged, why = convergence.review_convergence(groups, len(reviews))
633 if converged:
634 stop_reason = f"early stop after round 1: {why}"
635 log(stop_reason)
636 else:
637 log(f"early stop active: {why}; running debate up to {max_rounds} round(s)")
638 prior: list[AgentResult] = []
639 round_no = 1
640 while round_no < max_rounds:
641 if budget.expired():
642 budget_exhausted = True
643 stop_reason = f"stopped at round {rounds_executed}: run budget exhausted"
644 log(stop_reason)
645 break
646 round_no += 1
647 debate = _debate_round(
648 debaters,
649 reviews,
650 diff,
651 config,
652 run_rng,
653 agent_order,
654 prior,
655 budget,
656 retries,
657 log,
658 round_no,
659 template=tmpl["debate"],
660 )
661 rounds_executed = round_no
662 for r in debate:
663 emit("debate", r, round_no)
664 dconv, dwhy = convergence.debate_convergence(debate)
665 if dconv:
666 stop_reason = f"converged after round {round_no}: {dwhy}"
667 log(stop_reason)
668 break
669 prior = debate
670 stop_reason = f"ran {round_no} rounds: {dwhy}"
671 else:
672 stop_reason = stop_reason or (
673 f"reached max_rounds ({max_rounds}) with disagreement remaining"
674 )
675 else:
676 # Fixed-N: exactly the historical behaviour.
677 if config.rounds >= 2 and can_debate:
678 if budget.expired():
679 budget_exhausted = True
680 stop_reason = "round 2 skipped: run budget exhausted"
681 log(stop_reason)
682 else:
683 debate = _debate_round(
684 debaters,
685 reviews,
686 diff,
687 config,
688 run_rng,
689 agent_order,
690 [],
691 budget,
692 retries,
693 log,
694 2,
695 template=tmpl["debate"],
696 )
697 rounds_executed = 2
698 for r in debate:
699 emit("debate", r, 2)
700 elif config.rounds >= 2:
701 stop_reason = "round 2 skipped: need >=2 successful reviews to debate"
702 log(stop_reason)
703 else:
704 stop_reason = "single round (rounds = 1)"
706 # Verification: the chair judges candidate findings to reduce false
707 # positives. Skipped when the run budget is exhausted (issue #30) so a
708 # partial run still returns what completed instead of overrunning.
709 verify_result: AgentResult | None = None
710 verdicts: list[Verdict] = []
711 if config.verify:
712 if budget.expired():
713 budget_exhausted = True
714 msg = "verification skipped: run budget exhausted"
715 log(msg)
716 all_warnings.append(msg)
717 else:
718 verify_result, verdicts, verify_warnings = _verify(
719 chair_name,
720 usable,
721 all_findings,
722 diff,
723 context,
724 budget,
725 retries,
726 log,
727 template=tmpl["verify"],
728 )
729 all_warnings.extend(verify_warnings)
730 _apply_verdicts(groups, verdicts)
731 if verify_result is not None:
732 emit("verify", verify_result)
734 # Local-only demotion (issue #442) runs AFTER verification, never before:
735 # _reject_targets' member-tier guard (orchestrator._reject_targets) assumes
736 # group.severity == max(member severities) to decide whether a rejecting
737 # verdict may suppress the whole group. Demoting group.severity earlier
738 # would desync it from that invariant and could let a verdict aimed at a
739 # minor local-only duplicate collateral-reject a genuinely critical,
740 # never-verified co-located finding merged into the same group.
741 if config.demote_local_only:
742 vendor_by_reviewer = {a.name: a.vendor for a in config.agents}
743 demote_local_only_groups(groups, vendor_by_reviewer)
745 # Synthesis: the chair consolidates. When the resolved chair is ALSO a
746 # round-1 reviewer, feed it an anonymized view of the reviews (#38 guardrail)
747 # so it cannot preferentially weight its own findings; the report still
748 # attributes by real name because it renders the real outcome data, not this
749 # synthesis prompt.
750 synthesis: AgentResult | None = None
751 if budget.expired():
752 budget_exhausted = True
753 msg = "synthesis skipped: run budget exhausted"
754 log(msg)
755 if msg not in all_warnings: 755 ↛ 777line 755 didn't jump to line 777 because the condition on line 755 was always true
756 all_warnings.append(msg)
757 else:
758 chair_is_reviewer = chair_name in reviewer_names
759 anonymize_synthesis = config.anonymize_debate and chair_is_reviewer
760 synthesis = _synthesize(
761 chair_name,
762 usable,
763 reviews,
764 debate,
765 diff,
766 budget,
767 retries,
768 log,
769 verdicts=verdicts,
770 anonymize_reviews=anonymize_synthesis,
771 rng=run_rng,
772 template=tmpl["synthesis"],
773 )
774 if synthesis is not None:
775 emit("synthesis", synthesis)
777 if plan.escalated:
778 # What escalation DID, read off the run (#714, review rounds 1 and 2).
779 # Predicting it was wrong twice: a single-round run has no debate to
780 # join — `--auto` sets exactly one round on the `low` band that benched
781 # the seats — and an adaptive run can converge after round 1 and skip
782 # the debate it was going to have. The benched seats that actually
783 # produced a debate result are the answer, whatever the reason.
784 bench_names = {a.name for a in benched}
785 # `ok` matters: a benched seat whose debate call failed produced a row
786 # but no cross-examination, and counting it would put the frontier seat
787 # in the record for work it did not do (review round 3).
788 joined = [r.agent for r in debate if r.agent in bench_names and r.ok]
789 effect = routing.escalation_effect(joined, debate_ran=bool(debate))
790 plan.escalation_reason = f"{plan.escalation_reason}; {effect}"
791 log(f"tiered routing: {plan.escalation_reason}")
793 return JuryOutcome(
794 reviews=reviews,
795 debate=debate,
796 synthesis=synthesis,
797 chair=chair_name,
798 findings=all_findings,
799 warnings=all_warnings,
800 groups=groups,
801 verify=verify_result,
802 verdicts=verdicts,
803 context_mode=context_mode,
804 redact_secrets=redact_on,
805 redaction_count=redaction_count,
806 injection_hits=injection_hits,
807 skipped=skipped,
808 budget_exhausted=budget_exhausted,
809 rounds_executed=rounds_executed,
810 stop_reason=stop_reason,
811 changed=changed,
812 routing=plan.as_dict(),
813 # Redacted like the diff: the paths reach the prompt, the report and the JSON.
814 excluded_files=[(redact(p)[0], why) for p, why in excluded_files],
815 )
818def resolve_chair(
819 config: JuryConfig,
820 usable: list[str],
821 reviewers: list[str],
822 rng: random.Random,
823) -> str:
824 """Resolve the chair for a run as a PURE function of its inputs (#38).
826 Precedence:
827 1. ``chair = "rotate"`` — pick deterministically from the usable agents
828 using the shared run ``rng``. Same seed -> same chair; different seeds
829 may differ. Falls back to the first usable agent when none are usable.
830 2. An explicit ``config.chair`` that names a usable agent — honoured as-is
831 (an operator-chosen chair always wins).
832 3. ``prefer_non_reviewer_chair`` — when set and a usable agent that was NOT
833 a successful round-1 reviewer exists, prefer the first such agent
834 (neutral chair). This only applies when the configured chair is not
835 itself a usable agent.
836 4. Fallback to the first usable agent (legacy behaviour).
838 Keeping this pure (no Adapter objects, no I/O) makes it directly
839 unit-testable and guarantees ``_verify`` and ``_synthesize`` agree because
840 the caller resolves it ONCE and threads the result through both.
841 """
842 if not usable:
843 return config.chair
844 names = set(usable)
846 if config.chair == "rotate":
847 # Deterministic rotation: sort for a stable candidate order independent
848 # of dict/thread ordering, then index with the shared run RNG. Sorting
849 # the candidate list (not iterating the set) makes the pick a pure
850 # function of (seed, usable-name set): same seed + same agents -> same
851 # chair, regardless of RNG-consumption order elsewhere.
852 candidates = sorted(names)
853 return candidates[rng.randrange(len(candidates))]
855 if config.chair in names:
856 return config.chair
858 if config.prefer_non_reviewer_chair:
859 reviewer_set = set(reviewers)
860 non_reviewers = [n for n in usable if n not in reviewer_set]
861 if non_reviewers:
862 return non_reviewers[0]
864 return usable[0]
867def _format_findings_for_verify(findings: list[Finding]) -> str:
868 """Render candidate findings for the chair's verification prompt.
870 Reviewer identity is omitted (#250) so the chair can't favour its own
871 findings while judging them — parity with the #37/#38 anonymization.
872 Verdicts match back by file/line/claim, so dropping it is safe.
873 """
874 if not findings:
875 return "_(no candidate findings)_"
876 lines = []
877 for f in findings:
878 loc = f.file or "?"
879 if f.line is not None:
880 loc = f"{loc}:{f.line}"
881 lines.append(f"- [{f.severity}] {loc} — {f.claim}")
882 return "\n".join(lines)
885def _format_verdicts(verdicts: list[Verdict]) -> str:
886 if not verdicts:
887 return "_(no verification verdicts)_"
888 lines = []
889 for v in verdicts:
890 loc = v.file or "?"
891 if v.line is not None:
892 loc = f"{loc}:{v.line}"
893 lines.append(f"- [{v.status}] {loc} — {v.claim}: {v.reasoning}")
894 return "\n".join(lines)
897def _verify(
898 chair_name,
899 usable,
900 findings,
901 diff,
902 context,
903 budget,
904 retries,
905 log,
906 template=prompts.VERIFY,
907) -> tuple[AgentResult | None, list[Verdict], list[str]]:
908 chair = next((a for a in usable if a.name == chair_name), None)
909 if chair is None:
910 return None, [], []
911 log(f"verification: chair '{chair_name}' judging {len(findings)} candidate findings")
912 prompt = template.format(
913 diff=prompts.neutralize_sentinels(diff),
914 findings=prompts.neutralize_sentinels(_format_findings_for_verify(findings)),
915 context=prompts.neutralize_sentinels(context or "_(none)_"),
916 notice=prompts._UNTRUSTED_NOTICE,
917 )
918 result = _run_with_retry(chair, prompt, "verify", budget, retries, log)
919 if not result.ok:
920 return result, [], [f"verification failed: {result.error}"]
921 verdicts, warnings = parse_verdicts(result.output, chair_name)
922 return result, verdicts, warnings
925def _verdict_matches_group(verdict: Verdict, group: FindingGroup) -> bool:
926 from .consensus import _normalize_claim, _normalize_path
928 rep = group.representative
929 # Case-EXACT path match (fold_case=False): on a case-sensitive filesystem
930 # ``Config.py`` != ``config.py``, so a verdict must not reject a finding it
931 # only case-collapses onto (audit 2026-06-13 r6/M).
932 if _normalize_path(verdict.file, fold_case=False) != _normalize_path(rep.file, fold_case=False):
933 return False
934 if verdict.line is not None and rep.line is not None and abs(verdict.line - rep.line) > 3:
935 return False
936 v_claim = _normalize_claim(verdict.claim)
937 r_claim = _normalize_claim(rep.claim)
938 if not v_claim:
939 # An empty verdict claim is allowed to match the finding *at this
940 # location* (the verifier may omit the claim and refer to it by
941 # position). But a verdict with NEITHER a claim NOR a line has no
942 # location precision at all: it would otherwise match — and, when
943 # ``unsupported``, REJECT — every finding group in the file, including
944 # unrelated criticals, flipping the CI gate from FAIL to PASS. Such a
945 # claim-less, line-less verdict is a file-wide wildcard and must not
946 # match (security audit 2026-06-13 r6/M). Require a concrete line that
947 # actually pins the finding before honoring an empty-claim match.
948 return verdict.line is not None and rep.line is not None
949 if v_claim == r_claim:
950 return True
951 v_tokens, r_tokens = set(v_claim.split()), set(r_claim.split())
952 if not v_tokens or not r_tokens:
953 return False
954 inter = len(v_tokens & r_tokens)
955 union = len(v_tokens) + len(r_tokens) - inter
956 return (inter / union if union else 0.0) >= 0.5
959def _claim_sim(a_claim: str, b_claim: str) -> float:
960 """Token-set similarity between two claims: 1.0 exact, else Jaccard, 0.0 if
961 either side is empty."""
962 from .consensus import _normalize_claim
964 a = _normalize_claim(a_claim)
965 b = _normalize_claim(b_claim)
966 if not a or not b:
967 return 0.0
968 if a == b:
969 return 1.0
970 at, bt = set(a.split()), set(b.split())
971 inter = len(at & bt)
972 union = len(at) + len(bt) - inter
973 return (inter / union) if union else 0.0
976# Verdict statuses that move a finding into a non-blocking bucket (suppress it).
977_REJECTING_STATUSES = frozenset({"unsupported", "needs_human_decision"})
978# Apply most-blocking statuses first so a contradictory verdict pair on one
979# finding is fail-closed: a `verified` (blocking) judgement is recorded before
980# any `unsupported`/`needs_human_decision` and cannot then be flipped to
981# non-blocking by verdict array ordering (audit 2026-06-13 r8/M).
982_STATUS_PRIORITY = {"verified": 0, "needs_human_decision": 1, "unsupported": 2}
983# Minimum claim similarity for a verdict to be considered "about" a finding at
984# all. The PRIMARY defence against a verdict dismissing a co-located *distinct*
985# finding is that a rejection attaches to AT MOST the single best-matching group
986# (`_best_reject_target`), so a verdict whose claim copies a benign neighbour
987# routes to that neighbour, not to the co-located critical (audit 2026-06-13
988# r8/M). The threshold stays moderate so the verifier's legitimate paraphrased
989# rejections (it drops the reviewer-name prefix etc.) still apply.
990_REJECT_CLAIM_THRESHOLD = 0.5
993def _reject_targets(verdict: Verdict, groups: list[FindingGroup]) -> list[FindingGroup]:
994 """Un-statused groups a rejecting verdict may suppress (fail-closed, r7/r8).
996 Defences (each closes a distinct collateral-rejection vector found across
997 audit rounds 6-9):
999 0. **Line required.** A rejecting verdict must pin a concrete line. A
1000 line-less verdict is too imprecise to safely suppress a finding and would
1001 act as a file-wide-by-claim wildcard (audit r9/M, the claim-ful
1002 counterpart of the round-6 line-less-wildcard fix).
1003 1. **Member-tier guard.** A group may merge findings of different severities
1004 (consensus keeps the max). A verdict is "about" the member whose claim it
1005 best matches; if that member is *less severe* than the group's max, the
1006 verdict is dismissing a lesser co-located finding and must NOT suppress
1007 the (e.g. critical) group.
1008 2. **Best-tier only.** Across candidate groups, suppress only those at the
1009 highest match similarity — a verdict copying a benign neighbour rejects
1010 that neighbour (and its duplicate phrasings, which tie) but not a
1011 separate, less-similar critical group.
1012 3. **Least-severe within a tie.** If the best-similarity tier still spans
1013 severities (an exact `_claim_sim` tie between a critical and a benign
1014 decoy), suppress only the *least*-severe groups — a tie must never drag a
1015 critical down alongside a decoy (audit r9/M).
1016 """
1017 from .findings import SEVERITY_ORDER
1019 if verdict.line is None:
1020 return []
1021 scored: list[tuple[float, FindingGroup]] = []
1022 for group in groups:
1023 if group.status:
1024 continue
1025 if not _verdict_matches_group(verdict, group):
1026 continue
1027 members = getattr(group, "members", None) or [group.representative]
1028 best_sim, best_member = max(
1029 ((_claim_sim(verdict.claim, m.claim), m) for m in members),
1030 key=lambda t: t[0],
1031 )
1032 if best_sim < _REJECT_CLAIM_THRESHOLD:
1033 continue
1034 # Member-tier guard: refuse if the verdict best-names a member less
1035 # severe than the group's max severity (lower rank = more severe).
1036 if SEVERITY_ORDER.get(best_member.severity, 99) > SEVERITY_ORDER.get(group.severity, 99):
1037 continue
1038 scored.append((best_sim, group))
1039 if not scored:
1040 return []
1041 best = max(sim for sim, _ in scored)
1042 tier = [(sim, group) for sim, group in scored if sim >= best]
1043 # Within the top-similarity tier, keep only the least-severe groups.
1044 least_rank = max(SEVERITY_ORDER.get(g.severity, 99) for _, g in tier)
1045 return [g for _, g in tier if SEVERITY_ORDER.get(g.severity, 99) == least_rank]
1048def _apply_verdicts(groups: list[FindingGroup], verdicts: list[Verdict]) -> None:
1049 """Attach verification statuses to consensus groups.
1051 unsupported -> bucket 'rejected'; needs_human_decision -> bucket 'disputed';
1052 verified -> status recorded, bucket unchanged.
1053 """
1054 # Stable-sort by blocking priority so contradictions resolve fail-closed.
1055 for verdict in sorted(verdicts, key=lambda v: _STATUS_PRIORITY.get(v.status, 3)):
1056 if verdict.status in _REJECTING_STATUSES:
1057 # Suppress only the best-similarity tier this verdict names — never
1058 # collaterally a co-located, less-similar distinct finding.
1059 bucket = "rejected" if verdict.status == "unsupported" else "disputed"
1060 for target in _reject_targets(verdict, groups):
1061 target.status = verdict.status
1062 target.status_reasoning = verdict.reasoning
1063 target.bucket = bucket
1064 continue
1065 # A verifying (non-suppressing) verdict may attach to every matching
1066 # group: when reviewers phrase the same issue differently it can land in
1067 # more than one group, and all should carry the judgement.
1068 for group in groups:
1069 if group.status:
1070 continue
1071 if _verdict_matches_group(verdict, group):
1072 group.status = verdict.status
1073 group.status_reasoning = verdict.reasoning
1076def _synthesize(
1077 chair_name,
1078 usable,
1079 reviews,
1080 debate,
1081 diff,
1082 budget,
1083 retries,
1084 log,
1085 verdicts=None,
1086 anonymize_reviews=False,
1087 rng=None,
1088 template=prompts.SYNTHESIS,
1089) -> AgentResult | None:
1090 chair = next((a for a in usable if a.name == chair_name), None)
1091 if chair is None:
1092 return None
1093 log(f"synthesis: chair '{chair_name}' consolidating verdict")
1094 if anonymize_reviews:
1095 # Chair self-preference guardrail (#38): present round-1 reviews to the
1096 # chair under anonymous labels (no agent/vendor identity, no stable
1097 # order) so it cannot tell which review is "its own". Uses the shared run
1098 # RNG for deterministic-but-unstable ordering. ``me=None`` keeps ALL
1099 # reviews (we are not excluding a debater here, only stripping identity).
1100 peer_rng = random.Random(rng.random()) if rng is not None else random.Random()
1101 reviews_txt, _label_map = _anonymize_peers(reviews, None, peer_rng)
1102 else:
1103 reviews_txt = (
1104 "\n\n".join(
1105 f"### {r.agent} ({r.vendor})\n{r.output}" for r in reviews if r.ok and r.output
1106 )
1107 or "_(no reviews)_"
1108 )
1109 debate_txt = (
1110 "\n\n".join(f"### {r.agent}\n{r.output}" for r in debate if r.ok and r.output)
1111 or "_(no debate round)_"
1112 )
1113 prompt = template.format(
1114 diff=prompts.neutralize_sentinels(diff),
1115 reviews=prompts.neutralize_sentinels(reviews_txt),
1116 debate=prompts.neutralize_sentinels(debate_txt),
1117 notice=prompts._UNTRUSTED_NOTICE,
1118 )
1119 if verdicts:
1120 # The verdicts quote candidate findings, which transitively quote
1121 # untrusted diff text (issue v1.5.0/M-1: this addendum was the one slot
1122 # the #316/L-1 fix missed). Fence + neutralize it like every other
1123 # peer-output slot so an embedded closing token can't break out.
1124 prompt += (
1125 "\n\n=== VERIFICATION VERDICTS (may quote UNTRUSTED text) ===\n"
1126 "<<<UNTRUSTED_FINDINGS\n"
1127 + prompts.neutralize_sentinels(_format_verdicts(verdicts))
1128 + "\nUNTRUSTED_FINDINGS>>>\n"
1129 )
1130 return _run_with_retry(chair, prompt, "synthesis", budget, retries, log)
1133def _first_sent_model(parts: list[AgentResult]) -> str:
1134 """The first model id *parts* recorded sending (issue #722).
1136 Every chunk of one diff is one seat invoked repeatedly through one adapter,
1137 so the parts normally all carry the same stamped id (#709) and "the first
1138 non-empty one" is simply "the one". Where they disagree — which only an
1139 adapter that consults a live model listing between invocations can produce
1140 — there is no id that is true of the whole merge, and inventing one
1141 ("mixed") would be a string no invocation sent. The first is a model this
1142 run really did send, and it is the one whose output opens the merged body,
1143 so the ballot's id and the text a reader checks it against come from the
1144 same invocation.
1146 **Callers pass the parts whose output is in the merged body**, because that
1147 is what the id has to be true of. A failed part is not excluded by carrying
1148 ``""``: :func:`_run_with_retry` stamps ``result.model`` from
1149 ``adapter.resolved_model()`` after ``adapter.run`` returns, whether the run
1150 succeeded or not, so a chunk that failed *does* normally record an id — and
1151 it can be a different one from its siblings', since the fallback an adapter
1152 performs against a live listing is exactly the kind of failure that changes
1153 it. Scanning all the parts would then let a chunk that contributed no text
1154 name the model for text it did not produce.
1156 A part carries ``""`` only when nothing stamped it: a hand-assembled
1157 outcome, a pre-#709 record, or a failure raised before the adapter returned.
1158 A merge in which no scanned part recorded an id stays empty, which
1159 :func:`ai_jury.ballots.describe_model` labels ``recomputed`` rather than
1160 ``requested``. That fallback is unchanged.
1161 """
1162 for p in parts:
1163 model = (getattr(p, "model", "") or "").strip()
1164 if model:
1165 return model
1166 return ""
1169def _merge_results_by_agent(phase_lists: list[list[AgentResult]]) -> list[AgentResult]:
1170 """Merge per-chunk results for the same agent into one result (issue #31).
1172 Outputs are concatenated under per-chunk headers, durations summed, ``ok`` is
1173 true if the agent succeeded on any chunk, and ``attempts`` keeps the max so a
1174 retried chunk is still visible. Agent order follows first appearance.
1176 The model id the invocation sent (:attr:`AgentResult.model`, #709) rides
1177 along too — see :func:`_first_sent_model`. Dropping it here made a chunked
1178 review's provenance strictly weaker than a single-chunk one's (#722). It is
1179 read from the same parts the body is built from: the id has to be one that
1180 produced the text under it, and a chunk can fail *after* recording a
1181 different id than its siblings (an adapter that fell back against a live
1182 listing). Only when no part contributed body text at all — a seat that
1183 failed on every chunk — is the whole set scanned, so a failed seat still
1184 reports what it sent.
1185 """
1186 order: list[str] = []
1187 by_agent: dict[str, list[AgentResult]] = {}
1188 for lst in phase_lists:
1189 for r in lst:
1190 if r.agent not in by_agent:
1191 by_agent[r.agent] = []
1192 order.append(r.agent)
1193 by_agent[r.agent].append(r)
1195 merged: list[AgentResult] = []
1196 for name in order:
1197 parts = by_agent[name]
1199 # bolt: Consolidate multiple metrics (ok, body, total_duration, max_attempts)
1200 # into a single-pass O(N) explicit loop to bypass multiple generator instantiations
1201 ok = False
1202 body_parts = []
1203 body_sources: list[AgentResult] = []
1204 first_err = None
1205 total_duration = 0.0
1206 max_attempts = 0
1208 for i, p in enumerate(parts, 1):
1209 if p.ok:
1210 ok = True
1211 if p.output:
1212 body_parts.append(f"#### chunk {i}\n{p.output}")
1213 body_sources.append(p)
1214 elif first_err is None:
1215 first_err = p
1217 total_duration += p.duration_s
1218 if p.attempts > max_attempts:
1219 max_attempts = p.attempts
1221 body = "\n\n".join(body_parts)
1223 merged.append(
1224 AgentResult(
1225 name,
1226 parts[0].vendor,
1227 ok,
1228 body,
1229 round(total_duration, 3),
1230 error=None if ok else (first_err.error if first_err else None),
1231 error_code=None if ok else (first_err.error_code if first_err else None),
1232 attempts=max_attempts,
1233 model=_first_sent_model(body_sources or parts),
1234 )
1235 )
1236 return merged
1239def _combine_chair_results(results: list[AgentResult], chair: str) -> AgentResult | None:
1240 """Combine per-chunk chair results (verify/synthesis) into one labelled result.
1242 The combined record keeps the model id the chair's invocation sent (#722),
1243 taken from the same population as ``vendor`` — the parts whose output is in
1244 the body — by :func:`_first_sent_model`.
1245 """
1246 ok_parts = [r for r in results if r.ok and r.output]
1247 if not ok_parts:
1248 return results[0] if results else None
1249 vendor = ok_parts[0].vendor
1251 # bolt: Consolidate body text concatenation and duration sum into a single-pass O(N) loop
1252 body_parts = []
1253 total_duration = 0.0
1254 for i, r in enumerate(ok_parts, 1):
1255 body_parts.append(f"### chunk {i}\n{r.output}")
1256 total_duration += r.duration_s
1258 body = "\n\n".join(body_parts)
1259 return AgentResult(
1260 chair, vendor, True, body, round(total_duration, 3), model=_first_sent_model(ok_parts)
1261 )
1264def _merge_chunk_outcomes(outcomes: list[JuryOutcome], config: JuryConfig) -> JuryOutcome:
1265 """Fold per-chunk outcomes (issue #31) into one renderable JuryOutcome.
1267 Findings are unioned and re-grouped across all chunks so the consensus view
1268 is global; verdicts are re-applied to the merged groups. Review/debate/chair
1269 outputs are merged per agent with chunk labels so the report stays coherent.
1270 """
1271 if len(outcomes) == 1:
1272 return outcomes[0]
1273 base = outcomes[0]
1275 reviews = _merge_results_by_agent([o.reviews for o in outcomes])
1276 debate = (
1277 _merge_results_by_agent([o.debate for o in outcomes])
1278 if any(o.debate for o in outcomes)
1279 else []
1280 )
1281 findings = [f for o in outcomes for f in o.findings]
1282 groups = group_findings(findings, len(reviews))
1283 verdicts = [v for o in outcomes for v in o.verdicts]
1284 # Scope each chunk's verdicts to that chunk's own findings. A verdict is
1285 # produced while verifying ONE chunk (whose prompt held only that chunk's
1286 # findings, but whose attacker-controlled diff text could steer it); after
1287 # the global merge an unscoped verdict could reject a *different* chunk's
1288 # structured critical and flip the CI gate (audit 2026-06-13 r7/M). Chunks
1289 # are file-disjoint, so apply each chunk's verdicts only to groups whose
1290 # location is one of that chunk's files.
1291 from .consensus import _normalize_path
1293 for o in outcomes:
1294 chunk_files = {_normalize_path(f.file, fold_case=False) for f in o.findings if f.file}
1295 chunk_groups = [
1296 g
1297 for g in groups
1298 if _normalize_path(g.representative.file, fold_case=False) in chunk_files
1299 ]
1300 _apply_verdicts(chunk_groups, o.verdicts)
1302 # Runs AFTER verdicts are applied — see the matching comment in run_jury for
1303 # why (the _reject_targets member-tier guard assumes group.severity is the
1304 # true max member severity; demoting earlier would desync that invariant).
1305 if config.demote_local_only:
1306 vendor_by_reviewer = {a.name: a.vendor for a in config.agents}
1307 demote_local_only_groups(groups, vendor_by_reviewer)
1309 warnings = [w for o in outcomes for w in o.warnings]
1311 # ONE chair name for the whole merged record (#714, r5). A chunked tiered
1312 # run escalates per chunk, so the chunks can have different chairs; the run
1313 # publishes the chair of the first chunk that escalated, and the combined
1314 # synthesis and verify must carry that same name — labelling them with the
1315 # quiet first chunk's seat while the outcome names another is a report that
1316 # says one seat synthesised a body another seat's chunk opens.
1317 merged_chair = next(
1318 (o.chair for o in outcomes if (o.routing or {}).get("escalated")), base.chair
1319 )
1320 synthesis = _combine_chair_results([o.synthesis for o in outcomes if o.synthesis], merged_chair)
1321 verify = _combine_chair_results([o.verify for o in outcomes if o.verify], merged_chair)
1323 # bolt: Consolidate collection aggregations (sum, extend, max, any) into a single-pass O(N) explicit loop
1324 redaction_count = 0
1325 injection_hits = []
1326 budget_exhausted = False
1327 rounds_executed = 0
1329 for o in outcomes:
1330 redaction_count += o.redaction_count
1331 injection_hits.extend(o.injection_hits)
1332 if o.budget_exhausted:
1333 budget_exhausted = True
1334 if o.rounds_executed > rounds_executed:
1335 rounds_executed = o.rounds_executed
1337 return JuryOutcome(
1338 reviews=reviews,
1339 debate=debate,
1340 synthesis=synthesis,
1341 # A chunked run escalates per chunk, so a quiet first chunk must not
1342 # publish its economical chair for a run that escalated later: the
1343 # chair of the first chunk that escalated is the run's (#714, r4), and
1344 # the combined synthesis and verify above carry the same name (r5).
1345 chair=merged_chair,
1346 findings=findings,
1347 warnings=warnings,
1348 groups=groups,
1349 verify=verify,
1350 verdicts=verdicts,
1351 context_mode=base.context_mode,
1352 redact_secrets=base.redact_secrets,
1353 redaction_count=redaction_count,
1354 injection_hits=injection_hits,
1355 skipped=base.skipped,
1356 budget_exhausted=budget_exhausted,
1357 rounds_executed=rounds_executed,
1358 stop_reason=f"chunked review across {len(outcomes)} part(s)",
1359 # The whole change, not one chunk of it (#710): a reviewer that named a
1360 # file from another chunk named a file in this change.
1361 changed=largediff.merge_change_indexes([o.changed for o in outcomes]),
1362 # Every chunk routed off the same band and the same bench, so the first
1363 # chunk's plan describes the run; escalation is true when ANY chunk
1364 # escalated, because the benched seats really did review by then.
1365 routing=_merged_routing(outcomes, base, debate),
1366 excluded_files=list(base.excluded_files),
1367 )
1370def _merged_routing(outcomes: list[JuryOutcome], base: JuryOutcome, debate: list) -> dict:
1371 """The routing record of a chunked run, read off the merged run (#714, r3).
1373 Every chunk routed off the same band and the same bench, so the first
1374 chunk's plan describes the panel. Escalation is per chunk — round 1 of one
1375 chunk can carry a major finding while another is quiet — so the record says
1376 on how many it escalated, and what escalation then did is recomputed from
1377 the **merged** debate rather than copied from whichever chunk escalated
1378 first. Copying it was wrong the way predicting was wrong: the sentence
1379 described one chunk and was published as the run's.
1380 """
1381 escalated = [o for o in outcomes if (o.routing or {}).get("escalated")]
1382 merged = {**base.routing, "escalated": bool(escalated)}
1383 if escalated:
1384 bench = set(base.routing.get("benched") or [])
1385 joined = [r.agent for r in debate if r.agent in bench and r.ok]
1386 merged["escalation_reason"] = (
1387 f"escalated on {len(escalated)} of {len(outcomes)} chunk(s); "
1388 f"{routing.escalation_effect(joined, debate_ran=bool(debate))}"
1389 )
1390 return merged
1393def plan_for(config: JuryConfig, diff: str) -> largediff.DiffPlan:
1394 """The diff plan *config* selects for *diff* (pure).
1396 The one place that maps ``[jury.diff]`` onto :func:`largediff.plan_diff`, so
1397 every caller that needs to know which files the panel will actually be shown
1398 — :func:`review_diff` below, and the CLI's static-hints pre-pass (#737) —
1399 reads the same ``kept`` list off the same filters. ``plan_diff`` is pure, so
1400 planning the same diff twice returns the same answer.
1401 """
1402 dc = config.diff
1403 return largediff.plan_diff(
1404 diff,
1405 max_bytes=dc.max_bytes,
1406 chunk=dc.chunk,
1407 chunk_max_bytes=dc.chunk_max_bytes,
1408 exclude_generated=dc.exclude_generated,
1409 exclude=dc.exclude,
1410 include=dc.include,
1411 )
1414def review_diff(
1415 config: JuryConfig,
1416 diff: str,
1417 *,
1418 context: str = "",
1419 hints: str = "",
1420 mock: bool = False,
1421 strict: bool = False,
1422 seed: int | None = None,
1423 policy: ReviewPolicy | None = None,
1424 log=lambda _msg: None,
1425 on_event=None,
1426) -> tuple[JuryOutcome, largediff.DiffPlan]:
1427 """Plan a diff (filter + size + mode) then run the jury (issue #31).
1429 The single entry point the CLI uses: it measures and filters the diff,
1430 reports the size and the selected handling mode, and dispatches:
1432 - ``full`` — review the filtered diff in one ``run_jury`` pass;
1433 - ``chunked`` — review each chunk and merge the outcomes;
1434 - ``too_large`` — raise ``RuntimeError`` with an actionable message.
1436 Returns ``(outcome, plan)`` so the caller can surface the plan. Existing
1437 callers of :func:`run_jury` are unaffected.
1438 """
1439 plan = plan_for(config, diff)
1440 log(
1441 f"diff size: {plan.total_bytes} B total, {plan.kept_bytes} B after filters "
1442 f"({len(plan.kept)} file(s) kept, {len(plan.excluded)} excluded); "
1443 f"mode: {plan.mode}"
1444 )
1445 if plan.excluded:
1446 log("excluded: " + ", ".join(f"{redact(p)[0]} [{why}]" for p, why in plan.excluded))
1447 log(plan.reason)
1449 if plan.mode == largediff.MODE_TOO_LARGE:
1450 raise RuntimeError(f"diff too large to review: {plan.reason}")
1451 if not plan.chunks:
1452 binaries = prompts.clean_paths(
1453 [p for p, why in plan.excluded if why == largediff.EXCLUDE_BINARY]
1454 )
1455 others = len(plan.excluded) - len(
1456 [1 for _p, why in plan.excluded if why == largediff.EXCLUDE_BINARY]
1457 )
1458 hint = "(check [jury.diff] include/exclude patterns)"
1459 if binaries:
1460 shown = binaries[: prompts.BINARY_LIST_CAP]
1461 more = len(binaries) - len(shown)
1462 listing = (
1463 f"{len(binaries)} binary file(s) cannot be reviewed as text: "
1464 + ", ".join(shown)
1465 + (f" (+{more} more)" if more > 0 else "")
1466 )
1467 raise RuntimeError(
1468 "nothing to review after filters — "
1469 + listing
1470 + (f"; {others} other file(s) were excluded {hint}" if others else "")
1471 )
1472 raise RuntimeError(f"nothing to review after filters — all files were excluded {hint}")
1474 # One shared budget across all chunks so ``total_timeout`` bounds the WHOLE
1475 # review, not each chunk independently (review finding). ``phase_timeout`` and
1476 # per-agent timeouts still apply per call via the same budget.
1477 shared_budget = RunBudget(config.total_timeout, config.phase_timeout)
1479 # Redact the shared context ONCE here, before fan-out (#249). The same context
1480 # is reviewed against every chunk; letting each per-chunk ``run_jury`` redact
1481 # it would count its secrets once per chunk and ``_merge_chunk_outcomes`` would
1482 # sum them, inflating ``redaction_count`` (e.g. a 1-secret context over 8
1483 # chunks reported 8). Pre-redacting makes each chunk's re-redaction a no-op —
1484 # the ``[REDACTED:…]`` placeholders no longer match — so we add the one-time
1485 # context count back at the end. Diff/chunk redactions are still counted
1486 # per chunk and summed, which is correct (each chunk's diff is distinct).
1487 # `config.context` is the right path: `_from_dict` flattens the `[jury]`
1488 # table onto JuryConfig, so the `[jury.context]` sub-table is `config.context`
1489 # (a ContextConfig), NOT `config.jury.context` — there is no `config.jury`.
1490 # This mirrors how run_jury() reads it.
1491 ctx_cfg = getattr(config, "context", None)
1492 ctx_mode = getattr(ctx_cfg, "mode", "diff-only") if ctx_cfg else "diff-only"
1493 redact_on = getattr(ctx_cfg, "redact_secrets", True) if ctx_cfg else True
1494 context_redactions = 0
1495 if redact_on and ctx_mode != "diff-only" and context:
1496 context, context_redactions = redact(context)
1498 # One band for the whole change, computed on the FILTERED diff the panel
1499 # will actually see — the excluded files are not part of the change under
1500 # review, and profiling the raw argument would route off files the panel is
1501 # never shown (#714, r4). Every chunk then routes off the same answer.
1502 whole_risk = (
1503 profile_diff("".join(plan.chunks)).risk if config.routing == routing.MODE_TIERED else None
1504 )
1506 def _run(chunk: str) -> JuryOutcome:
1507 return run_jury(
1508 config,
1509 chunk,
1510 context=context,
1511 hints=hints,
1512 mock=mock,
1513 strict=strict,
1514 seed=seed,
1515 policy=policy,
1516 log=log,
1517 budget=shared_budget,
1518 on_event=on_event,
1519 risk=whole_risk,
1520 excluded_files=plan.excluded,
1521 )
1523 def _finalize(outcome: JuryOutcome) -> JuryOutcome:
1524 # Add the one-time context redaction count (per-chunk runs saw an already-
1525 # redacted context and counted 0 for it).
1526 if not context_redactions:
1527 return outcome
1528 return replace(outcome, redaction_count=outcome.redaction_count + context_redactions)
1530 if plan.mode == largediff.MODE_FULL:
1531 return _finalize(_run(plan.chunks[0])), plan
1533 outcomes = []
1534 for i, chunk in enumerate(plan.chunks, 1):
1535 log(f"reviewing chunk {i}/{len(plan.chunks)}")
1536 outcomes.append(_run(chunk))
1537 return _finalize(_merge_chunk_outcomes(outcomes, config)), plan