Coverage for src/ai_jury/metadata.py: 99%
108 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
1"""Run metadata and cost-awareness (wall-clock proxy) reporting.
3Builds a machine-readable metadata dict describing a jury run: which
4agents participated, their per-agent status and wall-clock duration, how many
5rounds ran, whether verification was enabled, and timestamps.
7IMPORTANT: This metadata deliberately contains NO diff text, NO prompt text,
8NO agent output, and NO secrets -- only structural/operational signals.
10There are no token counts available from the underlying CLIs, so wall-clock
11seconds are used as an approximate cost *proxy*, not a dollar cost.
12"""
14from __future__ import annotations
16from datetime import UTC, datetime
17from typing import TYPE_CHECKING
19from . import panel
20from .config import DEFAULT_MIN_VENDORS, normalise_vendor, vendor_identity
22if TYPE_CHECKING: # pragma: no cover - typing only
23 from .config import JuryConfig
24 from .orchestrator import JuryOutcome
26# v2 (issue #30/#40) added: stop_reason, skipped, retried, budget_exhausted,
27# execution{...}, and per-agent ``attempts``.
28# v4 (issue #501) added: ``panel`` (configured vs effective size, abstentions) and
29# per-agent ``review_status``. A slot that returns no review is an abstention, not an
30# approval, and until now nothing in the output said so.
31# v5 (issues #699/#700) added, inside ``panel``: ``ballots``, ``reviews_supplied``,
32# ``silent``, ``insubstantial``, ``refused``, ``adapter_failed``, ``chair`` and
33# ``chair_ballot`` — the number of reviews a downstream consumer actually
34# receives (the ballots that named something and voted; the chair's synthesis
35# record is not one of them, and neither is an abstaining ballot), the ways
36# a seat that ran produced no review, and whether the chairing agent's own ballot
37# is one of the counted reviews. The causes and ``reviews_supplied`` sum to
38# ``ballots``. Purely additive: every v4 key keeps its name and meaning.
39# v6 (issue #710) added, inside ``panel``: ``not_in_change`` — the seats whose
40# stated scope named only paths or symbols the change does not contain. Split
41# out of ``insubstantial``, which keeps meaning exactly "answered and named
42# nothing checkable": naming `src/made/up.py` is a different failure from naming
43# nothing, and the two ask for different fixes. Additive, and the buckets still
44# sum to ``ballots`` — the seats moved between buckets, none was added or lost.
45# v7 (issue #714) added ``routing``.
46# v8 (issue #863) added, inside ``panel``: ``zero_config_fallback`` — ``true`` when
47# the run had no config and no usable agent CLI, so the panel was the one local
48# seat the zero-config fallback seated. That run is single-vendor, and the default
49# cross-vendor guard does not fail it; this says so where ``--quiet`` cannot hide
50# it. Always present. Additive: every v7 key keeps its name and meaning.
51SCHEMA_VERSION = 8
54#: What a reviewer slot actually contributed (issue #501). ``clean`` and
55#: ``abstained`` both carry zero findings and used to be reported identically,
56#: which is how a run with two non-reviewing slots still described itself as a
57#: three-agent panel.
58REVIEW_STATUSES = ("findings", "clean", "abstained", "failed")
61def review_status(result) -> str:
62 """Classify one reviewer slot's contribution. Pure, and never judges content.
64 * ``failed`` — the adapter did not return a result at all.
65 * ``findings`` — produced at least one structured finding.
66 * ``clean`` — emitted a findings block that was empty: examined, found nothing.
67 * ``abstained`` — returned successfully with no findings block. Not an approval:
68 nothing reviewable came back, so this slot contributed no evidence either way.
69 """
70 if not getattr(result, "ok", False):
71 return "failed"
72 if getattr(result, "findings", None):
73 return "findings"
74 return "clean" if getattr(result, "structured", False) else "abstained"
77def panel_accounting(reviews, chair: str = "", ballots=None) -> dict:
78 """Configured versus *effective* panel size, and the per-status breakdown.
80 A consumer gating on the panel needs the effective number — keel downgrades a
81 jury to advisory below two participating vendors, and can only do that if the
82 report says the panel was short. ``vendors`` counts distinct vendors that
83 actually contributed a review, which is the number that matters for
84 cross-vendor consensus: three slots from one vendor are not three perspectives.
86 Vendors are counted by :func:`config.vendor_identity`, not by the raw
87 string: a seat whose vendor the tool does not recognise ran on the generic
88 ``cli`` fallback and is counted as ``cli``, so two unidentifiable seats are
89 one vendor here even though the report still names each one honestly
90 (issue #701).
92 ``reviews_supplied`` is a different number again, and the one #699 was about:
93 how many reviews the bundle hands on. It is not ``configured`` (every seat
94 that ran, silent ones included), not ``effective`` (a slot that returned
95 prose but no findings block may still have named nothing), and not
96 ``ballots`` plus the chair record — a consumer splits the ``reviewers`` array
97 on ``role`` and reads the ``chair`` entry as the panel's consensus rather
98 than as one more review.
100 It is :func:`ai_jury.panel.review_count` over the **ballot records**, which
101 is why they are passed in rather than derived here: whether a seat reviewed
102 is a fact about the record it produced — its scope and its verdict — and no
103 predicate over the raw round-1 result can see it. That was the defect at this
104 line (#700, round 2): a prose-only non-review abstained on its ballot and was
105 still counted here, so ``--min-reviews`` could be satisfied by seats that
106 reviewed nothing. With ``ballots`` omitted the count is reported as ``None``
107 rather than guessed, because every guess available over-counts.
109 ``silent``, ``insubstantial``, ``refused`` and ``adapter_failed`` split the
110 seats that produced no review by cause, for the shortfall message: a silent
111 agent is a CLI that broke or a budget that ran out, a seat that answered and
112 named nothing is a reviewer that did not review, a refusal is a model
113 declining the task, and a failed adapter is an invocation to fix. They come
114 from :func:`ai_jury.panel.abstention_buckets` — each ballot classified by the
115 cause it carries — so that they and ``reviews_supplied`` add up to
116 ``ballots`` with nothing left over.
118 ``insubstantial`` was derived by subtraction until #700's fifth round, and
119 the subtraction was the defect: ``ballots - silent - supplied`` swept up
120 every seat that was neither silent nor a counted review, and once a ballot
121 could carry a substantive scope and still abstain that included the seat that
122 named a file and then refused, and the one whose adapter died holding a file
123 name. Both were then rendered as "named nothing checkable" — a cause their
124 own ballot contradicted. It now means exactly ``named_nothing``: the seats
125 that answered and named nothing a reader could check, and only those.
127 ``chair_ballot`` says whether the chairing agent's own ballot is one of the
128 *counted* reviews — a chairing agent that ran and abstained has a ballot in
129 the bundle and has supplied no review.
130 """
131 reviews = list(reviews or [])
132 seats = panel.ballot_seats(reviews)
133 records = list(ballots) if ballots is not None else None
134 counted = [r for r in (records or []) if panel.is_review(r)]
135 supplied = len(counted) if records is not None else None
136 # Without the records there is nothing to classify: the cause is a fact the
137 # ballot carries, and every guess available from the raw results is the
138 # subtraction this replaced. ``silent`` is the exception, and only because
139 # :func:`ai_jury.ballots.abstention_cause` classifies silence on the same
140 # predicate (:func:`ai_jury.panel.silent_seat`: no output and no allowance
141 # failure) — the two readings are the same number by construction.
142 buckets = panel.abstention_buckets(records) if records is not None else None
143 silent = sum(1 for r in seats if panel.silent_seat(r))
145 # bolt: Consolidate multiple metrics into a single-pass O(N) explicit loop
146 effective_count = 0
147 abstained_count = 0
148 failed_count = 0
149 contributing_vendors = set()
151 for r in reviews:
152 st = review_status(r)
153 if st in ("findings", "clean"):
154 effective_count += 1
155 vendor = vendor_identity(getattr(r, "vendor", ""))
156 if vendor: 156 ↛ 151line 156 didn't jump to line 151 because the condition on line 156 was always true
157 contributing_vendors.add(vendor)
158 elif st == "abstained":
159 abstained_count += 1
160 elif st == "failed": 160 ↛ 151line 160 didn't jump to line 151 because the condition on line 160 was always true
161 failed_count += 1
163 return {
164 "configured": len(reviews),
165 "effective": effective_count,
166 "vendors": len(contributing_vendors),
167 "abstained": abstained_count,
168 "failed": failed_count,
169 "short": effective_count < len(reviews),
170 # One ballot record per seat that ran — including a silent one, recorded
171 # as an abstention so the report can name it. Deliberately no longer the
172 # same number as ``reviews_supplied``.
173 "ballots": len(seats),
174 "reviews_supplied": supplied,
175 # One entry per cause, from the one mapping, so a bucket cannot be added
176 # in `panel` and go unpublished here. ``silent`` is the only one that
177 # survives a call with no ballots, because it is the only one the raw
178 # results can answer.
179 **{
180 key: (buckets[cause] if buckets is not None else None)
181 for cause, key in panel.PANEL_METADATA_KEYS.items()
182 },
183 "silent": buckets[panel.SILENT] if buckets is not None else silent,
184 "chair": chair or "",
185 "chair_ballot": bool(chair) and any(r.get("name", "") == chair for r in counted),
186 }
189def distinct_vendors(specs) -> int:
190 """How many distinct vendors a set of agent specs represents (pure).
192 Slots, not vendors, is the mistake this exists to prevent: three
193 ``[[agent]]`` entries all pointing at one vendor are one perspective, so a
194 run configured that way never claimed cross-vendor consensus and must not be
195 failed for not delivering it.
197 Counted by :func:`config.vendor_identity`, so a pair of seats naming
198 vendors this build does not know collapses to the single ``cli`` identity
199 they actually share (issue #701) rather than reading as two.
200 """
201 return len({vendor_identity(getattr(s, "vendor", "")) for s in specs or []} - {""})
204def resolve_min_vendors(cli_value, config) -> tuple[int, bool]:
205 """The effective cross-vendor threshold, and whether a person asked for it.
207 PURE. ``cli_value`` is ``args.min_vendors``: ``None`` when neither
208 ``--min-vendors`` nor ``--no-min-vendors`` was passed, in which case the
209 value comes from ``[jury.ci] min_vendors`` (shipped as 2, #682).
211 The second element says whether the threshold was named on the command line.
212 Only an unnamed (default) threshold is scoped down to runs that actually
213 claimed cross-vendor consensus — someone who types ``--min-vendors 3`` on a
214 two-vendor panel is asking for the failure and gets it.
216 Lives here, beside the gate, so a run and ``jury --doctor`` resolve the same
217 threshold from the same flag (#863).
218 """
219 if cli_value is None:
220 return max(0, int(getattr(config.ci, "min_vendors", DEFAULT_MIN_VENDORS))), False
221 return max(0, int(cli_value)), True
224def claimed_vendors(enabled_agents, local_fallback=None) -> int:
225 """How many distinct vendors a run claimed, which scopes the default guard (pure).
227 The enabled seats, except when the zero-config local fallback seated a model
228 (#863): it does so only when none of the built-in seats can run, so the panel
229 that run can form is that one seat, and the missing seats claimed nothing.
230 The run's gate and ``jury --doctor`` both count through here.
231 """
232 return distinct_vendors([local_fallback] if local_fallback is not None else enabled_agents)
235def vendor_guard_fails(contributed: int, required: int, configured_vendors: int | None) -> bool:
236 """Whether ``contributed`` vendors fail a threshold of ``required`` (pure).
238 ``configured_vendors`` scopes the default: fewer than ``required`` means the
239 run never claimed cross-vendor consensus and is left alone. ``None`` is a
240 threshold named on the command line, enforced as asked. The run passes the
241 vendors that reviewed; ``jury --doctor`` passes the vendors it can reach, a
242 ceiling on that — so the two apply one rule and cannot drift (#863).
243 """
244 if required <= 0:
245 return False
246 if configured_vendors is not None and configured_vendors < required:
247 return False
248 return contributed < required
251def collapse_reason(reviews, required: int, configured_vendors: int | None = None) -> str | None:
252 """Why this run may not stand as cross-vendor consensus, or ``None``.
254 PURE. ``required`` is the number of distinct vendors that must have
255 *contributed* a review (:func:`panel_accounting`'s ``vendors``), not the
256 number configured — an agent that was installed, probed clean and then
257 returned nothing is exactly the failure this guards (#635/#682).
259 ``configured_vendors`` scopes the DEFAULT: when fewer distinct vendors are
260 enabled than ``required``, the run never claimed cross-vendor consensus and
261 is left alone, so turning the guard on by default cannot fail a
262 single-vendor install that was always honest about being one. Pass ``None``
263 for an explicitly requested threshold, which is enforced as asked.
265 The message NAMES the opt-out. Whoever reads it is looking at a red CI step
266 on a gate that ships on by default, quite possibly for the first time, and a
267 failure that does not say how to accept it sends them to the issue tracker
268 for a flag the tool already has.
269 """
270 contributed = panel_accounting(reviews).get("vendors", 0)
271 if not vendor_guard_fails(contributed, required, configured_vendors):
272 return None
273 return (
274 f"panel collapsed: {contributed} vendor(s) contributed a review, "
275 f"{required} required. An abstention is not an approval; "
276 f"cross-vendor consensus was not formed. To accept a collapsed panel, "
277 f"pass --no-min-vendors (or set [jury.ci] min_vendors = 0); to catch a "
278 f"missing CLI at startup instead, run with --strict."
279 )
282def _agent_entry(result) -> dict:
283 """Build a single agent metadata entry.
285 Only operational fields are copied -- never ``output`` or ``error`` text,
286 which could contain raw prompt/diff content or secrets.
287 """
288 return {
289 "name": result.agent,
290 # PROVENANCE: the vendor string as configured, never the collapsed
291 # identity. `panel.vendors` is the gate's count of `vendor_identity`;
292 # this is what the seat said it was (#701).
293 "vendor": result.vendor,
294 "status": "ok" if result.ok else "failed",
295 "duration_s": round(float(result.duration_s), 3),
296 "error_code": result.error_code,
297 # Number of attempts made (issue #30): >1 means a transient failure was
298 # retried before this outcome.
299 "attempts": int(getattr(result, "attempts", 1) or 1),
300 # What this slot contributed, not merely whether the CLI exited 0 (#501).
301 "review_status": review_status(result),
302 }
305def _rounds_executed(outcome: JuryOutcome) -> int:
306 # Prefer the orchestrator's authoritative count (adaptive rounds, issue #40);
307 # fall back to inferring it from the phases that produced output.
308 recorded = getattr(outcome, "rounds_executed", None)
309 if isinstance(recorded, int) and recorded >= 1:
310 return recorded
311 rounds = 1 if outcome.reviews else 0
312 if outcome.debate:
313 rounds += 1
314 return rounds
317def estimate_economics(results: list) -> dict:
318 """Estimate token counts and USD dollar cost across all executed agent slots (issue #528).
320 Uses conservative token heuristics (~4 chars/token from output + base context)
321 and published per-vendor pricing tiers. Local models (Ollama, local) are computed
322 at $0.00 (free offline).
323 """
324 vendor_rates_per_1m = {
325 "local": 0.0,
326 "ollama": 0.0,
327 "deepseek": 0.27,
328 "groq": 0.30,
329 "moonshot": 0.50,
330 "gemini": 1.25,
331 "google": 1.25,
332 "openai": 2.50,
333 "codex": 2.50,
334 "anthropic": 3.00,
335 "claude": 3.00,
336 }
337 breakdown = []
338 total_tokens = 0
339 total_cost_usd = 0.0
340 # bolt: Consolidate multiple metrics into a single-pass O(N) explicit loop
341 local_free_slots = 0
343 for r in results:
344 agent = getattr(r, "agent", "unknown")
345 vendor = normalise_vendor(getattr(r, "vendor", ""))
346 output_len = len(getattr(r, "output", "") or "")
347 # Heuristic: base prompt ~800 tokens + output tokens
348 tokens_est = max(100, 800 + (output_len // 4)) if getattr(r, "ok", False) else 200
350 # Match rate
351 rate_per_1m = 2.0 # default generic rate
352 for k, v in vendor_rates_per_1m.items():
353 if k in vendor or k in agent.lower():
354 rate_per_1m = v
355 break
357 cost_usd = (tokens_est / 1_000_000) * rate_per_1m
358 is_local = rate_per_1m == 0.0
360 total_tokens += tokens_est
361 total_cost_usd += cost_usd
362 if is_local:
363 local_free_slots += 1
365 breakdown.append(
366 {
367 "agent": agent,
368 "vendor": getattr(r, "vendor", ""),
369 "tokens_est": tokens_est,
370 "cost_usd_est": round(cost_usd, 6),
371 "is_local_free": is_local,
372 }
373 )
375 return {
376 "total_tokens_est": total_tokens,
377 "total_cost_usd_est": round(total_cost_usd, 4),
378 "local_free_slots": local_free_slots,
379 "breakdown": breakdown,
380 }
383def build_run_metadata(
384 outcome: JuryOutcome,
385 config: JuryConfig,
386 *,
387 decision=None,
388 vote=None,
389 mode: str = "code",
390 zero_config_fallback: bool = False,
391) -> dict:
392 """Return a machine-readable metadata dict for a jury run.
394 The dict is safe to serialize as JSON and contains no diff text, prompt
395 text, agent output, or secrets.
397 Per-agent entries reflect the review panel (round 1). Total wall-clock is
398 summed across every phase (review, debate, verify, synthesis) so it captures
399 the full run cost proxy even though debate/verify/synthesis are re-runs of
400 panel agents rather than distinct participants.
402 ``mode`` selects the ballot vocabulary, matching ``--issue``. It reaches here
403 because ``panel.reviews_supplied`` is counted over the **ballots** (#700,
404 round 2), and the ballots are what ``--issue`` changes: the metadata and the
405 ``reviewers`` array must be derived under the same mode, or the run's own
406 gate would count a different document than the one it printed.
408 ``zero_config_fallback`` says the panel is the zero-config fallback's one
409 local seat (#863); the caller knows, because it seated it.
410 """
411 # The panel is the set of round-1 participants; this is the canonical
412 # per-agent view and avoids duplicating the chair across later phases.
413 agents = [_agent_entry(r) for r in outcome.reviews]
415 all_results = list(outcome.reviews) + list(outcome.debate)
416 if outcome.synthesis is not None:
417 all_results.append(outcome.synthesis)
418 if outcome.verify is not None:
419 all_results.append(outcome.verify)
420 total_wall_clock_s = round(sum(float(r.duration_s) for r in all_results), 3)
422 # Reproducibility signals (issue #41): the run seed and a stable hash of the
423 # effective config let a run be reproduced/explained. The seed is whatever
424 # the run was configured with (may be None when unseeded). The config hash
425 # is a pure function of config, so it is stable across runs and over time.
426 from .ballots import reviewer_ballots
427 from .classification import classify
428 from .config import config_hash
430 # Execution / partial-result signals (issue #30) and adaptive-round signals
431 # (issue #40). ``skipped`` lists agents whose CLI was unavailable so they
432 # never ran; ``budget_exhausted`` flags a run that stopped early on the total
433 # timeout; ``stop_reason`` explains why debate ran or stopped.
434 skipped = [
435 {"name": name, "reason": reason} for name, reason in getattr(outcome, "skipped", []) or []
436 ]
437 retried = [a["name"] for a in agents if a["attempts"] > 1]
439 # Final-verdict mode (issue #220). ``decision`` is the effective mode (CLI
440 # override else config); ``vote`` is the tally dict when voting, else None.
441 decision = decision or config.decision
442 vote_meta = None
443 if vote is not None:
444 vote_meta = {
445 "verdict": vote.verdict,
446 "tally": vote.tally,
447 "ballots": [
448 {"reviewer": b.reviewer, "vote": b.vote, "reason": b.reason} for b in vote.ballots
449 ],
450 }
452 return {
453 "schema_version": SCHEMA_VERSION,
454 "decision": decision,
455 "vote": vote_meta,
456 "agents": agents,
457 # Configured vs effective panel size (issue #501): a slot that returned no
458 # review is an abstention, not an approval, and must not inflate the panel.
459 "panel": {
460 **panel_accounting(
461 outcome.reviews,
462 chair=getattr(outcome, "chair", "") or "",
463 ballots=reviewer_ballots(outcome, config, vote=vote, mode=mode),
464 ),
465 "zero_config_fallback": bool(zero_config_fallback),
466 },
467 "economics": estimate_economics(all_results),
468 "rounds_executed": _rounds_executed(outcome),
469 "from_cache": bool(getattr(outcome, "from_cache", False)),
470 "stop_reason": getattr(outcome, "stop_reason", "") or "",
471 # What routing decided (#714): mode, risk band, who sat, who was benched,
472 # the anchor, and whether round 1 escalated. The standard mode records
473 # {mode: "standard", panel: [...]} so the key is always present.
474 "routing": dict(getattr(outcome, "routing", None) or {"mode": "standard"}),
475 "skipped": skipped,
476 "retried": retried,
477 "budget_exhausted": bool(getattr(outcome, "budget_exhausted", False)),
478 "execution": {
479 "total_timeout": config.total_timeout,
480 "phase_timeout": config.phase_timeout,
481 "retries": config.retries,
482 "early_stop": config.early_stop,
483 "max_rounds": config.effective_max_rounds,
484 },
485 "verify_enabled": bool(config.verify),
486 "context_mode": outcome.context_mode,
487 "redact_secrets": bool(outcome.redact_secrets),
488 "redaction_count": outcome.redaction_count,
489 "seed": config.seed,
490 "config_hash": config_hash(config),
491 # PR-level classification (issue #7): deterministic summary derived from
492 # the structured findings + consensus groups. No diff text is included.
493 "classification": classify(outcome),
494 # Wall-clock is an approximate COST PROXY, not a dollar cost. No token
495 # counts are available from the underlying CLIs.
496 "total_wall_clock_s": total_wall_clock_s,
497 "cost_signal": "wall-clock-proxy",
498 "generated_at": datetime.now(UTC).isoformat(),
499 }
502# end