Coverage for src/ai_jury/metadata.py: 99%

108 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 18:15 +0000

1"""Run metadata and cost-awareness (wall-clock proxy) reporting. 

2 

3Builds a machine-readable metadata dict describing a jury run: which 

4agents participated, their per-agent status and wall-clock duration, how many 

5rounds ran, whether verification was enabled, and timestamps. 

6 

7IMPORTANT: This metadata deliberately contains NO diff text, NO prompt text, 

8NO agent output, and NO secrets -- only structural/operational signals. 

9 

10There are no token counts available from the underlying CLIs, so wall-clock 

11seconds are used as an approximate cost *proxy*, not a dollar cost. 

12""" 

13 

14from __future__ import annotations 

15 

16from datetime import UTC, datetime 

17from typing import TYPE_CHECKING 

18 

19from . import panel 

20from .config import DEFAULT_MIN_VENDORS, normalise_vendor, vendor_identity 

21 

22if TYPE_CHECKING: # pragma: no cover - typing only 

23 from .config import JuryConfig 

24 from .orchestrator import JuryOutcome 

25 

26# v2 (issue #30/#40) added: stop_reason, skipped, retried, budget_exhausted, 

27# execution{...}, and per-agent ``attempts``. 

28# v4 (issue #501) added: ``panel`` (configured vs effective size, abstentions) and 

29# per-agent ``review_status``. A slot that returns no review is an abstention, not an 

30# approval, and until now nothing in the output said so. 

31# v5 (issues #699/#700) added, inside ``panel``: ``ballots``, ``reviews_supplied``, 

32# ``silent``, ``insubstantial``, ``refused``, ``adapter_failed``, ``chair`` and 

33# ``chair_ballot`` — the number of reviews a downstream consumer actually 

34# receives (the ballots that named something and voted; the chair's synthesis 

35# record is not one of them, and neither is an abstaining ballot), the ways 

36# a seat that ran produced no review, and whether the chairing agent's own ballot 

37# is one of the counted reviews. The causes and ``reviews_supplied`` sum to 

38# ``ballots``. Purely additive: every v4 key keeps its name and meaning. 

39# v6 (issue #710) added, inside ``panel``: ``not_in_change`` — the seats whose 

40# stated scope named only paths or symbols the change does not contain. Split 

41# out of ``insubstantial``, which keeps meaning exactly "answered and named 

42# nothing checkable": naming `src/made/up.py` is a different failure from naming 

43# nothing, and the two ask for different fixes. Additive, and the buckets still 

44# sum to ``ballots`` — the seats moved between buckets, none was added or lost. 

45# v7 (issue #714) added ``routing``. 

46# v8 (issue #863) added, inside ``panel``: ``zero_config_fallback`` — ``true`` when 

47# the run had no config and no usable agent CLI, so the panel was the one local 

48# seat the zero-config fallback seated. That run is single-vendor, and the default 

49# cross-vendor guard does not fail it; this says so where ``--quiet`` cannot hide 

50# it. Always present. Additive: every v7 key keeps its name and meaning. 

51SCHEMA_VERSION = 8 

52 

53 

54#: What a reviewer slot actually contributed (issue #501). ``clean`` and 

55#: ``abstained`` both carry zero findings and used to be reported identically, 

56#: which is how a run with two non-reviewing slots still described itself as a 

57#: three-agent panel. 

58REVIEW_STATUSES = ("findings", "clean", "abstained", "failed") 

59 

60 

61def review_status(result) -> str: 

62 """Classify one reviewer slot's contribution. Pure, and never judges content. 

63 

64 * ``failed`` — the adapter did not return a result at all. 

65 * ``findings`` — produced at least one structured finding. 

66 * ``clean`` — emitted a findings block that was empty: examined, found nothing. 

67 * ``abstained`` — returned successfully with no findings block. Not an approval: 

68 nothing reviewable came back, so this slot contributed no evidence either way. 

69 """ 

70 if not getattr(result, "ok", False): 

71 return "failed" 

72 if getattr(result, "findings", None): 

73 return "findings" 

74 return "clean" if getattr(result, "structured", False) else "abstained" 

75 

76 

77def panel_accounting(reviews, chair: str = "", ballots=None) -> dict: 

78 """Configured versus *effective* panel size, and the per-status breakdown. 

79 

80 A consumer gating on the panel needs the effective number — keel downgrades a 

81 jury to advisory below two participating vendors, and can only do that if the 

82 report says the panel was short. ``vendors`` counts distinct vendors that 

83 actually contributed a review, which is the number that matters for 

84 cross-vendor consensus: three slots from one vendor are not three perspectives. 

85 

86 Vendors are counted by :func:`config.vendor_identity`, not by the raw 

87 string: a seat whose vendor the tool does not recognise ran on the generic 

88 ``cli`` fallback and is counted as ``cli``, so two unidentifiable seats are 

89 one vendor here even though the report still names each one honestly 

90 (issue #701). 

91 

92 ``reviews_supplied`` is a different number again, and the one #699 was about: 

93 how many reviews the bundle hands on. It is not ``configured`` (every seat 

94 that ran, silent ones included), not ``effective`` (a slot that returned 

95 prose but no findings block may still have named nothing), and not 

96 ``ballots`` plus the chair record — a consumer splits the ``reviewers`` array 

97 on ``role`` and reads the ``chair`` entry as the panel's consensus rather 

98 than as one more review. 

99 

100 It is :func:`ai_jury.panel.review_count` over the **ballot records**, which 

101 is why they are passed in rather than derived here: whether a seat reviewed 

102 is a fact about the record it produced — its scope and its verdict — and no 

103 predicate over the raw round-1 result can see it. That was the defect at this 

104 line (#700, round 2): a prose-only non-review abstained on its ballot and was 

105 still counted here, so ``--min-reviews`` could be satisfied by seats that 

106 reviewed nothing. With ``ballots`` omitted the count is reported as ``None`` 

107 rather than guessed, because every guess available over-counts. 

108 

109 ``silent``, ``insubstantial``, ``refused`` and ``adapter_failed`` split the 

110 seats that produced no review by cause, for the shortfall message: a silent 

111 agent is a CLI that broke or a budget that ran out, a seat that answered and 

112 named nothing is a reviewer that did not review, a refusal is a model 

113 declining the task, and a failed adapter is an invocation to fix. They come 

114 from :func:`ai_jury.panel.abstention_buckets` — each ballot classified by the 

115 cause it carries — so that they and ``reviews_supplied`` add up to 

116 ``ballots`` with nothing left over. 

117 

118 ``insubstantial`` was derived by subtraction until #700's fifth round, and 

119 the subtraction was the defect: ``ballots - silent - supplied`` swept up 

120 every seat that was neither silent nor a counted review, and once a ballot 

121 could carry a substantive scope and still abstain that included the seat that 

122 named a file and then refused, and the one whose adapter died holding a file 

123 name. Both were then rendered as "named nothing checkable" — a cause their 

124 own ballot contradicted. It now means exactly ``named_nothing``: the seats 

125 that answered and named nothing a reader could check, and only those. 

126 

127 ``chair_ballot`` says whether the chairing agent's own ballot is one of the 

128 *counted* reviews — a chairing agent that ran and abstained has a ballot in 

129 the bundle and has supplied no review. 

130 """ 

131 reviews = list(reviews or []) 

132 seats = panel.ballot_seats(reviews) 

133 records = list(ballots) if ballots is not None else None 

134 counted = [r for r in (records or []) if panel.is_review(r)] 

135 supplied = len(counted) if records is not None else None 

136 # Without the records there is nothing to classify: the cause is a fact the 

137 # ballot carries, and every guess available from the raw results is the 

138 # subtraction this replaced. ``silent`` is the exception, and only because 

139 # :func:`ai_jury.ballots.abstention_cause` classifies silence on the same 

140 # predicate (:func:`ai_jury.panel.silent_seat`: no output and no allowance 

141 # failure) — the two readings are the same number by construction. 

142 buckets = panel.abstention_buckets(records) if records is not None else None 

143 silent = sum(1 for r in seats if panel.silent_seat(r)) 

144 

145 # bolt: Consolidate multiple metrics into a single-pass O(N) explicit loop 

146 effective_count = 0 

147 abstained_count = 0 

148 failed_count = 0 

149 contributing_vendors = set() 

150 

151 for r in reviews: 

152 st = review_status(r) 

153 if st in ("findings", "clean"): 

154 effective_count += 1 

155 vendor = vendor_identity(getattr(r, "vendor", "")) 

156 if vendor: 156 ↛ 151line 156 didn't jump to line 151 because the condition on line 156 was always true

157 contributing_vendors.add(vendor) 

158 elif st == "abstained": 

159 abstained_count += 1 

160 elif st == "failed": 160 ↛ 151line 160 didn't jump to line 151 because the condition on line 160 was always true

161 failed_count += 1 

162 

163 return { 

164 "configured": len(reviews), 

165 "effective": effective_count, 

166 "vendors": len(contributing_vendors), 

167 "abstained": abstained_count, 

168 "failed": failed_count, 

169 "short": effective_count < len(reviews), 

170 # One ballot record per seat that ran — including a silent one, recorded 

171 # as an abstention so the report can name it. Deliberately no longer the 

172 # same number as ``reviews_supplied``. 

173 "ballots": len(seats), 

174 "reviews_supplied": supplied, 

175 # One entry per cause, from the one mapping, so a bucket cannot be added 

176 # in `panel` and go unpublished here. ``silent`` is the only one that 

177 # survives a call with no ballots, because it is the only one the raw 

178 # results can answer. 

179 **{ 

180 key: (buckets[cause] if buckets is not None else None) 

181 for cause, key in panel.PANEL_METADATA_KEYS.items() 

182 }, 

183 "silent": buckets[panel.SILENT] if buckets is not None else silent, 

184 "chair": chair or "", 

185 "chair_ballot": bool(chair) and any(r.get("name", "") == chair for r in counted), 

186 } 

187 

188 

189def distinct_vendors(specs) -> int: 

190 """How many distinct vendors a set of agent specs represents (pure). 

191 

192 Slots, not vendors, is the mistake this exists to prevent: three 

193 ``[[agent]]`` entries all pointing at one vendor are one perspective, so a 

194 run configured that way never claimed cross-vendor consensus and must not be 

195 failed for not delivering it. 

196 

197 Counted by :func:`config.vendor_identity`, so a pair of seats naming 

198 vendors this build does not know collapses to the single ``cli`` identity 

199 they actually share (issue #701) rather than reading as two. 

200 """ 

201 return len({vendor_identity(getattr(s, "vendor", "")) for s in specs or []} - {""}) 

202 

203 

204def resolve_min_vendors(cli_value, config) -> tuple[int, bool]: 

205 """The effective cross-vendor threshold, and whether a person asked for it. 

206 

207 PURE. ``cli_value`` is ``args.min_vendors``: ``None`` when neither 

208 ``--min-vendors`` nor ``--no-min-vendors`` was passed, in which case the 

209 value comes from ``[jury.ci] min_vendors`` (shipped as 2, #682). 

210 

211 The second element says whether the threshold was named on the command line. 

212 Only an unnamed (default) threshold is scoped down to runs that actually 

213 claimed cross-vendor consensus — someone who types ``--min-vendors 3`` on a 

214 two-vendor panel is asking for the failure and gets it. 

215 

216 Lives here, beside the gate, so a run and ``jury --doctor`` resolve the same 

217 threshold from the same flag (#863). 

218 """ 

219 if cli_value is None: 

220 return max(0, int(getattr(config.ci, "min_vendors", DEFAULT_MIN_VENDORS))), False 

221 return max(0, int(cli_value)), True 

222 

223 

224def claimed_vendors(enabled_agents, local_fallback=None) -> int: 

225 """How many distinct vendors a run claimed, which scopes the default guard (pure). 

226 

227 The enabled seats, except when the zero-config local fallback seated a model 

228 (#863): it does so only when none of the built-in seats can run, so the panel 

229 that run can form is that one seat, and the missing seats claimed nothing. 

230 The run's gate and ``jury --doctor`` both count through here. 

231 """ 

232 return distinct_vendors([local_fallback] if local_fallback is not None else enabled_agents) 

233 

234 

235def vendor_guard_fails(contributed: int, required: int, configured_vendors: int | None) -> bool: 

236 """Whether ``contributed`` vendors fail a threshold of ``required`` (pure). 

237 

238 ``configured_vendors`` scopes the default: fewer than ``required`` means the 

239 run never claimed cross-vendor consensus and is left alone. ``None`` is a 

240 threshold named on the command line, enforced as asked. The run passes the 

241 vendors that reviewed; ``jury --doctor`` passes the vendors it can reach, a 

242 ceiling on that — so the two apply one rule and cannot drift (#863). 

243 """ 

244 if required <= 0: 

245 return False 

246 if configured_vendors is not None and configured_vendors < required: 

247 return False 

248 return contributed < required 

249 

250 

251def collapse_reason(reviews, required: int, configured_vendors: int | None = None) -> str | None: 

252 """Why this run may not stand as cross-vendor consensus, or ``None``. 

253 

254 PURE. ``required`` is the number of distinct vendors that must have 

255 *contributed* a review (:func:`panel_accounting`'s ``vendors``), not the 

256 number configured — an agent that was installed, probed clean and then 

257 returned nothing is exactly the failure this guards (#635/#682). 

258 

259 ``configured_vendors`` scopes the DEFAULT: when fewer distinct vendors are 

260 enabled than ``required``, the run never claimed cross-vendor consensus and 

261 is left alone, so turning the guard on by default cannot fail a 

262 single-vendor install that was always honest about being one. Pass ``None`` 

263 for an explicitly requested threshold, which is enforced as asked. 

264 

265 The message NAMES the opt-out. Whoever reads it is looking at a red CI step 

266 on a gate that ships on by default, quite possibly for the first time, and a 

267 failure that does not say how to accept it sends them to the issue tracker 

268 for a flag the tool already has. 

269 """ 

270 contributed = panel_accounting(reviews).get("vendors", 0) 

271 if not vendor_guard_fails(contributed, required, configured_vendors): 

272 return None 

273 return ( 

274 f"panel collapsed: {contributed} vendor(s) contributed a review, " 

275 f"{required} required. An abstention is not an approval; " 

276 f"cross-vendor consensus was not formed. To accept a collapsed panel, " 

277 f"pass --no-min-vendors (or set [jury.ci] min_vendors = 0); to catch a " 

278 f"missing CLI at startup instead, run with --strict." 

279 ) 

280 

281 

282def _agent_entry(result) -> dict: 

283 """Build a single agent metadata entry. 

284 

285 Only operational fields are copied -- never ``output`` or ``error`` text, 

286 which could contain raw prompt/diff content or secrets. 

287 """ 

288 return { 

289 "name": result.agent, 

290 # PROVENANCE: the vendor string as configured, never the collapsed 

291 # identity. `panel.vendors` is the gate's count of `vendor_identity`; 

292 # this is what the seat said it was (#701). 

293 "vendor": result.vendor, 

294 "status": "ok" if result.ok else "failed", 

295 "duration_s": round(float(result.duration_s), 3), 

296 "error_code": result.error_code, 

297 # Number of attempts made (issue #30): >1 means a transient failure was 

298 # retried before this outcome. 

299 "attempts": int(getattr(result, "attempts", 1) or 1), 

300 # What this slot contributed, not merely whether the CLI exited 0 (#501). 

301 "review_status": review_status(result), 

302 } 

303 

304 

305def _rounds_executed(outcome: JuryOutcome) -> int: 

306 # Prefer the orchestrator's authoritative count (adaptive rounds, issue #40); 

307 # fall back to inferring it from the phases that produced output. 

308 recorded = getattr(outcome, "rounds_executed", None) 

309 if isinstance(recorded, int) and recorded >= 1: 

310 return recorded 

311 rounds = 1 if outcome.reviews else 0 

312 if outcome.debate: 

313 rounds += 1 

314 return rounds 

315 

316 

317def estimate_economics(results: list) -> dict: 

318 """Estimate token counts and USD dollar cost across all executed agent slots (issue #528). 

319 

320 Uses conservative token heuristics (~4 chars/token from output + base context) 

321 and published per-vendor pricing tiers. Local models (Ollama, local) are computed 

322 at $0.00 (free offline). 

323 """ 

324 vendor_rates_per_1m = { 

325 "local": 0.0, 

326 "ollama": 0.0, 

327 "deepseek": 0.27, 

328 "groq": 0.30, 

329 "moonshot": 0.50, 

330 "gemini": 1.25, 

331 "google": 1.25, 

332 "openai": 2.50, 

333 "codex": 2.50, 

334 "anthropic": 3.00, 

335 "claude": 3.00, 

336 } 

337 breakdown = [] 

338 total_tokens = 0 

339 total_cost_usd = 0.0 

340 # bolt: Consolidate multiple metrics into a single-pass O(N) explicit loop 

341 local_free_slots = 0 

342 

343 for r in results: 

344 agent = getattr(r, "agent", "unknown") 

345 vendor = normalise_vendor(getattr(r, "vendor", "")) 

346 output_len = len(getattr(r, "output", "") or "") 

347 # Heuristic: base prompt ~800 tokens + output tokens 

348 tokens_est = max(100, 800 + (output_len // 4)) if getattr(r, "ok", False) else 200 

349 

350 # Match rate 

351 rate_per_1m = 2.0 # default generic rate 

352 for k, v in vendor_rates_per_1m.items(): 

353 if k in vendor or k in agent.lower(): 

354 rate_per_1m = v 

355 break 

356 

357 cost_usd = (tokens_est / 1_000_000) * rate_per_1m 

358 is_local = rate_per_1m == 0.0 

359 

360 total_tokens += tokens_est 

361 total_cost_usd += cost_usd 

362 if is_local: 

363 local_free_slots += 1 

364 

365 breakdown.append( 

366 { 

367 "agent": agent, 

368 "vendor": getattr(r, "vendor", ""), 

369 "tokens_est": tokens_est, 

370 "cost_usd_est": round(cost_usd, 6), 

371 "is_local_free": is_local, 

372 } 

373 ) 

374 

375 return { 

376 "total_tokens_est": total_tokens, 

377 "total_cost_usd_est": round(total_cost_usd, 4), 

378 "local_free_slots": local_free_slots, 

379 "breakdown": breakdown, 

380 } 

381 

382 

383def build_run_metadata( 

384 outcome: JuryOutcome, 

385 config: JuryConfig, 

386 *, 

387 decision=None, 

388 vote=None, 

389 mode: str = "code", 

390 zero_config_fallback: bool = False, 

391) -> dict: 

392 """Return a machine-readable metadata dict for a jury run. 

393 

394 The dict is safe to serialize as JSON and contains no diff text, prompt 

395 text, agent output, or secrets. 

396 

397 Per-agent entries reflect the review panel (round 1). Total wall-clock is 

398 summed across every phase (review, debate, verify, synthesis) so it captures 

399 the full run cost proxy even though debate/verify/synthesis are re-runs of 

400 panel agents rather than distinct participants. 

401 

402 ``mode`` selects the ballot vocabulary, matching ``--issue``. It reaches here 

403 because ``panel.reviews_supplied`` is counted over the **ballots** (#700, 

404 round 2), and the ballots are what ``--issue`` changes: the metadata and the 

405 ``reviewers`` array must be derived under the same mode, or the run's own 

406 gate would count a different document than the one it printed. 

407 

408 ``zero_config_fallback`` says the panel is the zero-config fallback's one 

409 local seat (#863); the caller knows, because it seated it. 

410 """ 

411 # The panel is the set of round-1 participants; this is the canonical 

412 # per-agent view and avoids duplicating the chair across later phases. 

413 agents = [_agent_entry(r) for r in outcome.reviews] 

414 

415 all_results = list(outcome.reviews) + list(outcome.debate) 

416 if outcome.synthesis is not None: 

417 all_results.append(outcome.synthesis) 

418 if outcome.verify is not None: 

419 all_results.append(outcome.verify) 

420 total_wall_clock_s = round(sum(float(r.duration_s) for r in all_results), 3) 

421 

422 # Reproducibility signals (issue #41): the run seed and a stable hash of the 

423 # effective config let a run be reproduced/explained. The seed is whatever 

424 # the run was configured with (may be None when unseeded). The config hash 

425 # is a pure function of config, so it is stable across runs and over time. 

426 from .ballots import reviewer_ballots 

427 from .classification import classify 

428 from .config import config_hash 

429 

430 # Execution / partial-result signals (issue #30) and adaptive-round signals 

431 # (issue #40). ``skipped`` lists agents whose CLI was unavailable so they 

432 # never ran; ``budget_exhausted`` flags a run that stopped early on the total 

433 # timeout; ``stop_reason`` explains why debate ran or stopped. 

434 skipped = [ 

435 {"name": name, "reason": reason} for name, reason in getattr(outcome, "skipped", []) or [] 

436 ] 

437 retried = [a["name"] for a in agents if a["attempts"] > 1] 

438 

439 # Final-verdict mode (issue #220). ``decision`` is the effective mode (CLI 

440 # override else config); ``vote`` is the tally dict when voting, else None. 

441 decision = decision or config.decision 

442 vote_meta = None 

443 if vote is not None: 

444 vote_meta = { 

445 "verdict": vote.verdict, 

446 "tally": vote.tally, 

447 "ballots": [ 

448 {"reviewer": b.reviewer, "vote": b.vote, "reason": b.reason} for b in vote.ballots 

449 ], 

450 } 

451 

452 return { 

453 "schema_version": SCHEMA_VERSION, 

454 "decision": decision, 

455 "vote": vote_meta, 

456 "agents": agents, 

457 # Configured vs effective panel size (issue #501): a slot that returned no 

458 # review is an abstention, not an approval, and must not inflate the panel. 

459 "panel": { 

460 **panel_accounting( 

461 outcome.reviews, 

462 chair=getattr(outcome, "chair", "") or "", 

463 ballots=reviewer_ballots(outcome, config, vote=vote, mode=mode), 

464 ), 

465 "zero_config_fallback": bool(zero_config_fallback), 

466 }, 

467 "economics": estimate_economics(all_results), 

468 "rounds_executed": _rounds_executed(outcome), 

469 "from_cache": bool(getattr(outcome, "from_cache", False)), 

470 "stop_reason": getattr(outcome, "stop_reason", "") or "", 

471 # What routing decided (#714): mode, risk band, who sat, who was benched, 

472 # the anchor, and whether round 1 escalated. The standard mode records 

473 # {mode: "standard", panel: [...]} so the key is always present. 

474 "routing": dict(getattr(outcome, "routing", None) or {"mode": "standard"}), 

475 "skipped": skipped, 

476 "retried": retried, 

477 "budget_exhausted": bool(getattr(outcome, "budget_exhausted", False)), 

478 "execution": { 

479 "total_timeout": config.total_timeout, 

480 "phase_timeout": config.phase_timeout, 

481 "retries": config.retries, 

482 "early_stop": config.early_stop, 

483 "max_rounds": config.effective_max_rounds, 

484 }, 

485 "verify_enabled": bool(config.verify), 

486 "context_mode": outcome.context_mode, 

487 "redact_secrets": bool(outcome.redact_secrets), 

488 "redaction_count": outcome.redaction_count, 

489 "seed": config.seed, 

490 "config_hash": config_hash(config), 

491 # PR-level classification (issue #7): deterministic summary derived from 

492 # the structured findings + consensus groups. No diff text is included. 

493 "classification": classify(outcome), 

494 # Wall-clock is an approximate COST PROXY, not a dollar cost. No token 

495 # counts are available from the underlying CLIs. 

496 "total_wall_clock_s": total_wall_clock_s, 

497 "cost_signal": "wall-clock-proxy", 

498 "generated_at": datetime.now(UTC).isoformat(), 

499 } 

500 

501 

502# end