Coverage for src/ai_jury/metadata.py: 99%
64 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-17 21:31 +0000
1"""Run metadata and cost-awareness (wall-clock proxy) reporting.
3Builds a machine-readable metadata dict describing a jury run: which
4agents participated, their per-agent status and wall-clock duration, how many
5rounds ran, whether verification was enabled, and timestamps.
7IMPORTANT: This metadata deliberately contains NO diff text, NO prompt text,
8NO agent output, and NO secrets -- only structural/operational signals.
10There are no token counts available from the underlying CLIs, so wall-clock
11seconds are used as an approximate cost *proxy*, not a dollar cost.
12"""
14from __future__ import annotations
16from datetime import UTC, datetime
17from typing import TYPE_CHECKING
19if TYPE_CHECKING: # pragma: no cover - typing only
20 from .config import JuryConfig
21 from .orchestrator import JuryOutcome
23# v2 (issue #30/#40) added: stop_reason, skipped, retried, budget_exhausted,
24# execution{...}, and per-agent ``attempts``.
25# v4 (issue #501) added: ``panel`` (configured vs effective size, abstentions) and
26# per-agent ``review_status``. A slot that returns no review is an abstention, not an
27# approval, and until now nothing in the output said so.
28SCHEMA_VERSION = 4
31#: What a reviewer slot actually contributed (issue #501). ``clean`` and
32#: ``abstained`` both carry zero findings and used to be reported identically,
33#: which is how a run with two non-reviewing slots still described itself as a
34#: three-agent panel.
35REVIEW_STATUSES = ("findings", "clean", "abstained", "failed")
38def review_status(result) -> str:
39 """Classify one reviewer slot's contribution. Pure, and never judges content.
41 * ``failed`` — the adapter did not return a result at all.
42 * ``findings`` — produced at least one structured finding.
43 * ``clean`` — emitted a findings block that was empty: examined, found nothing.
44 * ``abstained`` — returned successfully with no findings block. Not an approval:
45 nothing reviewable came back, so this slot contributed no evidence either way.
46 """
47 if not getattr(result, "ok", False):
48 return "failed"
49 if getattr(result, "findings", None):
50 return "findings"
51 return "clean" if getattr(result, "structured", False) else "abstained"
54def panel_accounting(reviews) -> dict:
55 """Configured versus *effective* panel size, and the per-status breakdown.
57 A consumer gating on the panel needs the effective number — keel downgrades a
58 jury to advisory below two participating vendors, and can only do that if the
59 report says the panel was short. ``vendors`` counts distinct vendors that
60 actually contributed a review, which is the number that matters for
61 cross-vendor consensus: three slots from one vendor are not three perspectives.
62 """
63 reviews = list(reviews or [])
64 statuses = [review_status(r) for r in reviews]
65 contributing = [
66 r for r, st in zip(reviews, statuses, strict=True) if st in ("findings", "clean")
67 ]
68 return {
69 "configured": len(reviews),
70 "effective": len(contributing),
71 "vendors": len({getattr(r, "vendor", "") for r in contributing if getattr(r, "vendor", "")}),
72 "abstained": statuses.count("abstained"),
73 "failed": statuses.count("failed"),
74 "short": len(contributing) < len(reviews),
75 }
78def _agent_entry(result) -> dict:
79 """Build a single agent metadata entry.
81 Only operational fields are copied -- never ``output`` or ``error`` text,
82 which could contain raw prompt/diff content or secrets.
83 """
84 return {
85 "name": result.agent,
86 "vendor": result.vendor,
87 "status": "ok" if result.ok else "failed",
88 "duration_s": round(float(result.duration_s), 3),
89 "error_code": result.error_code,
90 # Number of attempts made (issue #30): >1 means a transient failure was
91 # retried before this outcome.
92 "attempts": int(getattr(result, "attempts", 1) or 1),
93 # What this slot contributed, not merely whether the CLI exited 0 (#501).
94 "review_status": review_status(result),
95 }
98def _rounds_executed(outcome: JuryOutcome) -> int:
99 # Prefer the orchestrator's authoritative count (adaptive rounds, issue #40);
100 # fall back to inferring it from the phases that produced output.
101 recorded = getattr(outcome, "rounds_executed", None)
102 if isinstance(recorded, int) and recorded >= 1:
103 return recorded
104 rounds = 1 if outcome.reviews else 0
105 if outcome.debate:
106 rounds += 1
107 return rounds
110def estimate_economics(results: list) -> dict:
111 """Estimate token counts and USD dollar cost across all executed agent slots (issue #528).
113 Uses conservative token heuristics (~4 chars/token from output + base context)
114 and published per-vendor pricing tiers. Local models (Ollama, local) are computed
115 at $0.00 (free offline).
116 """
117 vendor_rates_per_1m = {
118 "local": 0.0,
119 "ollama": 0.0,
120 "deepseek": 0.27,
121 "groq": 0.30,
122 "moonshot": 0.50,
123 "gemini": 1.25,
124 "google": 1.25,
125 "openai": 2.50,
126 "codex": 2.50,
127 "anthropic": 3.00,
128 "claude": 3.00,
129 }
130 breakdown = []
131 total_tokens = 0
132 total_cost_usd = 0.0
134 for r in results:
135 agent = getattr(r, "agent", "unknown")
136 vendor = (getattr(r, "vendor", "") or "").lower()
137 output_len = len(getattr(r, "output", "") or "")
138 # Heuristic: base prompt ~800 tokens + output tokens
139 tokens_est = max(100, 800 + (output_len // 4)) if getattr(r, "ok", False) else 200
141 # Match rate
142 rate_per_1m = 2.0 # default generic rate
143 for k, v in vendor_rates_per_1m.items(): 143 ↛ 148line 143 didn't jump to line 148 because the loop on line 143 didn't complete
144 if k in vendor or k in agent.lower():
145 rate_per_1m = v
146 break
148 cost_usd = (tokens_est / 1_000_000) * rate_per_1m
149 is_local = rate_per_1m == 0.0
151 total_tokens += tokens_est
152 total_cost_usd += cost_usd
154 breakdown.append({
155 "agent": agent,
156 "vendor": getattr(r, "vendor", ""),
157 "tokens_est": tokens_est,
158 "cost_usd_est": round(cost_usd, 6),
159 "is_local_free": is_local,
160 })
162 return {
163 "total_tokens_est": total_tokens,
164 "total_cost_usd_est": round(total_cost_usd, 4),
165 "local_free_slots": sum(1 for b in breakdown if b["is_local_free"]),
166 "breakdown": breakdown,
167 }
170def build_run_metadata(
171 outcome: JuryOutcome, config: JuryConfig, *, decision=None, vote=None
172) -> dict:
173 """Return a machine-readable metadata dict for a jury run.
175 The dict is safe to serialize as JSON and contains no diff text, prompt
176 text, agent output, or secrets.
178 Per-agent entries reflect the review panel (round 1). Total wall-clock is
179 summed across every phase (review, debate, verify, synthesis) so it captures
180 the full run cost proxy even though debate/verify/synthesis are re-runs of
181 panel agents rather than distinct participants.
182 """
183 # The panel is the set of round-1 participants; this is the canonical
184 # per-agent view and avoids duplicating the chair across later phases.
185 agents = [_agent_entry(r) for r in outcome.reviews]
187 all_results = list(outcome.reviews) + list(outcome.debate)
188 if outcome.synthesis is not None:
189 all_results.append(outcome.synthesis)
190 if outcome.verify is not None:
191 all_results.append(outcome.verify)
192 total_wall_clock_s = round(sum(float(r.duration_s) for r in all_results), 3)
194 # Reproducibility signals (issue #41): the run seed and a stable hash of the
195 # effective config let a run be reproduced/explained. The seed is whatever
196 # the run was configured with (may be None when unseeded). The config hash
197 # is a pure function of config, so it is stable across runs and over time.
198 from .classification import classify
199 from .config import config_hash
201 # Execution / partial-result signals (issue #30) and adaptive-round signals
202 # (issue #40). ``skipped`` lists agents whose CLI was unavailable so they
203 # never ran; ``budget_exhausted`` flags a run that stopped early on the total
204 # timeout; ``stop_reason`` explains why debate ran or stopped.
205 skipped = [
206 {"name": name, "reason": reason} for name, reason in getattr(outcome, "skipped", []) or []
207 ]
208 retried = [a["name"] for a in agents if a["attempts"] > 1]
210 # Final-verdict mode (issue #220). ``decision`` is the effective mode (CLI
211 # override else config); ``vote`` is the tally dict when voting, else None.
212 decision = decision or config.decision
213 vote_meta = None
214 if vote is not None:
215 vote_meta = {
216 "verdict": vote.verdict,
217 "tally": vote.tally,
218 "ballots": [
219 {"reviewer": b.reviewer, "vote": b.vote, "reason": b.reason} for b in vote.ballots
220 ],
221 }
223 return {
224 "schema_version": SCHEMA_VERSION,
225 "decision": decision,
226 "vote": vote_meta,
227 "agents": agents,
228 # Configured vs effective panel size (issue #501): a slot that returned no
229 # review is an abstention, not an approval, and must not inflate the panel.
230 "panel": panel_accounting(outcome.reviews),
231 "economics": estimate_economics(all_results),
232 "rounds_executed": _rounds_executed(outcome),
233 "from_cache": bool(getattr(outcome, "from_cache", False)),
234 "stop_reason": getattr(outcome, "stop_reason", "") or "",
235 "skipped": skipped,
236 "retried": retried,
237 "budget_exhausted": bool(getattr(outcome, "budget_exhausted", False)),
238 "execution": {
239 "total_timeout": config.total_timeout,
240 "phase_timeout": config.phase_timeout,
241 "retries": config.retries,
242 "early_stop": config.early_stop,
243 "max_rounds": config.effective_max_rounds,
244 },
245 "verify_enabled": bool(config.verify),
246 "context_mode": outcome.context_mode,
247 "redact_secrets": bool(outcome.redact_secrets),
248 "redaction_count": outcome.redaction_count,
249 "seed": config.seed,
250 "config_hash": config_hash(config),
251 # PR-level classification (issue #7): deterministic summary derived from
252 # the structured findings + consensus groups. No diff text is included.
253 "classification": classify(outcome),
254 # Wall-clock is an approximate COST PROXY, not a dollar cost. No token
255 # counts are available from the underlying CLIs.
256 "total_wall_clock_s": total_wall_clock_s,
257 "cost_signal": "wall-clock-proxy",
258 "generated_at": datetime.now(UTC).isoformat(),
259 }
262# end