Coverage for src/ai_jury/formats.py: 100%
46 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
1"""Machine-readable renderers for the jury outcome.
3Markdown rendering lives in :mod:`ai_jury.report`. This module
4adds structured outputs intended for tooling:
6* :func:`to_json` -- a structured JSON report (schema documented in the README).
7* :func:`to_sarif` -- a SARIF 2.1.0 document for CI / code scanning upload.
8* :func:`to_keel_reviews` -- a per-reviewer review bundle (issue #663) for a
9 consumer that renders one head-pinned verdict per panelist.
11Both renderers are deterministic for a deterministic outcome (e.g. under
12``mock=True``) and only emit legitimate finding fields -- never raw diff or
13prompt text.
14"""
16from __future__ import annotations
18import json
19from typing import Any
21from . import __version__
22from .findings import SEVERITIES, Finding
23from .metadata import build_run_metadata
25#: Version of the JSON report schema produced by :func:`to_json`.
26#:
27#: 1.1 (issue #663) ADDED the top-level ``reviewers`` array. Nothing was removed,
28#: renamed or reshaped, so every existing consumer of ``findings``/``consensus``/
29#: ``verdicts``/``verdict``/``metadata`` reads an identical document.
30#:
31#: 1.2 (issue #700) adds ``scope``, ``testing``, ``model_source``,
32#: ``scope_substantive`` and ``counts_as_review`` to each ``reviewers`` entry —
33#: and, the reason this is a version bump rather than a silent addition, CHANGES
34#: two things a consumer may have keyed on:
35#:
36#: * ``reviewers[].model`` was "the configured model id, or ``""`` when the CLI's
37#: default is in force"; it is now never empty for a slot that has an agent,
38#: carrying either the id actually requested or a statement that the CLI chose
39#: and does not report which. A consumer testing ``model == ""`` to detect the
40#: default case must read ``model_source == "cli_default"`` instead.
41#: * the ``reviewers`` array now carries one entry per seat that **ran**, not per
42#: seat that returned output: a silent agent is recorded as an abstention
43#: naming it rather than dropped. A consumer counting the non-``chair`` entries
44#: as reviews must read ``counts_as_review`` (equivalently:
45#: ``scope_substantive`` and ``verdict != "ABSTAIN"``), which is the same rule
46#: its own ``review-verdict-insubstantial`` gate applies.
47#:
48#: 1.3 (issues #709, #710) changes nothing structurally and two things
49#: semantically, which is why it is a bump rather than a silent correction:
50#:
51#: * ``reviewers[].model`` under ``model_source: "requested"`` is now the id the
52#: invocation actually sent, read back off the result the adapter produced. It
53#: was recomputed from the seat's ``vendor`` while every invocation path
54#: computes it from the seat's ``adapter``, so on a seat where those differ
55#: (possible since #705) the ballot could name a model the run never asked for
56#: — under a field whose whole claim is that it is the id that was sent.
57#: * ``reviewers[].model_source`` gains a fifth value, ``recomputed``: no
58#: invocation recorded an id for this slot, so the one in ``model`` was derived
59#: from the run's config. It used to ship as ``requested``, which is the same
60#: overstatement one source short — the derivation returns
61#: ``gemini-3-pro-high`` for the seat above, whose adapter sent
62#: ``gemini-3-pro``. A consumer switching on the four known values must accept
63#: a fifth.
64#: * ``reviewers[].abstention_cause`` gains a fifth value, ``not_in_change``: the
65#: seat stated a scope and none of it is in the change under review. Those
66#: ballots were ``named_nothing`` before, when they were recorded at all — a
67#: ``Checked:`` line naming anything at all used to make the ballot a **review**
68#: (#710), so some of these entries were previously counted rather than
69#: bucketed. A consumer switching on the four known values must accept a fifth.
70#:
71#: Every top-level key, and every ``reviewers`` key, keeps its name and shape.
72#:
73#: 1.4 (issue #714) embeds metadata schema 7, which added ``metadata.routing``.
74#:
75#: 1.5 (issue #863) embeds metadata schema 8, which adds
76#: ``metadata.panel.zero_config_fallback``. Additive: nothing else changes.
77#:
78#: 1.6 (issue #975) adds ``reviewers[].error``: the adapter's short, redacted,
79#: one-line error (``[code] message``, at most 200 chars), ``""`` when the adapter
80#: reported none. Also a seat that produced no output because its allowance was
81#: spent (``quota_exhausted``, or ``rate_limited``) is now ``abstention_cause:
82#: "adapter_failed"``, not ``"silent"``. Additive; no new cause value.
83#:
84#: 1.7 (issue #978) adds the top-level ``excluded_files``: ``[{path, reason}]``
85#: for the files the large-diff planner kept out of the review (``binary``,
86#: ``generated``, ``filtered``, ``not_included``). ``[]`` when none. Additive.
87#:
88#: 1.8 (issue #976) adds ``origin`` (``"introduced"`` | ``"pre_existing"`` | ``"unclear"``)
89#: to each finding in ``findings`` and each consensus group in ``consensus``. Additive.
90JSON_SCHEMA_VERSION = "1.8"
92#: Canonical SARIF schema URI and version emitted by :func:`to_sarif`.
93SARIF_SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json"
94SARIF_VERSION = "2.1.0"
96TOOL_NAME = "ai-jury"
97TOOL_URI = "https://github.com/berkayturanci/ai-jury"
99#: Mapping from finding severity to SARIF result level.
100_SARIF_LEVEL = {
101 "critical": "error",
102 "major": "error",
103 "minor": "warning",
104 "nit": "note",
105 "info": "note",
106}
109def severity_to_sarif_level(severity: str) -> str:
110 """Map a jury severity to a SARIF result ``level``.
112 critical/major -> ``error``, minor -> ``warning``, nit/info -> ``note``.
113 Unknown severities fall back to ``note``.
114 """
115 return _SARIF_LEVEL.get(severity, "note")
118def _finding_dict(f: Finding) -> dict[str, Any]:
119 """Serialise a finding to a stable, ordered dict of legitimate fields."""
120 return {
121 "severity": f.severity,
122 "file": f.file,
123 "line": f.line,
124 "claim": f.claim,
125 "evidence": f.evidence,
126 "suggested_fix": f.suggested_fix,
127 "confidence": f.confidence,
128 "reviewer": f.reviewer,
129 "origin": f.origin,
130 }
133def _group_dict(g: Any) -> dict[str, Any]:
134 """Serialise a consensus group to a stable, ordered dict."""
135 return {
136 "representative": _finding_dict(g.representative),
137 "agreement": len(g.reviewers),
138 "reviewers": list(g.reviewers),
139 "bucket": g.bucket,
140 "verification_status": g.status or None,
141 "origin": g.origin,
142 }
145#: The documented ``excluded_files[].reason`` vocabulary, keyed by the planner's
146#: constants (#978); a reason this table does not know passes through unchanged.
147_EXCLUDE_REASON = {
148 "binary": "binary",
149 "generated": "generated",
150 "excluded-by-filter": "filtered",
151 "not-in-include-filter": "not_included",
152}
155def to_json(
156 outcome: Any,
157 config: Any,
158 *,
159 decision=None,
160 vote=None,
161 mode: str = "code",
162 zero_config_fallback: bool = False,
163) -> str:
164 """Render the jury outcome as a structured, pretty-printed JSON report.
166 Top-level keys: ``schema_version``, ``metadata`` (from
167 :func:`build_run_metadata`), ``findings``, ``consensus``, ``reviewers``,
168 ``verdicts`` and ``verdict`` (the chair synthesis text, if any). The result is
169 deterministic for a deterministic outcome and contains only legitimate finding
170 fields.
172 ``decision``/``vote`` are threaded into the metadata so the JSON report
173 reflects an effective ``--decision vote`` override (issue #248); when omitted
174 the metadata falls back to ``config.decision`` as before. ``mode`` selects the
175 ballot vocabulary (``code`` → APPROVE/COMMENT/REQUEST_CHANGES, ``issue`` →
176 READY/UNCLEAR/NEEDS_INFO), matching ``--issue``. ``zero_config_fallback`` is
177 passed through to the metadata (#863).
178 """
179 synthesis = getattr(outcome, "synthesis", None)
180 verdict_text = ""
181 if synthesis is not None and getattr(synthesis, "ok", False):
182 verdict_text = (synthesis.output or "").strip()
184 # Drop the wall-clock timestamp so the report is deterministic for a
185 # deterministic run (matching report.py, which omits generated_at too).
186 metadata = build_run_metadata(
187 outcome,
188 config,
189 decision=decision,
190 vote=vote,
191 mode=mode,
192 zero_config_fallback=zero_config_fallback,
193 )
194 metadata.pop("generated_at", None)
196 # Surface the deterministic PR-level classification (issue #7) at the top
197 # level for easy machine consumption (it is also embedded in ``metadata``).
198 from .ballots import reviewer_ballots
199 from .classification import classify
201 doc: dict[str, Any] = {
202 "schema_version": JSON_SCHEMA_VERSION,
203 "metadata": metadata,
204 "classification": classify(outcome),
205 "findings": [_finding_dict(f) for f in outcome.findings],
206 "consensus": [_group_dict(g) for g in outcome.groups],
207 # Per-reviewer ballots (issue #663): who said what, with vendor/model
208 # provenance. Purely additive — every key above is unchanged.
209 "reviewers": reviewer_ballots(outcome, config, vote=vote, mode=mode),
210 "verdicts": [
211 {
212 "file": v.file,
213 "line": v.line,
214 "claim": v.claim,
215 "status": v.status,
216 "reasoning": v.reasoning,
217 }
218 for v in outcome.verdicts
219 ],
220 "verdict": verdict_text,
221 # Files kept out of what the reviewers saw (#978, schema 1.7): binary
222 # diffs, generated files, filter and include misses. Additive.
223 "excluded_files": [
224 {"path": p, "reason": _EXCLUDE_REASON.get(why, why)}
225 for p, why in getattr(outcome, "excluded_files", [])
226 ],
227 }
228 return json.dumps(doc, indent=2, sort_keys=False, ensure_ascii=False)
231def to_keel_reviews(outcome: Any, config: Any, *, vote=None, mode: str = "code") -> str:
232 """Render the panel as a per-reviewer review bundle (issue #663).
234 A JSON **array** — not an object — of ``{reviewer, verdict, scope, findings,
235 testing, vendor, model, model_source, counts_as_review}`` records, one per
236 seat that ran plus the chair as ``reviewer: "chair"``. That is the payload
237 keel's ``keel review --reviews <file>`` accepts, so a panel run can *be* the
238 review rather than merely inform one. A seat that returned nothing is present
239 as an abstention naming it, and ``counts_as_review`` is how a consumer tells
240 the reviews from the records of seats that did not review.
242 Deterministic for a deterministic outcome. Free text lifted from agent replies
243 (``scope``, ``testing``) is flattened and capped in :mod:`ai_jury.ballots`; no
244 diff text, prompt text or secret ever reaches this document.
245 """
246 from .ballots import keel_reviews
248 return json.dumps(
249 keel_reviews(outcome, config, vote=vote, mode=mode),
250 indent=2,
251 sort_keys=False,
252 ensure_ascii=False,
253 )
256def _sarif_result(f: Finding) -> dict[str, Any]:
257 """Map a finding to a SARIF result object."""
258 physical: dict[str, Any] = {"artifactLocation": {"uri": f.file or ""}}
259 # SARIF 2.1.0 requires ``region.startLine`` to be a positive (1-based)
260 # integer. A reviewer's structured output is attacker-influenced (a finding's
261 # ``line`` is parsed from agent JSON that can be steered by the diff), so a
262 # forged ``"line": 0`` / negative value would emit an invalid region and make
263 # GitHub code-scanning reject the WHOLE SARIF upload — suppressing every
264 # finding (denial-of-evidence). Drop the region in that case so the finding
265 # still surfaces at file level (security audit 2026-06-13 r5).
266 if f.line is not None and f.line >= 1:
267 physical["region"] = {"startLine": f.line}
268 return {
269 "ruleId": f"jury/{f.severity}",
270 "level": severity_to_sarif_level(f.severity),
271 "message": {"text": f.claim},
272 "locations": [{"physicalLocation": physical}],
273 }
276def to_sarif(outcome: Any, _config: Any) -> str:
277 """Render the jury outcome as a SARIF 2.1.0 document.
279 Consensus group representatives are preferred as the source of results; if
280 there are no groups the raw findings are used. Rules are derived from the
281 severities actually present. The output is deterministic.
282 """
283 if outcome.groups:
284 findings = [g.representative for g in outcome.groups]
285 else:
286 findings = list(outcome.findings)
288 # Rules: one per severity actually used, in canonical severity order.
289 used = {f.severity for f in findings}
290 rules = [
291 {
292 "id": f"jury/{sev}",
293 "name": f"jury-{sev}",
294 "shortDescription": {"text": f"{sev} finding reported by the review jury"},
295 "defaultConfiguration": {"level": severity_to_sarif_level(sev)},
296 }
297 for sev in SEVERITIES
298 if sev in used
299 ]
301 doc = {
302 "$schema": SARIF_SCHEMA,
303 "version": SARIF_VERSION,
304 "runs": [
305 {
306 "tool": {
307 "driver": {
308 "name": TOOL_NAME,
309 "informationUri": TOOL_URI,
310 "version": __version__,
311 "rules": rules,
312 }
313 },
314 "results": [_sarif_result(f) for f in findings],
315 }
316 ],
317 }
318 return json.dumps(doc, indent=2, sort_keys=False, ensure_ascii=False)