Coverage for src/ai_jury/formats.py: 100%

46 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 18:15 +0000

1"""Machine-readable renderers for the jury outcome. 

2 

3Markdown rendering lives in :mod:`ai_jury.report`. This module 

4adds structured outputs intended for tooling: 

5 

6* :func:`to_json` -- a structured JSON report (schema documented in the README). 

7* :func:`to_sarif` -- a SARIF 2.1.0 document for CI / code scanning upload. 

8* :func:`to_keel_reviews` -- a per-reviewer review bundle (issue #663) for a 

9 consumer that renders one head-pinned verdict per panelist. 

10 

11Both renderers are deterministic for a deterministic outcome (e.g. under 

12``mock=True``) and only emit legitimate finding fields -- never raw diff or 

13prompt text. 

14""" 

15 

16from __future__ import annotations 

17 

18import json 

19from typing import Any 

20 

21from . import __version__ 

22from .findings import SEVERITIES, Finding 

23from .metadata import build_run_metadata 

24 

25#: Version of the JSON report schema produced by :func:`to_json`. 

26#: 

27#: 1.1 (issue #663) ADDED the top-level ``reviewers`` array. Nothing was removed, 

28#: renamed or reshaped, so every existing consumer of ``findings``/``consensus``/ 

29#: ``verdicts``/``verdict``/``metadata`` reads an identical document. 

30#: 

31#: 1.2 (issue #700) adds ``scope``, ``testing``, ``model_source``, 

32#: ``scope_substantive`` and ``counts_as_review`` to each ``reviewers`` entry — 

33#: and, the reason this is a version bump rather than a silent addition, CHANGES 

34#: two things a consumer may have keyed on: 

35#: 

36#: * ``reviewers[].model`` was "the configured model id, or ``""`` when the CLI's 

37#: default is in force"; it is now never empty for a slot that has an agent, 

38#: carrying either the id actually requested or a statement that the CLI chose 

39#: and does not report which. A consumer testing ``model == ""`` to detect the 

40#: default case must read ``model_source == "cli_default"`` instead. 

41#: * the ``reviewers`` array now carries one entry per seat that **ran**, not per 

42#: seat that returned output: a silent agent is recorded as an abstention 

43#: naming it rather than dropped. A consumer counting the non-``chair`` entries 

44#: as reviews must read ``counts_as_review`` (equivalently: 

45#: ``scope_substantive`` and ``verdict != "ABSTAIN"``), which is the same rule 

46#: its own ``review-verdict-insubstantial`` gate applies. 

47#: 

48#: 1.3 (issues #709, #710) changes nothing structurally and two things 

49#: semantically, which is why it is a bump rather than a silent correction: 

50#: 

51#: * ``reviewers[].model`` under ``model_source: "requested"`` is now the id the 

52#: invocation actually sent, read back off the result the adapter produced. It 

53#: was recomputed from the seat's ``vendor`` while every invocation path 

54#: computes it from the seat's ``adapter``, so on a seat where those differ 

55#: (possible since #705) the ballot could name a model the run never asked for 

56#: — under a field whose whole claim is that it is the id that was sent. 

57#: * ``reviewers[].model_source`` gains a fifth value, ``recomputed``: no 

58#: invocation recorded an id for this slot, so the one in ``model`` was derived 

59#: from the run's config. It used to ship as ``requested``, which is the same 

60#: overstatement one source short — the derivation returns 

61#: ``gemini-3-pro-high`` for the seat above, whose adapter sent 

62#: ``gemini-3-pro``. A consumer switching on the four known values must accept 

63#: a fifth. 

64#: * ``reviewers[].abstention_cause`` gains a fifth value, ``not_in_change``: the 

65#: seat stated a scope and none of it is in the change under review. Those 

66#: ballots were ``named_nothing`` before, when they were recorded at all — a 

67#: ``Checked:`` line naming anything at all used to make the ballot a **review** 

68#: (#710), so some of these entries were previously counted rather than 

69#: bucketed. A consumer switching on the four known values must accept a fifth. 

70#: 

71#: Every top-level key, and every ``reviewers`` key, keeps its name and shape. 

72#: 

73#: 1.4 (issue #714) embeds metadata schema 7, which added ``metadata.routing``. 

74#: 

75#: 1.5 (issue #863) embeds metadata schema 8, which adds 

76#: ``metadata.panel.zero_config_fallback``. Additive: nothing else changes. 

77#: 

78#: 1.6 (issue #975) adds ``reviewers[].error``: the adapter's short, redacted, 

79#: one-line error (``[code] message``, at most 200 chars), ``""`` when the adapter 

80#: reported none. Also a seat that produced no output because its allowance was 

81#: spent (``quota_exhausted``, or ``rate_limited``) is now ``abstention_cause: 

82#: "adapter_failed"``, not ``"silent"``. Additive; no new cause value. 

83#: 

84#: 1.7 (issue #978) adds the top-level ``excluded_files``: ``[{path, reason}]`` 

85#: for the files the large-diff planner kept out of the review (``binary``, 

86#: ``generated``, ``filtered``, ``not_included``). ``[]`` when none. Additive. 

87#: 

88#: 1.8 (issue #976) adds ``origin`` (``"introduced"`` | ``"pre_existing"`` | ``"unclear"``) 

89#: to each finding in ``findings`` and each consensus group in ``consensus``. Additive. 

90JSON_SCHEMA_VERSION = "1.8" 

91 

92#: Canonical SARIF schema URI and version emitted by :func:`to_sarif`. 

93SARIF_SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json" 

94SARIF_VERSION = "2.1.0" 

95 

96TOOL_NAME = "ai-jury" 

97TOOL_URI = "https://github.com/berkayturanci/ai-jury" 

98 

99#: Mapping from finding severity to SARIF result level. 

100_SARIF_LEVEL = { 

101 "critical": "error", 

102 "major": "error", 

103 "minor": "warning", 

104 "nit": "note", 

105 "info": "note", 

106} 

107 

108 

109def severity_to_sarif_level(severity: str) -> str: 

110 """Map a jury severity to a SARIF result ``level``. 

111 

112 critical/major -> ``error``, minor -> ``warning``, nit/info -> ``note``. 

113 Unknown severities fall back to ``note``. 

114 """ 

115 return _SARIF_LEVEL.get(severity, "note") 

116 

117 

118def _finding_dict(f: Finding) -> dict[str, Any]: 

119 """Serialise a finding to a stable, ordered dict of legitimate fields.""" 

120 return { 

121 "severity": f.severity, 

122 "file": f.file, 

123 "line": f.line, 

124 "claim": f.claim, 

125 "evidence": f.evidence, 

126 "suggested_fix": f.suggested_fix, 

127 "confidence": f.confidence, 

128 "reviewer": f.reviewer, 

129 "origin": f.origin, 

130 } 

131 

132 

133def _group_dict(g: Any) -> dict[str, Any]: 

134 """Serialise a consensus group to a stable, ordered dict.""" 

135 return { 

136 "representative": _finding_dict(g.representative), 

137 "agreement": len(g.reviewers), 

138 "reviewers": list(g.reviewers), 

139 "bucket": g.bucket, 

140 "verification_status": g.status or None, 

141 "origin": g.origin, 

142 } 

143 

144 

145#: The documented ``excluded_files[].reason`` vocabulary, keyed by the planner's 

146#: constants (#978); a reason this table does not know passes through unchanged. 

147_EXCLUDE_REASON = { 

148 "binary": "binary", 

149 "generated": "generated", 

150 "excluded-by-filter": "filtered", 

151 "not-in-include-filter": "not_included", 

152} 

153 

154 

155def to_json( 

156 outcome: Any, 

157 config: Any, 

158 *, 

159 decision=None, 

160 vote=None, 

161 mode: str = "code", 

162 zero_config_fallback: bool = False, 

163) -> str: 

164 """Render the jury outcome as a structured, pretty-printed JSON report. 

165 

166 Top-level keys: ``schema_version``, ``metadata`` (from 

167 :func:`build_run_metadata`), ``findings``, ``consensus``, ``reviewers``, 

168 ``verdicts`` and ``verdict`` (the chair synthesis text, if any). The result is 

169 deterministic for a deterministic outcome and contains only legitimate finding 

170 fields. 

171 

172 ``decision``/``vote`` are threaded into the metadata so the JSON report 

173 reflects an effective ``--decision vote`` override (issue #248); when omitted 

174 the metadata falls back to ``config.decision`` as before. ``mode`` selects the 

175 ballot vocabulary (``code`` → APPROVE/COMMENT/REQUEST_CHANGES, ``issue`` → 

176 READY/UNCLEAR/NEEDS_INFO), matching ``--issue``. ``zero_config_fallback`` is 

177 passed through to the metadata (#863). 

178 """ 

179 synthesis = getattr(outcome, "synthesis", None) 

180 verdict_text = "" 

181 if synthesis is not None and getattr(synthesis, "ok", False): 

182 verdict_text = (synthesis.output or "").strip() 

183 

184 # Drop the wall-clock timestamp so the report is deterministic for a 

185 # deterministic run (matching report.py, which omits generated_at too). 

186 metadata = build_run_metadata( 

187 outcome, 

188 config, 

189 decision=decision, 

190 vote=vote, 

191 mode=mode, 

192 zero_config_fallback=zero_config_fallback, 

193 ) 

194 metadata.pop("generated_at", None) 

195 

196 # Surface the deterministic PR-level classification (issue #7) at the top 

197 # level for easy machine consumption (it is also embedded in ``metadata``). 

198 from .ballots import reviewer_ballots 

199 from .classification import classify 

200 

201 doc: dict[str, Any] = { 

202 "schema_version": JSON_SCHEMA_VERSION, 

203 "metadata": metadata, 

204 "classification": classify(outcome), 

205 "findings": [_finding_dict(f) for f in outcome.findings], 

206 "consensus": [_group_dict(g) for g in outcome.groups], 

207 # Per-reviewer ballots (issue #663): who said what, with vendor/model 

208 # provenance. Purely additive — every key above is unchanged. 

209 "reviewers": reviewer_ballots(outcome, config, vote=vote, mode=mode), 

210 "verdicts": [ 

211 { 

212 "file": v.file, 

213 "line": v.line, 

214 "claim": v.claim, 

215 "status": v.status, 

216 "reasoning": v.reasoning, 

217 } 

218 for v in outcome.verdicts 

219 ], 

220 "verdict": verdict_text, 

221 # Files kept out of what the reviewers saw (#978, schema 1.7): binary 

222 # diffs, generated files, filter and include misses. Additive. 

223 "excluded_files": [ 

224 {"path": p, "reason": _EXCLUDE_REASON.get(why, why)} 

225 for p, why in getattr(outcome, "excluded_files", []) 

226 ], 

227 } 

228 return json.dumps(doc, indent=2, sort_keys=False, ensure_ascii=False) 

229 

230 

231def to_keel_reviews(outcome: Any, config: Any, *, vote=None, mode: str = "code") -> str: 

232 """Render the panel as a per-reviewer review bundle (issue #663). 

233 

234 A JSON **array** — not an object — of ``{reviewer, verdict, scope, findings, 

235 testing, vendor, model, model_source, counts_as_review}`` records, one per 

236 seat that ran plus the chair as ``reviewer: "chair"``. That is the payload 

237 keel's ``keel review --reviews <file>`` accepts, so a panel run can *be* the 

238 review rather than merely inform one. A seat that returned nothing is present 

239 as an abstention naming it, and ``counts_as_review`` is how a consumer tells 

240 the reviews from the records of seats that did not review. 

241 

242 Deterministic for a deterministic outcome. Free text lifted from agent replies 

243 (``scope``, ``testing``) is flattened and capped in :mod:`ai_jury.ballots`; no 

244 diff text, prompt text or secret ever reaches this document. 

245 """ 

246 from .ballots import keel_reviews 

247 

248 return json.dumps( 

249 keel_reviews(outcome, config, vote=vote, mode=mode), 

250 indent=2, 

251 sort_keys=False, 

252 ensure_ascii=False, 

253 ) 

254 

255 

256def _sarif_result(f: Finding) -> dict[str, Any]: 

257 """Map a finding to a SARIF result object.""" 

258 physical: dict[str, Any] = {"artifactLocation": {"uri": f.file or ""}} 

259 # SARIF 2.1.0 requires ``region.startLine`` to be a positive (1-based) 

260 # integer. A reviewer's structured output is attacker-influenced (a finding's 

261 # ``line`` is parsed from agent JSON that can be steered by the diff), so a 

262 # forged ``"line": 0`` / negative value would emit an invalid region and make 

263 # GitHub code-scanning reject the WHOLE SARIF upload — suppressing every 

264 # finding (denial-of-evidence). Drop the region in that case so the finding 

265 # still surfaces at file level (security audit 2026-06-13 r5). 

266 if f.line is not None and f.line >= 1: 

267 physical["region"] = {"startLine": f.line} 

268 return { 

269 "ruleId": f"jury/{f.severity}", 

270 "level": severity_to_sarif_level(f.severity), 

271 "message": {"text": f.claim}, 

272 "locations": [{"physicalLocation": physical}], 

273 } 

274 

275 

276def to_sarif(outcome: Any, _config: Any) -> str: 

277 """Render the jury outcome as a SARIF 2.1.0 document. 

278 

279 Consensus group representatives are preferred as the source of results; if 

280 there are no groups the raw findings are used. Rules are derived from the 

281 severities actually present. The output is deterministic. 

282 """ 

283 if outcome.groups: 

284 findings = [g.representative for g in outcome.groups] 

285 else: 

286 findings = list(outcome.findings) 

287 

288 # Rules: one per severity actually used, in canonical severity order. 

289 used = {f.severity for f in findings} 

290 rules = [ 

291 { 

292 "id": f"jury/{sev}", 

293 "name": f"jury-{sev}", 

294 "shortDescription": {"text": f"{sev} finding reported by the review jury"}, 

295 "defaultConfiguration": {"level": severity_to_sarif_level(sev)}, 

296 } 

297 for sev in SEVERITIES 

298 if sev in used 

299 ] 

300 

301 doc = { 

302 "$schema": SARIF_SCHEMA, 

303 "version": SARIF_VERSION, 

304 "runs": [ 

305 { 

306 "tool": { 

307 "driver": { 

308 "name": TOOL_NAME, 

309 "informationUri": TOOL_URI, 

310 "version": __version__, 

311 "rules": rules, 

312 } 

313 }, 

314 "results": [_sarif_result(f) for f in findings], 

315 } 

316 ], 

317 } 

318 return json.dumps(doc, indent=2, sort_keys=False, ensure_ascii=False)