Coverage for src/ai_jury/findings.py: 99%
146 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-20 18:15 +0000
1"""Machine-readable finding schema and parser.
3Reviewer/chair output is human-readable markdown, which is hard to dedupe, score,
4gate in CI, or turn into inline comments. This module defines a structured
5``Finding`` schema and a tolerant parser that extracts findings from an agent's
6raw output (a fenced ``json`` code block).
7"""
9from __future__ import annotations
11import json
12import re
13from dataclasses import dataclass
15from .redaction import redact
17SEVERITIES: tuple[str, ...] = ("critical", "major", "minor", "nit", "info")
18CONFIDENCES: tuple[str, ...] = ("high", "medium", "low")
19ORIGINS: tuple[str, ...] = ("introduced", "pre_existing", "unclear")
21# Output-injection guards for attacker-influenced finding text rendered into the
22# human-facing markdown report that is posted verbatim to the PR/issue (security
23# audit 2026-06-13 round 3). The machine CI gate is a pure function of the
24# structured fields and is unaffected by this text; these helpers only stop a
25# forged ``## Verdict APPROVE`` heading or a broken code fence from corrupting
26# the comment a human (or a downstream grep) reads.
27_FENCE_RUN_RE = re.compile(r"`{3,}|~{3,}")
28_HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL)
31def flatten_inline(text: str) -> str:
32 """Collapse text to a single line for safe inline rendering.
34 Markdown headings, list items, and code fences must begin a line, so
35 flattening newlines (and runs of whitespace) neutralizes forged structure
36 when the value is rendered inside a one-line list item.
37 """
38 if not text:
39 return text
40 return " ".join(str(text).split())
43def fence_safe(text: str) -> str:
44 """Break 3+ backtick/tilde runs so text rendered *inside* a code fence
45 (e.g. a ``suggestion`` block) cannot close the fence and inject markdown."""
46 if not text: 46 ↛ 47line 46 didn't jump to line 47 because the condition on line 46 was never true
47 return text
48 return _FENCE_RUN_RE.sub(lambda m: m.group()[0], str(text))
51def strip_html_comments(text: str) -> str:
52 """Remove HTML comments so attacker text can't forge the jury's hidden
53 inline-comment markers (``<!-- arc-inline -->`` / ``<!-- arc-sig:… -->``)."""
54 if not text:
55 return text
56 return _HTML_COMMENT_RE.sub("", str(text))
59# Verification verdict statuses (issue #3).
60VERDICT_STATUSES: tuple[str, ...] = ("verified", "unsupported", "needs_human_decision")
62# Lower number = more severe; useful for ranking/sorting.
63SEVERITY_ORDER: dict[str, int] = {sev: i for i, sev in enumerate(SEVERITIES)}
65# Legacy severity names mapped onto the canonical schema.
66_SEVERITY_ALIASES: dict[str, str] = {"blocker": "critical"}
68#: Every spelling an *operator* may write where a severity is asked for
69#: (``--fail-on``, ``[jury.ci] fail_on``): the canonical vocabulary plus the
70#: documented legacy aliases. Reviewer output is not held to this — a model that
71#: invents a severity is normalised to ``info``, not refused — but a human
72#: typing a CI gate is, because there is no safe default for "which findings
73#: fail the build" (issue #718).
74SEVERITY_INPUTS: tuple[str, ...] = SEVERITIES + tuple(_SEVERITY_ALIASES)
76_DEFAULT_SEVERITY = "info"
77_DEFAULT_CONFIDENCE = "medium"
78_DEFAULT_ORIGIN = "unclear"
80# Matches a fenced ```json ... ``` block (case-insensitive on the language tag).
81_JSON_BLOCK_RE = re.compile(r"```[ \t]*json[ \t]*\r?\n(.*?)```", re.DOTALL | re.IGNORECASE)
84def _normalize_severity(value: object) -> str:
85 if isinstance(value, str):
86 v = value.strip().lower()
87 v = _SEVERITY_ALIASES.get(v, v)
88 if v in SEVERITY_ORDER:
89 return v
90 return _DEFAULT_SEVERITY
93def canonical_severity(value: object) -> str | None:
94 """Canonical severity for an operator-supplied spelling, or ``None``.
96 The strict counterpart of :func:`_normalize_severity`: an unrecognised value
97 is reported as unknown instead of being demoted to ``info``, so a caller
98 that cannot tolerate a silent default (the CI gate) can refuse it.
99 """
100 v = str(value).strip().lower()
101 v = _SEVERITY_ALIASES.get(v, v)
102 return v if v in SEVERITY_ORDER else None
105def _normalize_confidence(value: object) -> str:
106 if isinstance(value, str):
107 v = value.strip().lower()
108 if v in CONFIDENCES:
109 return v
110 return _DEFAULT_CONFIDENCE
113_ORIGIN_ALIASES: dict[str, str] = {
114 "preexisting": "pre_existing",
115 "pre_exist": "pre_existing",
116}
119def _normalize_origin(value: object) -> str:
120 if isinstance(value, str):
121 v = value.strip().lower().replace("-", "_").replace(" ", "_")
122 v = _ORIGIN_ALIASES.get(v, v)
123 if v in ORIGINS:
124 return v
125 return _DEFAULT_ORIGIN
128@dataclass
129class Finding:
130 """A single structured review finding."""
132 severity: str
133 file: str
134 claim: str
135 line: int | None = None
136 evidence: str = ""
137 suggested_fix: str = ""
138 confidence: str = _DEFAULT_CONFIDENCE
139 reviewer: str = ""
140 origin: str = _DEFAULT_ORIGIN
142 def __post_init__(self) -> None:
143 self.severity = _normalize_severity(self.severity)
144 self.confidence = _normalize_confidence(self.confidence)
145 self.origin = _normalize_origin(self.origin)
146 if self.line is not None and not isinstance(self.line, bool):
147 try:
148 self.line = int(self.line)
149 except (TypeError, ValueError):
150 self.line = None
151 else:
152 self.line = None
154 @classmethod
155 def from_obj(cls, obj: dict, reviewer: str) -> Finding:
156 """Build a Finding from a decoded JSON object, forcing ``reviewer``."""
157 return cls(
158 severity=str(obj.get("severity", _DEFAULT_SEVERITY)),
159 file=str(obj.get("file", "")),
160 claim=str(obj.get("claim", "")),
161 line=obj.get("line"),
162 evidence=str(obj.get("evidence", "")),
163 suggested_fix=str(obj.get("suggested_fix", "")),
164 confidence=str(obj.get("confidence", _DEFAULT_CONFIDENCE)),
165 reviewer=reviewer,
166 origin=str(obj.get("origin", _DEFAULT_ORIGIN)),
167 )
170def emitted_findings_block(text: str) -> bool:
171 """Did the agent emit a structured findings block at all? (issue #501)
173 This is the mechanical line between *reviewed and found nothing* and *never
174 produced a review*. Both currently arrive as zero findings, so the panel reports
175 the same size either way — and on keel PR #660 two of three reviewers returned
176 assistant-style chatter about a flag they saw in the diff, while the run still
177 described itself as a three-agent panel.
179 A reviewer that examined the diff and found nothing still emits ``[]`` in a
180 fenced json block, because that is what the prompt asks for. One that wandered
181 off emits prose and no block. Presence of the block is therefore the signal, and
182 it needs no judgement about *content* — which is what keeps this deterministic.
183 """
184 return bool(text) and bool(_JSON_BLOCK_RE.search(text))
187def parse_findings(text: str, reviewer: str) -> tuple[list[Finding], list[str]]:
188 """Extract structured findings from an agent's raw output.
190 The agent is asked to emit a fenced ```json block holding a JSON array of
191 finding objects. We locate the *last* such block, decode it, and build
192 Finding objects (forcing ``reviewer`` to preserve identity).
194 Never raises. On a malformed/wrong-typed ``json`` block, returns
195 ``([], [warning])``. A legitimately missing block yields ``([], [])``.
196 """
197 if not text:
198 return [], []
200 blocks = _JSON_BLOCK_RE.findall(text)
201 if not blocks:
202 return [], []
204 raw = blocks[-1].strip()
205 try:
206 data = json.loads(raw)
207 except (ValueError, TypeError, RecursionError) as exc:
208 # RecursionError (deeply nested JSON, e.g. "[[[[…") is not a ValueError;
209 # catching it keeps the documented "never raises" contract so one
210 # steerable reviewer can't abort the whole run (audit 2026-06-13/N-2).
211 return [], [f"{reviewer}: malformed or missing structured findings ({redact(str(exc))[0]})"]
213 if not isinstance(data, list):
214 return [], [
215 f"{reviewer}: malformed or missing structured findings "
216 f"(expected a JSON array, got {type(data).__name__})"
217 ]
219 findings: list[Finding] = []
220 warnings: list[str] = []
221 for i, obj in enumerate(data):
222 if not isinstance(obj, dict):
223 warnings.append(
224 f"{reviewer}: malformed or missing structured findings "
225 f"(item {i} is {type(obj).__name__}, expected object)"
226 )
227 continue
228 findings.append(Finding.from_obj(obj, reviewer))
229 return findings, warnings
232def _coerce_line(value: object) -> int | None:
233 if value is None or isinstance(value, bool):
234 return None
235 try:
236 return int(value)
237 except (TypeError, ValueError):
238 return None
241def _normalize_status(value: object) -> str:
242 if isinstance(value, str):
243 v = value.strip().lower().replace("-", "_").replace(" ", "_")
244 if v in VERDICT_STATUSES:
245 return v
246 return "needs_human_decision"
249@dataclass
250class Verdict:
251 """A verifier's judgement on a candidate finding."""
253 file: str | None = None
254 line: int | None = None
255 claim: str = ""
256 status: str = "needs_human_decision"
257 reasoning: str = ""
260def parse_verdicts(text: str, verifier: str = "") -> tuple[list[Verdict], list[str]]:
261 """Extract verification verdicts from a verifier's raw output.
263 The verifier is asked to emit a fenced ```json block holding a JSON array of
264 verdict objects. We locate the *last* such block and decode it. Never raises;
265 on malformed input returns ``([], [warning])``.
266 """
267 label = verifier or "verifier"
268 if not text:
269 return [], [f"{label}: no verdicts (empty output)"]
271 blocks = _JSON_BLOCK_RE.findall(text)
272 if not blocks:
273 return [], [f"{label}: no JSON verdicts block found"]
275 raw = blocks[-1].strip()
276 try:
277 data = json.loads(raw)
278 except (ValueError, TypeError, RecursionError) as exc:
279 # See parse_findings: RecursionError on deeply nested JSON must not
280 # escape (audit 2026-06-13/N-2).
281 return [], [f"{label}: malformed verdicts JSON ({redact(str(exc))[0]})"]
283 if isinstance(data, dict):
284 data = data.get("verdicts", data.get("findings", []))
285 if not isinstance(data, list):
286 return [], [f"{label}: verdicts block is not a JSON array"]
288 verdicts: list[Verdict] = []
289 warnings: list[str] = []
290 for i, obj in enumerate(data):
291 if not isinstance(obj, dict):
292 warnings.append(f"{label}: verdict item {i} is {type(obj).__name__}, expected object")
293 continue
294 verdicts.append(
295 Verdict(
296 file=(obj.get("file") or None),
297 line=_coerce_line(obj.get("line")),
298 claim=str(obj.get("claim", "")).strip(),
299 status=_normalize_status(obj.get("status")),
300 reasoning=str(obj.get("reasoning", "")).strip(),
301 )
302 )
303 return verdicts, warnings