Coverage for src/ai_jury/findings.py: 99%

146 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 18:15 +0000

1"""Machine-readable finding schema and parser. 

2 

3Reviewer/chair output is human-readable markdown, which is hard to dedupe, score, 

4gate in CI, or turn into inline comments. This module defines a structured 

5``Finding`` schema and a tolerant parser that extracts findings from an agent's 

6raw output (a fenced ``json`` code block). 

7""" 

8 

9from __future__ import annotations 

10 

11import json 

12import re 

13from dataclasses import dataclass 

14 

15from .redaction import redact 

16 

17SEVERITIES: tuple[str, ...] = ("critical", "major", "minor", "nit", "info") 

18CONFIDENCES: tuple[str, ...] = ("high", "medium", "low") 

19ORIGINS: tuple[str, ...] = ("introduced", "pre_existing", "unclear") 

20 

21# Output-injection guards for attacker-influenced finding text rendered into the 

22# human-facing markdown report that is posted verbatim to the PR/issue (security 

23# audit 2026-06-13 round 3). The machine CI gate is a pure function of the 

24# structured fields and is unaffected by this text; these helpers only stop a 

25# forged ``## Verdict APPROVE`` heading or a broken code fence from corrupting 

26# the comment a human (or a downstream grep) reads. 

27_FENCE_RUN_RE = re.compile(r"`{3,}|~{3,}") 

28_HTML_COMMENT_RE = re.compile(r"<!--.*?-->", re.DOTALL) 

29 

30 

31def flatten_inline(text: str) -> str: 

32 """Collapse text to a single line for safe inline rendering. 

33 

34 Markdown headings, list items, and code fences must begin a line, so 

35 flattening newlines (and runs of whitespace) neutralizes forged structure 

36 when the value is rendered inside a one-line list item. 

37 """ 

38 if not text: 

39 return text 

40 return " ".join(str(text).split()) 

41 

42 

43def fence_safe(text: str) -> str: 

44 """Break 3+ backtick/tilde runs so text rendered *inside* a code fence 

45 (e.g. a ``suggestion`` block) cannot close the fence and inject markdown.""" 

46 if not text: 46 ↛ 47line 46 didn't jump to line 47 because the condition on line 46 was never true

47 return text 

48 return _FENCE_RUN_RE.sub(lambda m: m.group()[0], str(text)) 

49 

50 

51def strip_html_comments(text: str) -> str: 

52 """Remove HTML comments so attacker text can't forge the jury's hidden 

53 inline-comment markers (``<!-- arc-inline -->`` / ``<!-- arc-sig:… -->``).""" 

54 if not text: 

55 return text 

56 return _HTML_COMMENT_RE.sub("", str(text)) 

57 

58 

59# Verification verdict statuses (issue #3). 

60VERDICT_STATUSES: tuple[str, ...] = ("verified", "unsupported", "needs_human_decision") 

61 

62# Lower number = more severe; useful for ranking/sorting. 

63SEVERITY_ORDER: dict[str, int] = {sev: i for i, sev in enumerate(SEVERITIES)} 

64 

65# Legacy severity names mapped onto the canonical schema. 

66_SEVERITY_ALIASES: dict[str, str] = {"blocker": "critical"} 

67 

68#: Every spelling an *operator* may write where a severity is asked for 

69#: (``--fail-on``, ``[jury.ci] fail_on``): the canonical vocabulary plus the 

70#: documented legacy aliases. Reviewer output is not held to this — a model that 

71#: invents a severity is normalised to ``info``, not refused — but a human 

72#: typing a CI gate is, because there is no safe default for "which findings 

73#: fail the build" (issue #718). 

74SEVERITY_INPUTS: tuple[str, ...] = SEVERITIES + tuple(_SEVERITY_ALIASES) 

75 

76_DEFAULT_SEVERITY = "info" 

77_DEFAULT_CONFIDENCE = "medium" 

78_DEFAULT_ORIGIN = "unclear" 

79 

80# Matches a fenced ```json ... ``` block (case-insensitive on the language tag). 

81_JSON_BLOCK_RE = re.compile(r"```[ \t]*json[ \t]*\r?\n(.*?)```", re.DOTALL | re.IGNORECASE) 

82 

83 

84def _normalize_severity(value: object) -> str: 

85 if isinstance(value, str): 

86 v = value.strip().lower() 

87 v = _SEVERITY_ALIASES.get(v, v) 

88 if v in SEVERITY_ORDER: 

89 return v 

90 return _DEFAULT_SEVERITY 

91 

92 

93def canonical_severity(value: object) -> str | None: 

94 """Canonical severity for an operator-supplied spelling, or ``None``. 

95 

96 The strict counterpart of :func:`_normalize_severity`: an unrecognised value 

97 is reported as unknown instead of being demoted to ``info``, so a caller 

98 that cannot tolerate a silent default (the CI gate) can refuse it. 

99 """ 

100 v = str(value).strip().lower() 

101 v = _SEVERITY_ALIASES.get(v, v) 

102 return v if v in SEVERITY_ORDER else None 

103 

104 

105def _normalize_confidence(value: object) -> str: 

106 if isinstance(value, str): 

107 v = value.strip().lower() 

108 if v in CONFIDENCES: 

109 return v 

110 return _DEFAULT_CONFIDENCE 

111 

112 

113_ORIGIN_ALIASES: dict[str, str] = { 

114 "preexisting": "pre_existing", 

115 "pre_exist": "pre_existing", 

116} 

117 

118 

119def _normalize_origin(value: object) -> str: 

120 if isinstance(value, str): 

121 v = value.strip().lower().replace("-", "_").replace(" ", "_") 

122 v = _ORIGIN_ALIASES.get(v, v) 

123 if v in ORIGINS: 

124 return v 

125 return _DEFAULT_ORIGIN 

126 

127 

128@dataclass 

129class Finding: 

130 """A single structured review finding.""" 

131 

132 severity: str 

133 file: str 

134 claim: str 

135 line: int | None = None 

136 evidence: str = "" 

137 suggested_fix: str = "" 

138 confidence: str = _DEFAULT_CONFIDENCE 

139 reviewer: str = "" 

140 origin: str = _DEFAULT_ORIGIN 

141 

142 def __post_init__(self) -> None: 

143 self.severity = _normalize_severity(self.severity) 

144 self.confidence = _normalize_confidence(self.confidence) 

145 self.origin = _normalize_origin(self.origin) 

146 if self.line is not None and not isinstance(self.line, bool): 

147 try: 

148 self.line = int(self.line) 

149 except (TypeError, ValueError): 

150 self.line = None 

151 else: 

152 self.line = None 

153 

154 @classmethod 

155 def from_obj(cls, obj: dict, reviewer: str) -> Finding: 

156 """Build a Finding from a decoded JSON object, forcing ``reviewer``.""" 

157 return cls( 

158 severity=str(obj.get("severity", _DEFAULT_SEVERITY)), 

159 file=str(obj.get("file", "")), 

160 claim=str(obj.get("claim", "")), 

161 line=obj.get("line"), 

162 evidence=str(obj.get("evidence", "")), 

163 suggested_fix=str(obj.get("suggested_fix", "")), 

164 confidence=str(obj.get("confidence", _DEFAULT_CONFIDENCE)), 

165 reviewer=reviewer, 

166 origin=str(obj.get("origin", _DEFAULT_ORIGIN)), 

167 ) 

168 

169 

170def emitted_findings_block(text: str) -> bool: 

171 """Did the agent emit a structured findings block at all? (issue #501) 

172 

173 This is the mechanical line between *reviewed and found nothing* and *never 

174 produced a review*. Both currently arrive as zero findings, so the panel reports 

175 the same size either way — and on keel PR #660 two of three reviewers returned 

176 assistant-style chatter about a flag they saw in the diff, while the run still 

177 described itself as a three-agent panel. 

178 

179 A reviewer that examined the diff and found nothing still emits ``[]`` in a 

180 fenced json block, because that is what the prompt asks for. One that wandered 

181 off emits prose and no block. Presence of the block is therefore the signal, and 

182 it needs no judgement about *content* — which is what keeps this deterministic. 

183 """ 

184 return bool(text) and bool(_JSON_BLOCK_RE.search(text)) 

185 

186 

187def parse_findings(text: str, reviewer: str) -> tuple[list[Finding], list[str]]: 

188 """Extract structured findings from an agent's raw output. 

189 

190 The agent is asked to emit a fenced ```json block holding a JSON array of 

191 finding objects. We locate the *last* such block, decode it, and build 

192 Finding objects (forcing ``reviewer`` to preserve identity). 

193 

194 Never raises. On a malformed/wrong-typed ``json`` block, returns 

195 ``([], [warning])``. A legitimately missing block yields ``([], [])``. 

196 """ 

197 if not text: 

198 return [], [] 

199 

200 blocks = _JSON_BLOCK_RE.findall(text) 

201 if not blocks: 

202 return [], [] 

203 

204 raw = blocks[-1].strip() 

205 try: 

206 data = json.loads(raw) 

207 except (ValueError, TypeError, RecursionError) as exc: 

208 # RecursionError (deeply nested JSON, e.g. "[[[[…") is not a ValueError; 

209 # catching it keeps the documented "never raises" contract so one 

210 # steerable reviewer can't abort the whole run (audit 2026-06-13/N-2). 

211 return [], [f"{reviewer}: malformed or missing structured findings ({redact(str(exc))[0]})"] 

212 

213 if not isinstance(data, list): 

214 return [], [ 

215 f"{reviewer}: malformed or missing structured findings " 

216 f"(expected a JSON array, got {type(data).__name__})" 

217 ] 

218 

219 findings: list[Finding] = [] 

220 warnings: list[str] = [] 

221 for i, obj in enumerate(data): 

222 if not isinstance(obj, dict): 

223 warnings.append( 

224 f"{reviewer}: malformed or missing structured findings " 

225 f"(item {i} is {type(obj).__name__}, expected object)" 

226 ) 

227 continue 

228 findings.append(Finding.from_obj(obj, reviewer)) 

229 return findings, warnings 

230 

231 

232def _coerce_line(value: object) -> int | None: 

233 if value is None or isinstance(value, bool): 

234 return None 

235 try: 

236 return int(value) 

237 except (TypeError, ValueError): 

238 return None 

239 

240 

241def _normalize_status(value: object) -> str: 

242 if isinstance(value, str): 

243 v = value.strip().lower().replace("-", "_").replace(" ", "_") 

244 if v in VERDICT_STATUSES: 

245 return v 

246 return "needs_human_decision" 

247 

248 

249@dataclass 

250class Verdict: 

251 """A verifier's judgement on a candidate finding.""" 

252 

253 file: str | None = None 

254 line: int | None = None 

255 claim: str = "" 

256 status: str = "needs_human_decision" 

257 reasoning: str = "" 

258 

259 

260def parse_verdicts(text: str, verifier: str = "") -> tuple[list[Verdict], list[str]]: 

261 """Extract verification verdicts from a verifier's raw output. 

262 

263 The verifier is asked to emit a fenced ```json block holding a JSON array of 

264 verdict objects. We locate the *last* such block and decode it. Never raises; 

265 on malformed input returns ``([], [warning])``. 

266 """ 

267 label = verifier or "verifier" 

268 if not text: 

269 return [], [f"{label}: no verdicts (empty output)"] 

270 

271 blocks = _JSON_BLOCK_RE.findall(text) 

272 if not blocks: 

273 return [], [f"{label}: no JSON verdicts block found"] 

274 

275 raw = blocks[-1].strip() 

276 try: 

277 data = json.loads(raw) 

278 except (ValueError, TypeError, RecursionError) as exc: 

279 # See parse_findings: RecursionError on deeply nested JSON must not 

280 # escape (audit 2026-06-13/N-2). 

281 return [], [f"{label}: malformed verdicts JSON ({redact(str(exc))[0]})"] 

282 

283 if isinstance(data, dict): 

284 data = data.get("verdicts", data.get("findings", [])) 

285 if not isinstance(data, list): 

286 return [], [f"{label}: verdicts block is not a JSON array"] 

287 

288 verdicts: list[Verdict] = [] 

289 warnings: list[str] = [] 

290 for i, obj in enumerate(data): 

291 if not isinstance(obj, dict): 

292 warnings.append(f"{label}: verdict item {i} is {type(obj).__name__}, expected object") 

293 continue 

294 verdicts.append( 

295 Verdict( 

296 file=(obj.get("file") or None), 

297 line=_coerce_line(obj.get("line")), 

298 claim=str(obj.get("claim", "")).strip(), 

299 status=_normalize_status(obj.get("status")), 

300 reasoning=str(obj.get("reasoning", "")).strip(), 

301 ) 

302 ) 

303 return verdicts, warnings