Coverage for src/ai_jury/largediff.py: 96%

1"""Large-diff handling: filtering and chunking (issue #31).

3The jury sends the diff to every agent, so a large or generated diff inflates

4cost, runtime, and prompt size. This module measures a diff, drops files that

5should not be reviewed (binary blobs, generated/vendored files, and anything the

6configured path filters exclude), and decides a handling mode:

8- ``full`` — kept diff fits the budget; review it in one pass.

9- ``chunked`` — kept diff is over budget and chunking is enabled; split it

10 into per-file chunks each within the chunk budget.

11- ``too_large`` — over budget and chunking is disabled; the caller should fail

12 with a clear message.

14Everything here is PURE and deterministic: parsing, classification, and chunk

15boundaries are a function of the diff text and config only, so the plan is

16reproducible and unit-testable.

17"""

19from __future__ import annotations

21import fnmatch

22import re

23from dataclasses import dataclass, field

25# Default "generated / not worth reviewing" path globs. Conservative and

26# language-agnostic; users extend or replace via ``[jury.diff] exclude``.

27DEFAULT_GENERATED_GLOBS: tuple[str, ...] = (

28 # Dependency lockfiles.

29 "*.lock",

30 "package-lock.json",

31 "yarn.lock",

32 "pnpm-lock.yaml",

33 "poetry.lock",

34 "Cargo.lock",

35 "composer.lock",

36 "Gemfile.lock",

37 "go.sum",

38 # Minified / map artifacts.

39 "*.min.js",

40 "*.min.css",

41 "*.map",

42 # Snapshots and common generated code.

43 "*.snap",

44 "*.pb.go",

45 "*_pb2.py",

46 "*_pb2_grpc.py",

47 # Vendored / build output directories.

48 "vendor/**",

49 "node_modules/**",

50 "dist/**",

51 "build/**",

52)

54EXCLUDE_BINARY = "binary"

55EXCLUDE_GENERATED = "generated"

56EXCLUDE_FILTER = "excluded-by-filter"

57EXCLUDE_NOT_INCLUDED = "not-in-include-filter"

59MODE_FULL = "full"

60MODE_CHUNKED = "chunked"

61MODE_TOO_LARGE = "too_large"

64@dataclass

65class DiffFile:

66 """One file's segment of a unified diff."""

68 path: str

69 text: str

71 @property

72 def size_bytes(self) -> int:

73 return len(self.text.encode("utf-8"))

76@dataclass

77class DiffPlan:

78 mode: str

79 chunks: list[str] = field(default_factory=list)

80 kept: list[DiffFile] = field(default_factory=list)

81 excluded: list[tuple[str, str]] = field(default_factory=list)

82 total_bytes: int = 0

83 kept_bytes: int = 0

84 reason: str = ""

86 @property

87 def kept_paths(self) -> list[str]:

88 return [f.path for f in self.kept]

91def _strip_ab(path: str) -> str:

92 for prefix in ("a/", "b/"):

93 if path.startswith(prefix):

94 return path[len(prefix) :]

95 return path

98def _unquote_git_path(path: str) -> str:

99 """Undo git's C-style quoting of paths with special chars (best-effort).

100

101 git wraps a path in double quotes and octal-escapes special/non-ASCII bytes

102 when ``core.quotepath`` is on. We decode it back so the full path is

103 recovered for glob filtering and classification.

104 """

105 if len(path) >= 2 and path.startswith('"') and path.endswith('"'):

106 inner = path[1:-1]

107 try:

108 return (

109 inner.encode("latin-1", "backslashreplace")

110 .decode("unicode_escape")

111 .encode("latin-1")

112 .decode("utf-8", "replace")

113 )

114 except (UnicodeDecodeError, UnicodeEncodeError):

115 return inner.replace('\\"', '"').replace("\\\\", "\\")

116 return path

117

118

119def _path_from_marker(line: str) -> str | None:

120 """Path from a ``+++ b/`` or ``--- a/`` line, or None for /dev/null.

121

122 These marker lines carry a single, unambiguous path even when it contains

123 spaces or quoted special chars — unlike the ``diff --git a/ b/``

124 header, which a ``str.split()`` truncates at the first space, hiding or

125 mislabeling the file (security audit 2026-06-13/L-4,N-3).

126 """

127 rest = line[4:].rstrip("\r\n")

128 # Some diff formats append a tab + timestamp; the path ends at the tab.

129 if "\t" in rest: 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true

130 rest = rest.split("\t", 1)[0]

131 if rest == "/dev/null": 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true

132 return None

133 return _strip_ab(_unquote_git_path(rest))

134

135

136def _path_from_git_header(line: str) -> str:

137 """Best-effort new-side path from a ``diff --git a/ b/`` header.

138

139 ``str.split()[3]`` truncates a space-containing name; split on the last

140 `` b/`` separator instead so the full b-side path is recovered, then unquote

141 git's C-quoting (audit 2026-06-13/L-4, r3 marker-less case).

142 """

143 rest = line[len("diff --git ") :].rstrip("\r\n")

144 # Non-rename headers are symmetric: ``a/ b/`` with the SAME on both

145 # sides. Recover by halving, which is robust even when itself

146 # contains `` b/`` (a mode-change-only segment has no +++/--- or rename

147 # marker to fall back on — audit 2026-06-13 r4/L). len(body) = 2*len(p)+3.

148 if rest.startswith("a/"):

149 body = rest[2:]

150 half = (len(body) - 3) // 2

151 if len(body) >= 3 and body[half : half + 3] == " b/" and body[:half] == body[half + 3 :]:

152 return _unquote_git_path(body[:half])

153 idx = rest.rfind(" b/")

154 if idx != -1:

155 return _strip_ab(_unquote_git_path(rest[idx + 1 :]))

156 # Quoted b-side: git C-quotes special/spaced paths as `"a/" "b/"`, so

157 # the separator is `` "b/`` not `` b/`` (audit 2026-06-13 r5/L).

158 qidx = rest.rfind(' "b/')

159 if qidx != -1: 159 ↛ 161line 159 didn't jump to line 161 because the condition on line 159 was always true

160 return _strip_ab(_unquote_git_path(rest[qidx + 1 :]))

161 parts = line.split()

162 return _strip_ab(parts[3]) if len(parts) >= 4 else _strip_ab(parts[-1])

163

164

165def split_diff(diff: str) -> list[DiffFile]:

166 """Split a unified diff into per-file segments.

167

168 Segments start at ``diff --git a/ b/`` headers (the git format the

169 adapters emit). Any preamble before the first header is attached to the first

170 file so no bytes are silently dropped. A diff with no ``diff --git`` header is

171 returned as a single unnamed segment (it cannot be chunked by file).

172 """

173 if not diff:

174 return []

175

176 files: list[DiffFile] = []

177

178 parts = []

179 # bolt: avoid allocating a huge list of strings from splitlines(keepends=True)

180 # by splitting chunks directly and only iterating their header lines.

181 idx = diff.find("diff --git ")

182 if idx == -1:

183 parts = [diff]

184 else:

185 if idx > 0:

186 parts.append(diff[:idx])

187

188 while idx != -1: 188 ↛ 196line 188 didn't jump to line 196 because the condition on line 188 was always true

189 next_idx = diff.find("\ndiff --git ", idx)

190 if next_idx == -1:

191 parts.append(diff[idx:])

192 break

193 parts.append(diff[idx : next_idx + 1])

194 idx = next_idx + 1

195

196 for part in parts:

197 cur_path = None

198

199 p_idx = 0

200 while p_idx < len(part):

201 next_nl = part.find("\n", p_idx)

202 line = part[p_idx:] if next_nl == -1 else part[p_idx : next_nl + 1]

203

204 if line.startswith("diff --git "):

205 cur_path = _path_from_git_header(line)

206 elif line.startswith("+++ ") or (line.startswith("--- ") and not cur_path):

207 p = _path_from_marker(line)

208 if p is not None: 208 ↛ 215line 208 didn't jump to line 215 because the condition on line 208 was always true

209 cur_path = p

210 elif line.startswith(("rename to ", "copy to ")):

211 p = _strip_ab(_unquote_git_path(line.split(" to ", 1)[1].rstrip("\r\n")))

212 if p: 212 ↛ 215line 212 didn't jump to line 215 because the condition on line 212 was always true

213 cur_path = p

214

215 if cur_path is None:

216 cur_path = ""

217

218 if line.startswith("@@ "):

219 break

220

221 if next_nl == -1:

222 break

223 p_idx = next_nl + 1

224

225 files.append(DiffFile(path=cur_path or "", text=part))

226

227 return files

228

229

230# Anchored at the TRUE start of a line — no leading whitespace (#739). Git writes

231# both markers at column 0; every line inside a hunk carries a one-character

232# prefix (``+``, ``-``, or a space for context), so column 0 is unreachable from

233# a file's content and the anchor is what separates a marker from a mention.

234_BINARY_RE = re.compile(r"(?m)^(?:GIT binary patch|Binary files .* differ)\s*$")

235

236

237def _is_binary(text: str) -> bool:

238 """True when a file segment is a git *binary* diff.

239

240 Matches the binary marker on its own header line — ``Binary files … differ``

241 or a standalone ``GIT binary patch`` — rather than the substring anywhere in

242 the text. A diff's content lines are prefixed with ``+``/``-``/`` ``, so this

243 never misfires on source code that merely *mentions* those strings (e.g. this

244 module's own detector).

245

246 The prefix is the whole of the guarantee, so the pattern must not skip it.

247 It used to allow leading whitespace, which ate the single space in front of a

248 **context** line: an unchanged line reading ``Binary files a/x b/x differ``

249 inside a hunk made the entire text file read as binary and silently dropped

250 it from the review (#739). An *added* (``+``) or *removed* (``-``) line is

251 content for the same reason and is likewise not a marker — the ballot should

252 review the change that wrote that line, not skip the file because of it.

253 """

254 # bolt: avoid allocating a huge list of strings from splitlines()

255 # and generator overhead by using C-optimized regex finding.

256 return bool(_BINARY_RE.search(text))

257

258

259def _matches_any(path: str, patterns) -> bool:

260 """True when ``path`` matches any glob in ``patterns``.

261

262 Supports a trailing ``/**`` to mean "anything under this directory" and the

263 basename for simple ``*.ext`` patterns, in addition to a full-path match.

264 """

265 name = path.rsplit("/", 1)[-1]

266 for pat in patterns:

267 if pat.endswith("/**"):

268 prefix = pat[:-2] # keep trailing slash

269 if path.startswith(prefix):

270 return True

271 if fnmatch.fnmatch(path, pat) or fnmatch.fnmatch(name, pat):

272 return True

273 return False

274

275

276def _split_file_at_hunk_boundaries(f: DiffFile, chunk_max_bytes: int) -> list[str]:

277 """Split a single large diff file across semantic hunk headers (issue #522).

278

279 Preserves the file header preamble on subsequent chunks so context is not lost.

280 """

281 text = f.text

282 hunk_pattern = re.compile(r"(?m)^(@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@.*)$")

283 parts = hunk_pattern.split(text)

284 if len(parts) <= 1:

285 return [text]

286

287 header = parts[0]

288 chunks: list[str] = []

289 current: list[str] = [header]

290 current_bytes = len(header.encode("utf-8"))

291

292 for i in range(1, len(parts), 2):

293 hunk = parts[i] + (parts[i + 1] if i + 1 < len(parts) else "")

294 hb = len(hunk.encode("utf-8"))

295 if current_bytes + hb > chunk_max_bytes and len(current) > 1:

296 chunks.append("".join(current))

297 current = [header, hunk]

298 current_bytes = len(header.encode("utf-8")) + hb

299 else:

300 current.append(hunk)

301 current_bytes += hb

302

303 if current: 303 ↛ 305line 303 didn't jump to line 305 because the condition on line 303 was always true

304 chunks.append("".join(current))

305 return chunks

306

307

308def _chunk_files(kept: list[DiffFile], chunk_max_bytes: int) -> list[str]:

309 """Greedily pack kept files into chunks no larger than the budget.

310

311 Files keep their order. Large files exceeding budget are semantically split

312 at hunk boundaries with file header preservation (issue #522).

313 """

314 chunks: list[str] = []

315 current: list[str] = []

316 current_bytes = 0

317 for f in kept:

318 fb = f.size_bytes

319 if fb > chunk_max_bytes:

320 if current:

321 chunks.append("".join(current))

322 current, current_bytes = [], 0

323 chunks.extend(_split_file_at_hunk_boundaries(f, chunk_max_bytes))

324 continue

325 if current and current_bytes + fb > chunk_max_bytes:

326 chunks.append("".join(current))

327 current, current_bytes = [], 0

328 current.append(f.text)

329 current_bytes += fb

330 if current:

331 chunks.append("".join(current))

332 return chunks

333

334

335def plan_diff(

336 diff: str,

337 *,

338 max_bytes: int,

339 chunk: bool,

340 chunk_max_bytes: int | None = None,

341 exclude_generated: bool = True,

342 exclude: tuple[str, ...] | list[str] = (),

343 include: tuple[str, ...] | list[str] = (),

344) -> DiffPlan:

345 """Measure, filter, and decide a handling mode for ``diff`` (issue #31)."""

346 files = split_diff(diff)

347 total_bytes = len(diff.encode("utf-8"))

348 chunk_max_bytes = chunk_max_bytes or max_bytes

349

350 kept: list[DiffFile] = []

351 excluded: list[tuple[str, str]] = []

352 generated_globs = tuple(DEFAULT_GENERATED_GLOBS) + tuple(exclude)

353

354 for f in files:

355 # An include allow-list, when present, drops anything not matching.

356 if include and not _matches_any(f.path, include):

357 excluded.append((f.path, EXCLUDE_NOT_INCLUDED))

358 continue

359 if _is_binary(f.text):

360 excluded.append((f.path, EXCLUDE_BINARY))

361 continue

362 if exclude_generated and _matches_any(f.path, generated_globs):

363 excluded.append((f.path, EXCLUDE_GENERATED))

364 continue

365 if exclude and _matches_any(f.path, exclude):

366 excluded.append((f.path, EXCLUDE_FILTER))

367 continue

368 kept.append(f)

369

370 filtered_diff = "".join(f.text for f in kept)

371 kept_bytes = len(filtered_diff.encode("utf-8"))

372

373 if kept_bytes <= max_bytes:

374 mode = MODE_FULL

375 chunks = [filtered_diff] if kept_bytes else []

376 reason = (

377 f"{kept_bytes} B within budget ({max_bytes} B); reviewing in one pass"

378 if kept_bytes

379 else "nothing left to review after filters"

380 )

381 elif chunk:

382 mode = MODE_CHUNKED

383 chunks = _chunk_files(kept, chunk_max_bytes)

384 reason = (

385 f"{kept_bytes} B over budget ({max_bytes} B); chunked into "

386 f"{len(chunks)} part(s) of <= {chunk_max_bytes} B"

387 )

388 else:

389 mode = MODE_TOO_LARGE

390 chunks = []

391 reason = (

392 f"{kept_bytes} B over budget ({max_bytes} B) and chunking is disabled; "

393 f"enable [jury.diff] chunk = true or narrow the diff with "

394 f"include/exclude filters"

395 )

396

397 return DiffPlan(

398 mode=mode,

399 chunks=chunks,

400 kept=kept,

401 excluded=excluded,

402 total_bytes=total_bytes,

403 kept_bytes=kept_bytes,

404 reason=reason,

405 )

406

407

408# --- What the change actually contains (issue #710) -------------------------

409#

410# A ballot's ``Checked:`` line is free text an agent wrote. `Checked: nothing`

411# has the *shape* of an anchor — one backticked token — without naming anything,

412# and a rule that accepts the shape can be satisfied by an agent that read

413# nothing, which is the failure #700 exists to remove. So the token is resolved

414# against the change the panel was actually shown, and the index below is what

415# it is resolved against: the paths in the diff, and the symbols in it.

416#

417# It lives here, with the rest of "what this diff contains", so that

418# :mod:`ai_jury.ballots` holds the *rule* and not a second diff parser.

419

420#: Word-shaped tokens in the diff. Deliberately not "every word": a symbol is

421#: what a reader can go and look up, and an English word that happens to appear

422#: in a comment is not one. See :func:`_is_symbol_shaped`.

423_WORD_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")

424

425#: A token immediately followed by ``(`` — a call or a definition. This is what

426#: lets a one-word, all-lowercase symbol (``run``, ``main``) resolve while the

427#: one-word, all-lowercase *non*-symbol (``nothing``, ``everything``) does not.

428#: *Immediately*: ``nothing(`` is code, ``nothing (just wording)`` is prose, and

429#: the pattern used to allow whitespace between the two and so indexed the prose

430#: (#711 round 6).

431_CALLABLE_RE = re.compile(r"([A-Za-z_][A-Za-z0-9_]*)\(")

432

433#: Interior case change: ``ChangeIndex``, ``buildArgv``. With ``_`` this is the

434#: whole of "looks like an identifier rather than a word".

435_CAMEL_RE = re.compile(r"[a-z0-9][A-Z]")

436

437#: Caps. The index rides along on the outcome and into the result cache, so a

438#: hostile 5 MB diff must not turn into a 5 MB index. Truncation can only make a

439#: token fail to resolve, never make one resolve that should not: an unresolved

440#: token is reported as unresolved, which is the safe direction.

441MAX_INDEXED_PATHS = 1000

442MAX_INDEXED_SYMBOLS = 5000

443

444

445def _is_symbol_shaped(token: str) -> bool:

446 """Does *token* look like an identifier rather than an English word? (pure)"""

447 return len(token) > 1 and ("_" in token or bool(_CAMEL_RE.search(token)))

448

449

450@dataclass(frozen=True)

451class ChangeIndex:

452 """The paths and symbols present in the change under review (pure).

453

454 Built once per run from the (already redacted) diff and carried on the

455 outcome, so every renderer resolves a reviewer's stated scope against the

456 same bytes the panel was shown. ``None`` in place of one of these — a

457 hand-built outcome, a library caller that never had a diff — means "not

458 verifiable here", never "nothing exists".

459

460 Both tuples are sorted, so an outcome serialized into the result cache and

461 read back is byte-identical to the one that produced it.

462 """

463

464 paths: tuple[str, ...] = ()

465 symbols: tuple[str, ...] = ()

466

467 def has_path(self, path: str) -> bool:

468 """Is *path* one of the changed files? (pure)

469

470 Matched on a path-component boundary, in **both** directions: a reviewer

471 that writes ``ballots.py`` or ``ai_jury/ballots.py`` for the diff's

472 ``src/ai_jury/ballots.py`` named the file, and so does one that writes

473 ``src/a.py`` for a diff generated from a subdirectory that spells it

474 ``a.py``. Which root a path is quoted against is a property of how the

475 diff was produced, not of whether the reviewer read the file, and

476 abstaining over it would discard real reviews. ``lots.py`` matches

477 nothing: the boundary is what keeps a suffix from being a substring.

478 """

479 # ``./ballots.py`` and ``src/./ai_jury/ballots.py`` name the same file as

480 # ``ballots.py``: a current-directory segment is spelling, not a place

481 # (#711 round 10), so it is dropped before the boundary match.

482 candidate = "/".join(

483 segment for segment in (path or "").strip().split("/") if segment not in ("", ".")

484 )

485 if not candidate:

486 return False

487 for known in self.paths:

488 if (

489 known == candidate

490 or known.endswith("/" + candidate)

491 or candidate.endswith("/" + known)

492 ):

493 return True

494 return False

495

496 def has_symbol(self, token: str) -> bool:

497 """Is *token* a symbol that appears in the change? (pure)"""

498 return bool(token) and token in self.symbols

499

500

501def change_index(diff: str) -> ChangeIndex:

502 """Index the paths and symbols in *diff* (pure).

503

504 Paths come from :func:`split_diff`, so the one diff parser in this project

505 answers "which files" here too. Symbols are the identifier-shaped tokens in

506 the diff text — the whole diff, context lines included, because a reviewer

507 that names a symbol it read in the context around a hunk read it in this

508 change. A token qualifies when it carries an ``_`` or an interior case

509 change, or when it is called somewhere in the diff; a bare lowercase word is

510 not a symbol, which is exactly why ``Checked: nothing`` resolves to nothing

511 even in a diff whose prose contains the word.

512 """

513 text = diff or ""

514 paths = sorted({f.path for f in split_diff(text) if f.path})[:MAX_INDEXED_PATHS]

515 callables = set(_CALLABLE_RE.findall(text))

516 symbols = sorted(

517 {

518 token

519 for token in _WORD_RE.findall(text)

520 if _is_symbol_shaped(token) or token in callables

521 }

522 )[:MAX_INDEXED_SYMBOLS]

523 return ChangeIndex(paths=tuple(paths), symbols=tuple(symbols))

524

525

526def merge_change_indexes(indexes) -> ChangeIndex | None:

527 """One index over several chunks of the same review (pure).

528

529 A chunked review runs one jury per chunk and merges the outcomes; the scope

530 rule has to see the whole change, or a reviewer that named a file from

531 another chunk would be told it does not exist.

532 """

533 present = [i for i in indexes if i is not None]

534 if not present:

535 return None

536 paths: set[str] = set()

537 symbols: set[str] = set()

538 for index in present:

539 paths.update(index.paths)

540 symbols.update(index.symbols)

541 return ChangeIndex(

542 paths=tuple(sorted(paths)[:MAX_INDEXED_PATHS]),

543 symbols=tuple(sorted(symbols)[:MAX_INDEXED_SYMBOLS]),

544 )