Coverage for src/ai_jury/largediff.py: 94%
198 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-17 21:31 +0000
1"""Large-diff handling: filtering and chunking (issue #31).
3The jury sends the diff to every agent, so a large or generated diff inflates
4cost, runtime, and prompt size. This module measures a diff, drops files that
5should not be reviewed (binary blobs, generated/vendored files, and anything the
6configured path filters exclude), and decides a handling mode:
8- ``full`` — kept diff fits the budget; review it in one pass.
9- ``chunked`` — kept diff is over budget and chunking is enabled; split it
10 into per-file chunks each within the chunk budget.
11- ``too_large`` — over budget and chunking is disabled; the caller should fail
12 with a clear message.
14Everything here is PURE and deterministic: parsing, classification, and chunk
15boundaries are a function of the diff text and config only, so the plan is
16reproducible and unit-testable.
17"""
19from __future__ import annotations
21import fnmatch
22import re
23from dataclasses import dataclass, field
25# Default "generated / not worth reviewing" path globs. Conservative and
26# language-agnostic; users extend or replace via ``[jury.diff] exclude``.
27DEFAULT_GENERATED_GLOBS: tuple[str, ...] = (
28 # Dependency lockfiles.
29 "*.lock",
30 "package-lock.json",
31 "yarn.lock",
32 "pnpm-lock.yaml",
33 "poetry.lock",
34 "Cargo.lock",
35 "composer.lock",
36 "Gemfile.lock",
37 "go.sum",
38 # Minified / map artifacts.
39 "*.min.js",
40 "*.min.css",
41 "*.map",
42 # Snapshots and common generated code.
43 "*.snap",
44 "*.pb.go",
45 "*_pb2.py",
46 "*_pb2_grpc.py",
47 # Vendored / build output directories.
48 "vendor/**",
49 "node_modules/**",
50 "dist/**",
51 "build/**",
52)
54EXCLUDE_BINARY = "binary"
55EXCLUDE_GENERATED = "generated"
56EXCLUDE_FILTER = "excluded-by-filter"
57EXCLUDE_NOT_INCLUDED = "not-in-include-filter"
59MODE_FULL = "full"
60MODE_CHUNKED = "chunked"
61MODE_TOO_LARGE = "too_large"
64@dataclass
65class DiffFile:
66 """One file's segment of a unified diff."""
68 path: str
69 text: str
71 @property
72 def size_bytes(self) -> int:
73 return len(self.text.encode("utf-8"))
76@dataclass
77class DiffPlan:
78 mode: str
79 chunks: list[str] = field(default_factory=list)
80 kept: list[DiffFile] = field(default_factory=list)
81 excluded: list[tuple[str, str]] = field(default_factory=list)
82 total_bytes: int = 0
83 kept_bytes: int = 0
84 reason: str = ""
86 @property
87 def kept_paths(self) -> list[str]:
88 return [f.path for f in self.kept]
91def _strip_ab(path: str) -> str:
92 for prefix in ("a/", "b/"):
93 if path.startswith(prefix):
94 return path[len(prefix) :]
95 return path
98def _unquote_git_path(path: str) -> str:
99 """Undo git's C-style quoting of paths with special chars (best-effort).
101 git wraps a path in double quotes and octal-escapes special/non-ASCII bytes
102 when ``core.quotepath`` is on. We decode it back so the full path is
103 recovered for glob filtering and classification.
104 """
105 if len(path) >= 2 and path.startswith('"') and path.endswith('"'):
106 inner = path[1:-1]
107 try:
108 return (
109 inner.encode("latin-1", "backslashreplace")
110 .decode("unicode_escape")
111 .encode("latin-1")
112 .decode("utf-8", "replace")
113 )
114 except (UnicodeDecodeError, UnicodeEncodeError):
115 return inner.replace('\\"', '"').replace("\\\\", "\\")
116 return path
119def _path_from_marker(line: str) -> str | None:
120 """Path from a ``+++ b/<p>`` or ``--- a/<p>`` line, or None for /dev/null.
122 These marker lines carry a single, unambiguous path even when it contains
123 spaces or quoted special chars — unlike the ``diff --git a/<p> b/<p>``
124 header, which a ``str.split()`` truncates at the first space, hiding or
125 mislabeling the file (security audit 2026-06-13/L-4,N-3).
126 """
127 rest = line[4:].rstrip("\r\n")
128 # Some diff formats append a tab + timestamp; the path ends at the tab.
129 if "\t" in rest: 129 ↛ 130line 129 didn't jump to line 130 because the condition on line 129 was never true
130 rest = rest.split("\t", 1)[0]
131 if rest == "/dev/null": 131 ↛ 132line 131 didn't jump to line 132 because the condition on line 131 was never true
132 return None
133 return _strip_ab(_unquote_git_path(rest))
136def _path_from_git_header(line: str) -> str:
137 """Best-effort new-side path from a ``diff --git a/<p> b/<p>`` header.
139 ``str.split()[3]`` truncates a space-containing name; split on the last
140 `` b/`` separator instead so the full b-side path is recovered, then unquote
141 git's C-quoting (audit 2026-06-13/L-4, r3 marker-less case).
142 """
143 rest = line[len("diff --git ") :].rstrip("\r\n")
144 # Non-rename headers are symmetric: ``a/<p> b/<p>`` with the SAME <p> on both
145 # sides. Recover <p> by halving, which is robust even when <p> itself
146 # contains `` b/`` (a mode-change-only segment has no +++/--- or rename
147 # marker to fall back on — audit 2026-06-13 r4/L). len(body) = 2*len(p)+3.
148 if rest.startswith("a/"):
149 body = rest[2:]
150 half = (len(body) - 3) // 2
151 if (
152 len(body) >= 3
153 and body[half : half + 3] == " b/"
154 and body[:half] == body[half + 3 :]
155 ):
156 return _unquote_git_path(body[:half])
157 idx = rest.rfind(" b/")
158 if idx != -1:
159 return _strip_ab(_unquote_git_path(rest[idx + 1 :]))
160 # Quoted b-side: git C-quotes special/spaced paths as `"a/<p>" "b/<p>"`, so
161 # the separator is `` "b/`` not `` b/`` (audit 2026-06-13 r5/L).
162 qidx = rest.rfind(' "b/')
163 if qidx != -1: 163 ↛ 165line 163 didn't jump to line 165 because the condition on line 163 was always true
164 return _strip_ab(_unquote_git_path(rest[qidx + 1 :]))
165 parts = line.split()
166 return _strip_ab(parts[3]) if len(parts) >= 4 else _strip_ab(parts[-1])
169def split_diff(diff: str) -> list[DiffFile]:
170 """Split a unified diff into per-file segments.
172 Segments start at ``diff --git a/<p> b/<p>`` headers (the git format the
173 adapters emit). Any preamble before the first header is attached to the first
174 file so no bytes are silently dropped. A diff with no ``diff --git`` header is
175 returned as a single unnamed segment (it cannot be chunked by file).
176 """
177 if not diff:
178 return []
180 files: list[DiffFile] = []
182 parts = []
183 # bolt: avoid allocating a huge list of strings from splitlines(keepends=True)
184 # by splitting chunks directly and only iterating their header lines.
185 idx = diff.find("diff --git ")
186 if idx == -1:
187 parts = [diff]
188 else:
189 if idx > 0: 189 ↛ 190line 189 didn't jump to line 190 because the condition on line 189 was never true
190 parts.append(diff[:idx])
192 while idx != -1: 192 ↛ 200line 192 didn't jump to line 200 because the condition on line 192 was always true
193 next_idx = diff.find("\ndiff --git ", idx)
194 if next_idx == -1:
195 parts.append(diff[idx:])
196 break
197 parts.append(diff[idx:next_idx+1])
198 idx = next_idx + 1
200 for part in parts:
201 cur_path = None
203 p_idx = 0
204 while p_idx < len(part):
205 next_nl = part.find("\n", p_idx)
206 line = part[p_idx:] if next_nl == -1 else part[p_idx : next_nl + 1]
208 if line.startswith("diff --git "):
209 cur_path = _path_from_git_header(line)
210 elif line.startswith("+++ ") or (line.startswith("--- ") and not cur_path):
211 p = _path_from_marker(line)
212 if p is not None: 212 ↛ 219line 212 didn't jump to line 219 because the condition on line 212 was always true
213 cur_path = p
214 elif line.startswith(("rename to ", "copy to ")):
215 p = _strip_ab(_unquote_git_path(line.split(" to ", 1)[1].rstrip("\r\n")))
216 if p: 216 ↛ 219line 216 didn't jump to line 219 because the condition on line 216 was always true
217 cur_path = p
219 if cur_path is None:
220 cur_path = ""
222 if line.startswith("@@ "):
223 break
225 if next_nl == -1: 225 ↛ 226line 225 didn't jump to line 226 because the condition on line 225 was never true
226 break
227 p_idx = next_nl + 1
229 files.append(DiffFile(path=cur_path or "", text=part))
231 return files
234_BINARY_RE = re.compile(r"(?m)^\s*(?:GIT binary patch|Binary files .* differ)\s*$")
237def _is_binary(text: str) -> bool:
238 """True when a file segment is a git *binary* diff.
240 Matches the binary marker on its own header line — ``Binary files … differ``
241 or a standalone ``GIT binary patch`` — rather than the substring anywhere in
242 the text. A diff's content lines are prefixed with ``+``/``-``/`` ``, so this
243 never misfires on source code that merely *mentions* those strings (e.g. this
244 module's own detector).
245 """
246 # bolt: avoid allocating a huge list of strings from splitlines()
247 # and generator overhead by using C-optimized regex finding.
248 return bool(_BINARY_RE.search(text))
251def _matches_any(path: str, patterns) -> bool:
252 """True when ``path`` matches any glob in ``patterns``.
254 Supports a trailing ``/**`` to mean "anything under this directory" and the
255 basename for simple ``*.ext`` patterns, in addition to a full-path match.
256 """
257 name = path.rsplit("/", 1)[-1]
258 for pat in patterns:
259 if pat.endswith("/**"):
260 prefix = pat[:-2] # keep trailing slash
261 if path.startswith(prefix):
262 return True
263 if fnmatch.fnmatch(path, pat) or fnmatch.fnmatch(name, pat):
264 return True
265 return False
268def _split_file_at_hunk_boundaries(f: DiffFile, chunk_max_bytes: int) -> list[str]:
269 """Split a single large diff file across semantic hunk headers (issue #522).
271 Preserves the file header preamble on subsequent chunks so context is not lost.
272 """
273 text = f.text
274 hunk_pattern = re.compile(r"(?m)^(@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@.*)$")
275 parts = hunk_pattern.split(text)
276 if len(parts) <= 1:
277 return [text]
279 header = parts[0]
280 chunks: list[str] = []
281 current: list[str] = [header]
282 current_bytes = len(header.encode("utf-8"))
284 for i in range(1, len(parts), 2):
285 hunk = parts[i] + (parts[i + 1] if i + 1 < len(parts) else "")
286 hb = len(hunk.encode("utf-8"))
287 if current_bytes + hb > chunk_max_bytes and len(current) > 1:
288 chunks.append("".join(current))
289 current = [header, hunk]
290 current_bytes = len(header.encode("utf-8")) + hb
291 else:
292 current.append(hunk)
293 current_bytes += hb
295 if current: 295 ↛ 297line 295 didn't jump to line 297 because the condition on line 295 was always true
296 chunks.append("".join(current))
297 return chunks
300def _chunk_files(kept: list[DiffFile], chunk_max_bytes: int) -> list[str]:
301 """Greedily pack kept files into chunks no larger than the budget.
303 Files keep their order. Large files exceeding budget are semantically split
304 at hunk boundaries with file header preservation (issue #522).
305 """
306 chunks: list[str] = []
307 current: list[str] = []
308 current_bytes = 0
309 for f in kept:
310 fb = f.size_bytes
311 if fb > chunk_max_bytes:
312 if current:
313 chunks.append("".join(current))
314 current, current_bytes = [], 0
315 chunks.extend(_split_file_at_hunk_boundaries(f, chunk_max_bytes))
316 continue
317 if current and current_bytes + fb > chunk_max_bytes:
318 chunks.append("".join(current))
319 current, current_bytes = [], 0
320 current.append(f.text)
321 current_bytes += fb
322 if current:
323 chunks.append("".join(current))
324 return chunks
327def plan_diff(
328 diff: str,
329 *,
330 max_bytes: int,
331 chunk: bool,
332 chunk_max_bytes: int | None = None,
333 exclude_generated: bool = True,
334 exclude: tuple[str, ...] | list[str] = (),
335 include: tuple[str, ...] | list[str] = (),
336) -> DiffPlan:
337 """Measure, filter, and decide a handling mode for ``diff`` (issue #31)."""
338 files = split_diff(diff)
339 total_bytes = len(diff.encode("utf-8"))
340 chunk_max_bytes = chunk_max_bytes or max_bytes
342 kept: list[DiffFile] = []
343 excluded: list[tuple[str, str]] = []
344 generated_globs = tuple(DEFAULT_GENERATED_GLOBS) + tuple(exclude)
346 for f in files:
347 # An include allow-list, when present, drops anything not matching.
348 if include and not _matches_any(f.path, include):
349 excluded.append((f.path, EXCLUDE_NOT_INCLUDED))
350 continue
351 if _is_binary(f.text):
352 excluded.append((f.path, EXCLUDE_BINARY))
353 continue
354 if exclude_generated and _matches_any(f.path, generated_globs):
355 excluded.append((f.path, EXCLUDE_GENERATED))
356 continue
357 if exclude and _matches_any(f.path, exclude):
358 excluded.append((f.path, EXCLUDE_FILTER))
359 continue
360 kept.append(f)
362 filtered_diff = "".join(f.text for f in kept)
363 kept_bytes = len(filtered_diff.encode("utf-8"))
365 if kept_bytes <= max_bytes:
366 mode = MODE_FULL
367 chunks = [filtered_diff] if kept_bytes else []
368 reason = (
369 f"{kept_bytes} B within budget ({max_bytes} B); reviewing in one pass"
370 if kept_bytes
371 else "nothing left to review after filters"
372 )
373 elif chunk:
374 mode = MODE_CHUNKED
375 chunks = _chunk_files(kept, chunk_max_bytes)
376 reason = (
377 f"{kept_bytes} B over budget ({max_bytes} B); chunked into "
378 f"{len(chunks)} part(s) of <= {chunk_max_bytes} B"
379 )
380 else:
381 mode = MODE_TOO_LARGE
382 chunks = []
383 reason = (
384 f"{kept_bytes} B over budget ({max_bytes} B) and chunking is disabled; "
385 f"enable [jury.diff] chunk = true or narrow the diff with "
386 f"include/exclude filters"
387 )
389 return DiffPlan(
390 mode=mode,
391 chunks=chunks,
392 kept=kept,
393 excluded=excluded,
394 total_bytes=total_bytes,
395 kept_bytes=kept_bytes,
396 reason=reason,
397 )