Coverage for src/ai_jury/redaction.py: 97%
63 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-09-30 06:29 +0000
1"""Secret redaction for prompt text sent to external agents (issue #6).
3Deterministic: the same input always yields the same redacted output and count.
4Each match is replaced with ``[REDACTED:<kind>]``.
5"""
7from __future__ import annotations
9import re
10from urllib.parse import urlsplit, urlunsplit
12# An already-emitted redaction marker (`[REDACTED:<kind>]`). Used to avoid
13# re-redacting a value an earlier, more-specific pattern already replaced
14# (issue v1.5.0/L-3): without this guard a secret inside a basic-auth URL —
15# `https://user:AKIA…@host` — is redacted by `aws_access_key`, then the
16# resulting `[REDACTED:aws_access_key]` token is re-redacted by `basic_auth`,
17# losing the informative kind and double-counting.
18_REDACTED_MARKER = re.compile(r"^\[REDACTED:[a-z_]+\]$")
20# Ordered list of (kind, compiled pattern). Order matters: more specific
21# patterns run before the generic key=value catch-all so secrets are labeled
22# with the most informative kind.
23_PATTERNS: list[tuple[str, re.Pattern]] = [
24 (
25 "pem_private_key",
26 re.compile(
27 r"-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----"
28 r".*?-----END (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY-----",
29 re.DOTALL,
30 ),
31 ),
32 ("aws_access_key", re.compile(r"AKIA[0-9A-Z]{16}")),
33 ("github_token", re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}")),
34 # Classic `sk-…` AND modern project/service keys `sk-proj-…` /
35 # `sk-svcacct-…` / `sk-admin-…`, which embed hyphens the old `[A-Za-z0-9]`
36 # class stopped at (issue #122). First char after `sk-` is alphanumeric, then
37 # 18+ of alphanumeric / hyphen / underscore.
38 ("openai_key", re.compile(r"sk-[A-Za-z0-9][A-Za-z0-9_-]{18,}")),
39 ("bearer_token", re.compile(r"Bearer\s+[A-Za-z0-9._\-]+")),
40 # Common provider token formats (issue #290). Each runs BEFORE the generic
41 # `secret_assignment` catch-all so the value is replaced with its most
42 # informative kind and cannot be double-counted by the assignment pattern.
43 # Stripe keys use an underscore form (`sk_live_…`) distinct from the OpenAI
44 # `sk-…` hyphen pattern above; GitHub fine-grained PATs (`github_pat_…`) are
45 # not covered by the `gh[pousr]_` class.
46 ("slack_token", re.compile(r"xox[baprs]-[0-9A-Za-z-]{10,}")),
47 ("google_api_key", re.compile(r"AIza[0-9A-Za-z_\-]{35}")),
48 ("stripe_key", re.compile(r"(?:sk|rk|pk)_(?:live|test)_[0-9A-Za-z]{16,}")),
49 ("github_pat", re.compile(r"github_pat_[0-9A-Za-z_]{20,}")),
50 ("jwt", re.compile(r"eyJ[0-9A-Za-z_\-]+\.[0-9A-Za-z_\-]+\.[0-9A-Za-z_\-]+")),
51 # More provider formats (issue #316/L-2), each anchored on a distinctive
52 # prefix to avoid false positives. (Twilio Account SIDs and bare hex are
53 # deliberately NOT added — a SID is an identifier, and a broad hex rule would
54 # over-match commit SHAs.)
55 ("sendgrid_key", re.compile(r"SG\.[A-Za-z0-9_\-]{22}\.[A-Za-z0-9_\-]{43}")),
56 ("pypi_token", re.compile(r"pypi-[A-Za-z0-9_\-]{16,}")),
57 ("npm_token", re.compile(r"npm_[A-Za-z0-9]{36}")),
58 ("slack_webhook", re.compile(r"https://hooks\.slack\.com/services/[A-Za-z0-9/]+")),
59 # Basic-auth credentials in a URL (issue #302): `scheme://user:password@host`
60 # (e.g. `redis://default:s3cr3t…@cache:6379`). Only the password (group 2) is
61 # redacted; the `://user:` prefix (group 1) and `@` suffix (group 3) are kept
62 # so the URL stays readable. The trailing `@` requirement means a normal
63 # `host:port` (no `@`) never matches. Handled group-wise in `redact`.
64 #
65 # The username run is `*`, not `+` (review of #302): a token-as-password URL
66 # with an EMPTY username — `https://:SECRET@host` — is common, and `+` made
67 # the match fail there and leak the secret.
68 #
69 # The password run is `{1,}`, not `{6,}` (issue v1.5.0/L-1): a short password
70 # (`http://user:pass@host`, 4 chars) is still a leaked credential.
71 ("basic_auth", re.compile(r"(://[^/:@\s]*:)([^@\s]+)(@)")),
72 # Colon-less userinfo — a bare token in the userinfo position with no
73 # password colon (`http://apitoken12345@host/v1`) (issue v1.5.0/L-1). The
74 # colon form above never matches this (it requires a `:`), so the token would
75 # otherwise reach an external agent in cleartext. Group 1 (`://`) and group 3
76 # (`@`) are kept; group 2 (the token) is redacted. Disjoint from the colon
77 # form: `[^/:@\s]+` stops at a `:`, so a `user:pass@` URL is handled above.
78 #
79 # Note: this also redacts a bare, non-secret username (`https://user@host` ->
80 # `https://[REDACTED:basic_auth]@host`). That is deliberate safe-by-default
81 # over-redaction — userinfo in a reviewed diff is rare and a credential more
82 # often than not, and over-masking a username never leaks anything.
83 ("basic_auth", re.compile(r"(://)([^/:@\s]+)(@)")),
84 # Capture the surrounding quotes (groups 3 and 4) so they are PRESERVED in
85 # the replacement (issue #102): redacting only the value keeps a quoted
86 # assignment a valid string literal instead of producing a broken,
87 # unterminated string that misleads reviewers into phantom syntax findings.
88 #
89 # The key side (group 1) allows surrounding identifier chars so a keyword
90 # embedded mid-name is still recognized (issue #289): `aws_secret_access_key`
91 # matches via `secret`, where the old anchored `(secret)` required the `=`
92 # to follow `secret` directly and so leaked the canonical AWS variable name.
93 # `password`/`passwd` and a few more key names are included for the same
94 # reason. The surrounding identifier runs are BOUNDED (`{0,40}`), not `*`:
95 # a real secret variable name is short, and an unbounded run on both sides of
96 # the keyword makes the scan quadratic on a long word-char input with no
97 # separator (a ReDoS vector). The bound keeps it linear.
98 #
99 # `account[_-]?key` is included for Azure `AccountKey=…` / connection strings
100 # (issue #302). The separator group tolerates an optional closing quote
101 # (`["']?`) BEFORE the `=`/`:`, so a JSON-quoted key — `"private_key_id": "…"`
102 # in a GCP service-account blob — is matched too (its value is redacted while
103 # the structure stays valid). A non-secret like `client_email` is not caught:
104 # its value contains `@`/`.` outside the value char class.
105 (
106 "secret_assignment",
107 re.compile(
108 r"([A-Za-z0-9_]{0,40}(?:api[_-]?key|secret|token|password|passwd|"
109 r"access[_-]?key|account[_-]?key|private[_-]?key|client[_-]?secret|"
110 r"credential)"
111 r"[A-Za-z0-9_]{0,40})"
112 r"([\"']?\s*[=:]\s*)([\"']?)[A-Za-z0-9_\-+/=]{16,}([\"']?)",
113 re.IGNORECASE,
114 ),
115 ),
116]
119def redact(text: str) -> tuple[str, int]:
120 """Replace recognized secrets with ``[REDACTED:<kind>]``.
122 Returns ``(redacted_text, count)`` where count is the number of replacements.
123 """
124 if not text:
125 return text, 0
126 count = 0
127 result = text
128 for kind, pattern in _PATTERNS:
129 if kind == "secret_assignment":
131 def _sub_assign(m, _kind=kind):
132 nonlocal count
133 count += 1
134 # Preserve the key, separator, AND surrounding quotes; redact
135 # only the value so a quoted assignment stays syntactically valid.
136 return f"{m.group(1)}{m.group(2)}{m.group(3)}[REDACTED:{_kind}]{m.group(4)}"
138 result = pattern.sub(_sub_assign, result)
139 elif kind == "basic_auth":
141 def _sub_basic_auth(m, _kind=kind):
142 nonlocal count
143 # Don't re-redact a value an earlier pattern already replaced
144 # (issue v1.5.0/L-3): keep the more-informative kind and the
145 # accurate count.
146 if _REDACTED_MARKER.match(m.group(2)):
147 return m.group(0)
148 count += 1
149 # Keep the prefix (group 1: `://user:` or `://`) and `@` suffix
150 # (group 3); redact only the credential so the URL stays readable.
151 return f"{m.group(1)}[REDACTED:{_kind}]{m.group(3)}"
153 result = pattern.sub(_sub_basic_auth, result)
154 else:
156 def _sub(_m, _kind=kind):
157 nonlocal count
158 count += 1
159 return f"[REDACTED:{_kind}]"
161 result = pattern.sub(_sub, result)
162 return result, count
165# The complete alphabet a POSIX-shell environment variable name may use. Kept as
166# a module constant on purpose: :func:`safe_env_var_name` rebuilds its result out
167# of THESE characters rather than slicing the caller's string, so no byte of the
168# input can survive into the output. See that function for why that matters.
169_ENV_NAME_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_"
171# A real env var name is short; refuse anything longer rather than echoing it.
172_MAX_ENV_NAME_LEN = 128
174#: Human-readable statement of what :func:`safe_env_var_name` accepts. Lives here
175#: so a diagnostic can describe the rule without quoting the value that broke it.
176ENV_VAR_NAME_RULE = f"[A-Za-z_][A-Za-z0-9_]*, at most {_MAX_ENV_NAME_LEN} characters"
179def safe_env_var_name(raw, fallback: str) -> str:
180 """A syntactically valid environment-variable NAME, safe to display.
182 Diagnostics tell an operator *which variable to set* — "OPENAI_API_KEY is not
183 set in the environment" — so a config-supplied name is echoed into the
184 ``--doctor`` report, the machine-readable export, and warning text. Two
185 reasons that name must be rebuilt rather than passed through:
187 1. **Output integrity.** ``[[agent]] api_key_env`` is an arbitrary operator
188 string. A value containing a newline, a quote, or an ANSI escape would be
189 spliced straight into a JSON document or a terminal report. Only
190 ``[A-Za-z_][A-Za-z0-9_]*`` can survive here, so the export cannot be
191 reshaped by a config value.
192 2. **Provenance.** The returned string is assembled from
193 :data:`_ENV_NAME_ALPHABET`, a constant — only the *indices* come from the
194 input. So this is a genuine barrier, not a rename: nothing derived from a
195 credential-shaped config field can reach a log or an export through it,
196 which is what static analysis (CodeQL ``py/clear-text-logging-sensitive-data``)
197 flags on this path and what it should flag.
199 Anything that is not a valid name — empty, over-long, leading digit, any
200 character outside the alphabet — yields *fallback* unchanged.
201 """
202 text = raw if isinstance(raw, str) else ""
203 if not text or len(text) > _MAX_ENV_NAME_LEN or text[0].isdigit():
204 return fallback
205 rebuilt: list[str] = []
206 for char in text:
207 position = _ENV_NAME_ALPHABET.find(char)
208 if position < 0:
209 return fallback
210 # Appending from the constant, NOT `char`: the input contributes only
211 # the position it matched. This is the whole point of the function.
212 rebuilt.append(_ENV_NAME_ALPHABET[position])
213 return "".join(rebuilt)
216def redact_url_userinfo(url: str) -> str:
217 """Strip any userinfo (``user[:password]``) from a URL before display.
219 Structural counterpart to :func:`redact` for the one place a base endpoint
220 URL can carry a credential (issue v1.5.0/L-1). Rather than relying on the
221 `basic_auth` regex — which can miss short or colon-less userinfo — this
222 parses the URL and replaces the whole ``userinfo@`` run with
223 ``[REDACTED]@``, keeping the scheme, host, port, path, and query verbatim
224 (so an IPv6 literal or non-default port is preserved exactly). A URL that
225 `urlsplit` cannot parse falls back to the regex-based :func:`redact`.
226 """
227 if not url:
228 return url
229 try:
230 parts = urlsplit(url)
231 except ValueError:
232 return redact(url)[0]
233 netloc = parts.netloc
234 if "@" in netloc:
235 # The host part has no unencoded '@'; split on the last one so a userinfo
236 # that contains a percent-encoded '@' is still handled correctly.
237 hostport = netloc[netloc.rfind("@") + 1 :]
238 return urlunsplit(parts._replace(netloc=f"[REDACTED]@{hostport}"))
239 # A scheme-less URL ("user:pass@host/p") leaves netloc empty — urlsplit dumps
240 # the authority into the scheme/path, so the userinfo would slip through
241 # unredacted (security audit 2026-06-13; continues v1.5.0/L-1). If the URL
242 # has no "://" authority but still carries an "@", re-parse with a synthetic
243 # "//" authority so netloc is populated, redact, then strip it back off.
244 if "@" in url and "://" not in url:
245 try:
246 reparsed = urlsplit("//" + url)
247 except ValueError:
248 return redact(url)[0]
249 if "@" in reparsed.netloc: 249 ↛ 253line 249 didn't jump to line 253 because the condition on line 249 was always true
250 hostport = reparsed.netloc[reparsed.netloc.rfind("@") + 1 :]
251 redacted = urlunsplit(reparsed._replace(netloc=f"[REDACTED]@{hostport}"))
252 return redacted[2:] if redacted.startswith("//") else redacted
253 return url