Coverage for src/ai_jury/doctor.py: 98%
162 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-17 21:31 +0000
1"""Local diagnostics for the agent review jury (``jury --doctor``).
3The ``--doctor`` command reports local readiness and common configuration
4problems. Its output is intentionally SAFE to share:
6- It includes tool/Python/OS versions, a redacted config summary, agent
7 availability (which agent CLIs are on PATH), each agent's detected CLI
8 version and capability summary, and detected config warnings.
9- It NEVER includes the raw diff under review or any agent output.
10- Secret-like values in the config summary are redacted via
11 :func:`ai_jury.redaction.redact`.
13This project collects and transmits NO telemetry. Diagnostics are built
14locally and only written where you explicitly ask (stdout, or ``--write``).
15"""
17from __future__ import annotations
19import platform
20import shutil
21import sys
22import tomllib
23from pathlib import Path
25from . import __version__
26from .adapters import make_adapter
27from .config import ConfigError, load_config
28from .redaction import redact, redact_url_userinfo
31def _redact_value(value):
32 """Redact a single config value if it looks secret-like.
34 ``redact`` operates on text and returns ``(text, count)``; non-string
35 values are returned unchanged.
36 """
37 if isinstance(value, str):
38 return redact(value)[0]
39 return value
42def _detect_capabilities(spec):
43 """Best-effort capability/version probe for one agent spec.
45 Uses the real adapter (NOT the mock) so doctor reports actual installed
46 versions, but guards against any failure: an unavailable CLI just reports
47 ``status="unavailable"`` and a crashing probe degrades to ``unknown_version``.
48 This must stay fast (short subprocess timeout) and never crash doctor.
49 """
50 try:
51 adapter = make_adapter(spec)
52 return adapter.detect_capabilities()
53 except Exception as exc: # noqa: BLE001 - diagnostics must never crash
54 return {
55 "version": None,
56 "supports_headless": None,
57 "supports_model_selection": None,
58 "raw_version_output": "",
59 "status": "unknown_version",
60 "warnings": [f"capability probe raised: {redact(str(exc))[0]}"],
61 }
64def _is_available(spec) -> bool:
65 """Whether an agent is reachable, via its adapter's own check.
67 Uses ``adapter.available()`` rather than ``shutil.which`` so a local/HTTP
68 agent (issue #43), which has no ``command`` and probes its endpoint instead,
69 is reported correctly. Guarded — any failure reads as unavailable.
70 """
71 try:
72 return make_adapter(spec).available()
73 except Exception: # noqa: BLE001 - diagnostics must never crash
74 return False
77def _resolved_command(spec):
78 """Absolute path a CLI agent's command resolves to on PATH (issue #296).
80 Lets an operator verify *which* binary will run (a poisoned PATH could
81 resolve a bare name to a shim). None for a local/HTTP agent (no command) or
82 when nothing is found on PATH.
83 """
84 command = getattr(spec, "command", "") or ""
85 vendor = (getattr(spec, "vendor", "") or "").lower()
86 has_endpoint = bool(getattr(spec, "endpoint", None))
87 if not command or vendor in ("local", "anthropic-api", "openai-api", "google-api", "openai-compatible") or vendor.endswith("-api") or has_endpoint:
88 return None
89 try:
90 return shutil.which(command)
91 except Exception: # noqa: BLE001 - diagnostics must never crash
92 return None
95def _agent_entry(spec):
96 caps = _detect_capabilities(spec)
97 return {
98 "name": _redact_value(spec.name),
99 "command": _redact_value(spec.command),
100 "resolved": _resolved_command(spec),
101 "vendor": _redact_value(spec.vendor),
102 "available": _is_available(spec),
103 "version": _redact_value(caps.get("version")),
104 "capabilities": {
105 "supports_headless": caps.get("supports_headless"),
106 "supports_model_selection": caps.get("supports_model_selection"),
107 "status": caps.get("status"),
108 },
109 "capability_warnings": [_redact_value(w) for w in caps.get("warnings", [])],
110 }
113def _config_summary(cfg):
114 """Build a redacted, secret-free summary of the loaded config."""
115 return {
116 "rounds": cfg.rounds,
117 "chair": _redact_value(cfg.chair),
118 "context_mode": _redact_value(cfg.context.mode),
119 "enabled_agents": [_redact_value(a.name) for a in cfg.enabled_agents],
120 }
123# Hosted-API vendors (issue #430/#432): no `command`/`endpoint`, so neither
124# the "local" nor the "CLI on PATH" branch below is the right diagnosis when
125# one is unavailable.
126_HOSTED_API_VENDORS = ("anthropic-api", "openai-api", "google-api")
129def _detect_warnings(cfg) -> list[str]:
130 """Best-effort config sanity checks reported to the user."""
131 warnings: list[str] = []
132 if not cfg.agents:
133 warnings.append("no agents are configured")
134 enabled = cfg.enabled_agents
135 if cfg.agents and not enabled:
136 warnings.append("all configured agents are disabled")
137 names = {a.name for a in cfg.agents}
138 if cfg.chair not in names:
139 warnings.append(f"chair '{_redact_value(cfg.chair)}' does not match any configured agent")
140 for agent in enabled:
141 if _is_available(agent):
142 continue
143 if agent.vendor == "local":
144 warnings.append(
145 f"agent '{_redact_value(agent.name)}' (local) endpoint "
146 f"'{redact_url_userinfo(agent.endpoint or 'http://localhost:11434/v1')}' "
147 f"is not reachable"
148 )
149 elif agent.vendor in _HOSTED_API_VENDORS:
150 # Reuse the adapter's own capability warning (issue #430) instead
151 # of re-deriving the vendor -> env-var mapping here, so the
152 # message can't drift from what the adapter actually reports.
153 caps = _detect_capabilities(agent)
154 reason = "; ".join(caps.get("warnings", [])) or "the hosted API is not reachable"
155 warnings.append(f"agent '{_redact_value(agent.name)}' (hosted API): {reason}")
156 else:
157 warnings.append(
158 f"agent '{_redact_value(agent.name)}' command "
159 f"'{_redact_value(agent.command)}' is not on PATH"
160 )
161 return warnings
164def _recommendations(config_path, config_summary, agents) -> dict:
165 """Build actionable next-steps from the diagnostics (issue: doctor UX).
167 Returns ``{"ready": bool, "steps": [str, ...]}``. ``ready`` is true when at
168 least one agent is reachable. Steps point the user at the cheapest fix:
169 scaffold a config, install a CLI, or use a reachable local model.
170 """
171 steps: list[str] = []
172 available = [a for a in agents if a.get("available")]
173 ready = bool(available)
175 # No config file in play -> suggest scaffolding one.
176 if config_path is None and not Path("jury.toml").exists():
177 steps.append("No jury.toml found — run `jury init` to create one.")
179 if not ready:
180 from .adapters import list_local_models
182 models = list_local_models()
183 if models:
184 steps.append(
185 f"No agent CLI is available, but a local model server is reachable "
186 f"({len(models)} model(s): {', '.join(models[:3])}). Add a free local "
187 f"reviewer: `jury init --preset offline` (or `--list-models`)."
188 )
189 else:
190 steps.append(
191 "No reviewer is available. Install an agent CLI (claude / codex / agy), "
192 "or run a local model (e.g. `ollama serve` + `ollama pull "
193 'qwen2.5-coder:7b`) and add a `vendor = "local"` agent — or use '
194 "`--mock` for an offline demo."
195 )
196 else:
197 missing = [
198 a["name"]
199 for a in agents
200 if not a.get("available")
201 and config_summary
202 and a["name"] in config_summary.get("enabled_agents", [])
203 ]
204 if missing:
205 steps.append(
206 f"Enabled but unavailable (will be skipped): {', '.join(missing)}. "
207 f"Install them or run with `--strict` to fail instead."
208 )
210 return {"ready": ready, "steps": steps}
213def build_diagnostics(config_path=None):
214 """Build a SAFE diagnostics dict for the given config path.
216 Best-effort: if the config cannot be loaded, the error is captured as a
217 string under ``config_warnings`` and ``config`` is left ``None``. Never
218 raises for a bad/missing config. The returned dict never contains the raw
219 diff or any agent output.
220 """
221 config_summary = None
222 config_warnings: list[str] = []
223 agents: list = []
225 try:
226 cfg = load_config(config_path)
227 except FileNotFoundError as exc:
228 config_warnings.append(f"config error: {redact(str(exc))[0]}")
229 except tomllib.TOMLDecodeError as exc:
230 config_warnings.append(f"config error: invalid TOML: {redact(str(exc))[0]}")
231 except ConfigError as exc:
232 config_warnings.append(f"config error: {redact(str(exc))[0]}")
233 except (KeyError, ValueError, TypeError) as exc:
234 config_warnings.append(f"config error: {redact(str(exc))[0]}")
235 else:
236 config_summary = _config_summary(cfg)
237 agents = [_agent_entry(spec) for spec in cfg.agents]
238 config_warnings = _detect_warnings(cfg)
239 # Fold capability/version probe warnings (e.g. an available CLI whose
240 # version could not be detected) into the user-facing warnings list.
241 # Probes already ran while building the agent entries above.
242 enabled_names = {a.name for a in cfg.enabled_agents}
243 for spec, entry in zip(cfg.agents, agents, strict=False):
244 if spec.name not in enabled_names:
245 continue
246 for warning in entry.get("capability_warnings", []):
247 config_warnings.append(f"agent '{entry['name']}': {warning}")
249 return {
250 "tool_version": __version__,
251 "python_version": platform.python_version(),
252 "python_implementation": platform.python_implementation(),
253 "python_executable": sys.executable,
254 "os": platform.platform(),
255 "config_path": str(config_path) if config_path else "(default)",
256 "agents": agents,
257 "config": config_summary,
258 "config_warnings": config_warnings,
259 "recommendations": _recommendations(config_path, config_summary, agents),
260 }
263def render_report(diagnostics) -> str:
264 """Render a human-readable text report from a diagnostics dict."""
265 lines = []
266 lines.append("jury doctor")
267 lines.append("=" * 40)
268 lines.append(f"tool version: {diagnostics['tool_version']}")
269 lines.append(
270 f"python: {diagnostics['python_version']} ({diagnostics['python_implementation']})"
271 )
272 lines.append(f"python exe: {diagnostics['python_executable']}")
273 lines.append(f"os: {diagnostics['os']}")
274 lines.append(f"config path: {diagnostics['config_path']}")
275 lines.append("")
277 lines.append("Agents")
278 lines.append("-" * 40)
279 agents = diagnostics["agents"]
280 if not agents:
281 lines.append(" (no agents loaded)")
282 else:
283 for agent in agents:
284 status = "available" if agent["available"] else "MISSING"
285 lines.append(
286 f" [{status:>9}] {agent['name']} "
287 f"(vendor={agent['vendor']}, command={agent['command']})"
288 )
289 if agent.get("command") and agent.get("vendor") != "local": 289 ↛ 292line 289 didn't jump to line 292 because the condition on line 289 was always true
290 resolved = agent.get("resolved") or "(not found on PATH)"
291 lines.append(f" resolved: {resolved}")
292 version = agent.get("version") or "unknown"
293 caps = agent.get("capabilities") or {}
294 cap_bits = []
295 if caps.get("supports_headless"):
296 cap_bits.append("headless")
297 if caps.get("supports_model_selection"):
298 cap_bits.append("model-selection")
299 cap_summary = ", ".join(cap_bits) or "none"
300 cap_status = caps.get("status") or "unknown"
301 lines.append(
302 f" version={version}, capabilities=[{cap_summary}] "
303 f"(probe: {cap_status})"
304 )
305 lines.append("")
307 lines.append("Config summary")
308 lines.append("-" * 40)
309 config = diagnostics["config"]
310 if config is None:
311 lines.append(" (config could not be loaded)")
312 else:
313 lines.append(f" rounds: {config['rounds']}")
314 lines.append(f" chair: {config['chair']}")
315 lines.append(f" context mode: {config['context_mode']}")
316 enabled = ", ".join(config["enabled_agents"]) or "(none)"
317 lines.append(f" enabled: {enabled}")
318 lines.append("")
320 lines.append("Warnings")
321 lines.append("-" * 40)
322 warnings = diagnostics["config_warnings"]
323 if not warnings:
324 lines.append(" (none)")
325 else:
326 for warning in warnings:
327 lines.append(f" - {warning}")
328 lines.append("")
330 rec = diagnostics.get("recommendations") or {}
331 lines.append("Next steps")
332 lines.append("-" * 40)
333 lines.append(f" ready to run: {'yes' if rec.get('ready') else 'no'}")
334 for step in rec.get("steps", []):
335 lines.append(f" - {step}")
336 lines.append("")
338 lines.append(
339 "Privacy: no telemetry is collected or sent. This report is "
340 "local-only and redacts secret-like values."
341 )
343 return "\n".join(lines)