Coverage for src/ai_jury/ci.py: 100%

56 statements  

« prev     ^ index     » next       coverage.py v7.16.2, created at 2026-10-20 18:15 +0000

1"""Severity-gated CI exit policy (issue #4). 

2 

3A pure decision function over the consensus groups: given the configured blocking 

4severities and how to treat unverified findings, decide a process exit code. 

5""" 

6 

7from __future__ import annotations 

8 

9from .findings import SEVERITY_INPUTS, canonical_severity, flatten_inline 

10 

11 

12def resolve_fail_on(fail_on) -> tuple[set[str], list[str]]: 

13 """Split a configured fail-on list into canonical severities and unknowns. 

14 

15 Blank entries are dropped. Recognised entries come back canonicalised — so 

16 the documented ``blocker`` alias matches ``critical`` groups instead of 

17 silently never firing — and anything outside the vocabulary is returned, in 

18 the spelling the operator wrote, for a caller to refuse. 

19 """ 

20 known: set[str] = set() 

21 unknown: list[str] = [] 

22 for entry in fail_on or []: 

23 text = str(entry).strip() 

24 if not text: 

25 continue 

26 severity = canonical_severity(text) 

27 if severity is None: 

28 unknown.append(text) 

29 else: 

30 known.add(severity) 

31 return known, unknown 

32 

33 

34def _vocabulary_error(unknown: list[str], where: str) -> str: 

35 return ( 

36 f"{where} contains unknown severities {unknown!r} " 

37 f"(expected one of {', '.join(SEVERITY_INPUTS)})." 

38 ) 

39 

40 

41def fail_on_error(fail_on, where: str) -> str | None: 

42 """Message naming every unrecognised severity in ``fail_on``, else ``None``. 

43 

44 Shared by ``validate_config`` and the ``--fail-on`` flag so a typo is 

45 reported the same way whichever surface it was written on (issue #718). 

46 """ 

47 _, unknown = resolve_fail_on(fail_on) 

48 return _vocabulary_error(unknown, where) if unknown else None 

49 

50 

51def evaluate_ci( 

52 groups_with_status, 

53 fail_on, 

54 ignore_unverified: bool, 

55 pre_existing: str = "report", 

56) -> tuple[int, str]: 

57 """Decide a CI exit code from consensus groups. 

58 

59 Returns ``(exit_code, reason)``. 

60 

61 A group fails CI when its severity is in ``fail_on`` AND it is either 

62 verified (status == "verified") or, when ``ignore_unverified`` is False, has 

63 any non-"unsupported" status. Findings the verifier marked "unsupported" 

64 never fail CI. When ``ignore_unverified`` is True, groups that were never 

65 verified (empty status) do not fail CI; only explicitly verified ones can. 

66 When ``pre_existing`` is "ignore", groups whose origin is "pre_existing" 

67 do not fail CI and are counted in the gate reason (issue #976). Note that 

68 ``origin`` is a trust-the-model label from the reviewers (a group is 

69 "pre_existing" only when every member says so, which for a single-reviewer 

70 group is that one seat, while any "introduced" member makes the group 

71 "introduced"); keep ``pre_existing = "report"`` when the gate must be 

72 fail-closed against model-manipulated ``origin`` labels. 

73 

74 Raises ``ValueError`` when ``fail_on`` names a severity outside the 

75 vocabulary. A misspelling matches no group, so treating it as "blocks 

76 nothing" would report a green PASS quoting the typo and disable the gate 

77 forever; the CLI and ``validate_config`` refuse one first, and this guard 

78 keeps a caller that bypasses both from reopening the hole (issue #718). 

79 """ 

80 fail_set, unknown = resolve_fail_on(fail_on) 

81 if unknown: 

82 raise ValueError(_vocabulary_error(unknown, "fail_on")) 

83 blocking = [] 

84 ignored_pre_existing = 0 

85 for g in groups_with_status: 

86 severity = getattr(g, "severity", "") 

87 status = getattr(g, "status", "") or "" 

88 if severity not in fail_set: 

89 continue 

90 if status == "unsupported": 

91 continue 

92 if ignore_unverified and status != "verified": 

93 continue 

94 if pre_existing == "ignore" and getattr(g, "origin", "") == "pre_existing": 

95 ignored_pre_existing += 1 

96 continue 

97 blocking.append(g) 

98 

99 ignored_note = ( 

100 f"; {ignored_pre_existing} pre-existing finding(s) ignored" if ignored_pre_existing else "" 

101 ) 

102 if blocking: 

103 bits = [] 

104 for g in blocking: 

105 rep = getattr(g, "representative", None) 

106 loc = "" 

107 if rep is not None and getattr(rep, "file", None): 

108 loc = flatten_inline(rep.file) 

109 if getattr(rep, "line", None) is not None: 

110 loc += f":{rep.line}" 

111 # Flatten the attacker-influenced file/claim: this reason line is 

112 # posted to the PR as the CI-gate section, so a multi-line claim could 

113 # otherwise forge a heading/marker in the comment (audit 2026-06-13 

114 # r7/M). This stays a pure function. 

115 claim = flatten_inline(getattr(rep, "claim", "")) if rep is not None else "" 

116 bits.append(f"[{g.severity}] {loc or '(no location)'} {claim}".strip()) 

117 mode_suffix = ( 

118 f" (pre_existing=ignore{ignored_note})" if pre_existing == "ignore" else ignored_note 

119 ) 

120 reason = ( 

121 f"FAIL: {len(blocking)} blocking finding(s) at severities " 

122 f"{sorted(fail_set)}: " + "; ".join(bits) + mode_suffix 

123 ) 

124 return 1, reason 

125 

126 mode_opt = ", pre_existing=ignore" if pre_existing == "ignore" else "" 

127 reason = ( 

128 f"PASS: no blocking findings at severities {sorted(fail_set)} " 

129 f"(ignore_unverified={ignore_unverified}{mode_opt}{ignored_note})." 

130 ) 

131 return 0, reason