"""reviewing-agent-diffs-checklist.py — flag the usual agent damage in a unified diff. What it does: reads a unified diff (git diff / git show output) from a file or stdin and reports, with file and line references, the patterns that a human reviewer of agent output should look at first: deleted-test a removed line that was a test function or test file skipped-test a new skip/xfail decorator or a test renamed so it no longer runs weakened-assert an assertion removed, or replaced by a bare truthiness check broad-except a new `except:` / `except Exception` / catch-all new-dependency a new import of a third-party-looking module, or a change to a requirements / package manifest ci-config a change under .github/, a Dockerfile, or a CI config file secret-like a new line that looks like a key or token assignment leftover-marker a new fixme / to-do style comment big-deletion a file that loses more than 40 lines net out-of-scope a changed file outside the --scope paths, when --scope is given Run: git diff main...HEAD | python reviewing-agent-diffs-checklist.py --scope src/ tests/ python reviewing-agent-diffs-checklist.py change.diff Exit: 0 when nothing is flagged, 1 when anything is flagged. Findings are pointers, not verdicts: read the lines it names. Python 3.13, standard library only. """ from __future__ import annotations import argparse import re import sys STDLIB_HINT = {"os", "re", "sys", "json", "csv", "math", "time", "datetime", "pathlib", "typing", "subprocess", "unittest", "argparse", "collections", "itertools", "functools", "logging", "io", "shutil", "tempfile", "textwrap", "hashlib", "random", "string", "dataclasses", "enum", "abc", "glob"} MANIFESTS = ("requirements", "pyproject.toml", "package.json", "package-lock.json", "go.mod", "Cargo.toml", "Gemfile", "poetry.lock", "uv.lock", "setup.py", "setup.cfg") CI_HINTS = (".github/", ".gitlab-ci", "Dockerfile", "docker-compose", ".circleci", "Jenkinsfile", "azure-pipelines", ".pre-commit-config", "web.config") SECRET = re.compile(r"(api[_-]?key|secret|token|password|passwd|private[_-]?key)\s*[:=]\s*['\"][A-Za-z0-9_\-/+=]{12,}", re.I) MARKER = re.compile(r"#\s*(to-?do|fixme|hack|xxx)\b|//\s*(to-?do|fixme|hack)\b", re.I) SKIP = re.compile(r"@(unittest\.)?skip|@pytest\.mark\.(skip|xfail)|\.skip\(|xit\(|it\.skip|test\.skip|describe\.skip") ASSERT = re.compile(r"\bassert\b|self\.assert\w*\(|expect\(|assert_eq!|t\.Errorf|t\.Fatal") BROAD = re.compile(r"except\s*:|except\s+(BaseException|Exception)\s*:|catch\s*\(\s*(\w+\s*)?\)\s*\{\s*\}|catch\s*\{\s*\}") TEST_DEF = re.compile(r"^\s*(def test_\w+|async def test_\w+|it\(|test\(|func Test\w+|#\[test\])") def parse(diff: str): """Yield (path, old_line_no, new_line_no, kind, text) for every changed line.""" path = None old = new = 0 for raw in diff.splitlines(): if raw.startswith("+++ "): path = raw[4:].strip() path = path[2:] if path.startswith("b/") else path continue if raw.startswith("--- "): continue m = re.match(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@", raw) if m: old, new = int(m.group(1)), int(m.group(2)) continue if path is None: continue if raw.startswith("+"): yield path, None, new, "+", raw[1:] new += 1 elif raw.startswith("-"): yield path, old, None, "-", raw[1:] old += 1 elif raw.startswith(" ") or raw == "": old += 1 new += 1 def is_test_file(path: str) -> bool: name = path.split("/")[-1] return "test" in name.lower() or "/tests/" in path or "/test/" in path def scan(diff: str, scope: list[str]) -> list[tuple[str, str, int | None, str]]: findings = [] net: dict[str, int] = {} files: set[str] = set() for path, o, n, kind, text in parse(diff): files.add(path) net[path] = net.get(path, 0) + (1 if kind == "+" else -1) line = n if kind == "+" else o stripped = text.strip() if kind == "-": if TEST_DEF.match(text): findings.append(("deleted-test", path, line, stripped)) elif ASSERT.search(text): findings.append(("weakened-assert", path, line, stripped)) continue if SKIP.search(text): findings.append(("skipped-test", path, line, stripped)) if BROAD.search(text): findings.append(("broad-except", path, line, stripped)) if re.match(r"^\s*assert\s+(True|1)\b", text) or re.search(r"assertTrue\(\s*(True|1)\s*\)", text): findings.append(("weakened-assert", path, line, stripped)) imp = re.match(r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))", text) if imp and path.endswith(".py"): mod = (imp.group(1) or imp.group(2)).split(".")[0] if mod not in STDLIB_HINT and not mod.startswith("_"): findings.append(("new-dependency", path, line, stripped)) if SECRET.search(text): findings.append(("secret-like", path, line, stripped[:60] + "...")) if MARKER.search(text): findings.append(("leftover-marker", path, line, stripped)) for path in sorted(files): if any(k in path for k in MANIFESTS): findings.append(("new-dependency", path, None, "manifest changed")) if any(k in path for k in CI_HINTS): findings.append(("ci-config", path, None, "CI or build configuration changed")) if net[path] < -40: findings.append(("big-deletion", path, None, f"net {net[path]} lines")) if scope and not any(path.startswith(s) for s in scope): findings.append(("out-of-scope", path, None, f"not under {', '.join(scope)}")) return findings def main(argv: list[str]) -> int: ap = argparse.ArgumentParser(description="flag common agent damage in a unified diff") ap.add_argument("diff", nargs="?", help="diff file (default: stdin)") ap.add_argument("--scope", nargs="*", default=[], help="path prefixes the task was allowed to touch") a = ap.parse_args(argv[1:]) diff = open(a.diff, encoding="utf-8").read() if a.diff else sys.stdin.read() findings = scan(diff, a.scope) for kind, path, line, text in findings: where = f"{path}:{line}" if line else path print(f"{kind:16} {where:34} {text}") kinds = sorted({k for k, *_ in findings}) print(f"{len(findings)} finding(s) in {len(kinds)} categor{'y' if len(kinds) == 1 else 'ies'}: {', '.join(kinds) or 'none'}") return 1 if findings else 0 if __name__ == "__main__": sys.exit(main(sys.argv))