"""reviewing-agent-diffs-swe-bench-scan.py - run the ten-pattern diff scanner from reviewing-agent-diffs-checklist.py
over real patches: the 500 SWE-bench Verified fixes written by each project's maintainers, their 500 test patches,
and the 5,500 patches eleven models submitted through mini-swe-agent 2.0.0 for the same tasks.

Two steps, so every number can be recomputed offline from a saved copy:

  python reviewing-agent-diffs-swe-bench-scan.py fetch --cache swe-bench-patches
      Reads robots.txt for each host first and stops if a rule disallows ClaudeBot, Claude-User, anthropic-ai or *
      (a 4xx answer means no rules, as RFC 9309 says; a 5xx or network error stops the run). Then saves:
        - the 500 Verified tasks (gold patch, test patch, repo) from the Hugging Face datasets-server rows API;
        - each run's per-task record (per_instance_details.json) from SWE-bench/experiments at commit 40f164d5b8f1;
        - each run's submitted patch for every task, logs/<task>/patch.diff, from the public swe-bench-submissions
          bucket the entry names (5,500 files). Some runs also have a preds.json there; it is saved as a cross-check
          but not used, because on 2026-10-29 it held fewer than 500 tasks for 3 of the 4 runs that have one.
      One request per second, honest user agent, every response logged in fetch_log.jsonl. Re-running skips saved files.

  python reviewing-agent-diffs-swe-bench-scan.py scan --cache swe-bench-patches [--csv out.csv] [--show CHECK]
      Offline. For each patch it calls scan() from the scanner with --scope set to the files the maintainers' fix
      touched and --own set to the repository's own package, and counts a patch as tripping a check when the check
      fires at least once. Prints, per check, the share of gold fixes, gold test patches, resolved model patches and
      unresolved model patches that trip it. --scanner runs another copy of the scanner instead (for example the
      5 September version, which has no --own; the script then reports how many import hits were the repository's
      own package).

Words: "resolved" is the entry's own per-task record, not a re-run. An empty submission is a patch.diff with no diff;
it trips nothing, and it is counted in its column (resolved or not) so every column adds up to the attempts. Standard library only (Python 3.9+); the scanner file
must sit next to this one.
"""
from __future__ import annotations

import argparse
import collections
import csv
import hashlib
import importlib.util
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import urllib.robotparser
from datetime import datetime, timezone

UA = "vibecodetextbook-research/1.0 (+https://vibecodetextbook.com; SWE-bench patch snapshot for one article)"
AGENTS = ("ClaudeBot", "Claude-User", "anthropic-ai", "*")
SHA = "40f164d5b8f1d249bf95a6df8b74b577fd8e519d"
RUNS = ["20260217_mini-v2.0.0_claude-4-5-opus-high", "20260217_mini-v2.0.0_gemini-3-flash-high",
        "20260217_mini-v2.0.0_minimax-2-5-high", "20260217_mini-v2.0.0_claude-4-6-opus",
        "20260217_mini-v2.0.0_glm-5-high", "20260217_mini-v2.0.0_gpt-5-2-high",
        "20260217_mini-v2.0.0_claude-4-5-sonnet-high", "20260217_mini-v2.0.0_kimi-k2-5-high",
        "20260217_mini-v2.0.0_deepseek-3-2-high", "20260217_mini-v2.0.0_claude-4-5-haiku-high",
        "20260217_mini-v2.0.0_gpt-5-mini"]
ROWS = "https://datasets-server.huggingface.co/rows?dataset=SWE-bench%2FSWE-bench_Verified&config=default&split=test&offset={}&length=100"
RAW = f"https://raw.githubusercontent.com/SWE-bench/experiments/{SHA}/evaluation/verified/{{}}/per_instance_details.json"
S3 = "https://swe-bench-submissions.s3.amazonaws.com/bash-only/{}/preds.json"
S3_DIFF = "https://swe-bench-submissions.s3.amazonaws.com/bash-only/{}/logs/{}/patch.diff"
CHECKS = ["deleted-test", "skipped-test", "weakened-assert", "broad-except", "new-dependency", "out-of-scope",
          "ci-config", "secret-like", "leftover-marker", "big-deletion"]
HERE = os.path.dirname(os.path.abspath(__file__))


def load_scanner(path=None):
    spec = importlib.util.spec_from_file_location("scanner", path or os.path.join(HERE, "reviewing-agent-diffs-checklist.py"))
    mod = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


# ------------------------------------------------------------------ fetch ---
def cmd_fetch(a):
    os.makedirs(a.cache, exist_ok=True)
    log = open(os.path.join(a.cache, "fetch_log.jsonl"), "a", encoding="utf-8")
    last = [0.0]
    checked = {}

    def robots_ok(url):
        host = "{0.scheme}://{0.netloc}".format(urllib.parse.urlsplit(url))
        if host not in checked:
            req = urllib.request.Request(host + "/robots.txt", headers={"User-Agent": UA, "Accept": "text/plain"})
            rp = urllib.robotparser.RobotFileParser()
            try:
                rp.parse(urllib.request.urlopen(req, timeout=30).read().decode("utf-8", "replace").splitlines())
                status = 200
            except urllib.error.HTTPError as e:
                status = e.code
                rp.parse([] if 400 <= e.code < 500 else ["User-agent: *", "Disallow: /"])
            except Exception as e:  # noqa: BLE001
                status = repr(e)[:80]
                rp.parse(["User-agent: *", "Disallow: /"])
            checked[host] = rp
            print(f"robots {host}: {status}")
        return all(checked[host].can_fetch(ag, url) for ag in AGENTS)

    def get(url, rel, quiet=False):
        dst = os.path.join(a.cache, rel.replace("/", os.sep))
        if os.path.exists(dst):
            return True
        if not robots_ok(url):
            print("robots.txt disallows", url)
            return False
        wait = last[0] + 1.0 - time.time()
        if wait > 0:
            time.sleep(wait)
        status, body = None, b""
        try:
            body = urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=120).read()
            status = 200
        except urllib.error.HTTPError as e:
            status = e.code
        except (urllib.error.URLError, OSError) as e:  # a reset connection is logged; re-running fetches it again
            status = repr(e)[:120]
        last[0] = time.time()
        log.write(json.dumps({"url": url, "file": rel, "status": status, "bytes": len(body),
                              "sha256": hashlib.sha256(body).hexdigest() if body else None,
                              "utc": datetime.now(timezone.utc).isoformat(timespec="seconds")}) + "\n")
        log.flush()
        if status == 200:
            os.makedirs(os.path.dirname(dst), exist_ok=True)
            with open(dst, "wb") as fh:
                fh.write(body)
            return True
        if not quiet:
            print("failed", status, url)
        return False

    for off in range(0, 500, 100):
        get(ROWS.format(off), f"hf_rows_{off:03d}.json")
    ids = sorted(load_tasks(a.cache))
    for run in RUNS:
        get(RAW.format(run), f"runs/{run}/per_instance_details.json")
        pj = os.path.join(a.cache, "runs", run, "preds.json")
        if not os.path.exists(pj) and not os.path.isdir(os.path.join(a.cache, "runs", run, "patches")):
            get(S3.format(run), f"runs/{run}/preds.json", quiet=True)
        got = sum(get(S3_DIFF.format(run, iid), f"runs/{run}/patches/{iid}.diff", quiet=True) for iid in ids)
        print(f"{run}: {got} of {len(ids)} patch.diff files saved", flush=True)
    return 0


# ------------------------------------------------------------------- scan ---
def load_tasks(cache):
    tasks = {}
    for off in range(0, 500, 100):
        with open(os.path.join(cache, f"hf_rows_{off:03d}.json"), encoding="utf-8") as fh:
            for r in json.load(fh)["rows"]:
                tasks[r["row"]["instance_id"]] = r["row"]
    return tasks


def load_run(cache, run):
    with open(os.path.join(cache, "runs", run, "per_instance_details.json"), encoding="utf-8") as fh:
        record = json.load(fh)
    preds, pd = {}, os.path.join(cache, "runs", run, "patches")
    for name in os.listdir(pd):
        with open(os.path.join(pd, name), encoding="utf-8", newline="") as fh:
            preds[name[:-len(".diff")]] = {"model_patch": fh.read()}
    return record, preds


def files_of(scanner, diff):
    return sorted({path for path, *_ in scanner.parse(diff)})


def own_package(repo):
    """The import names a repository's own code uses: django/django -> (django,), scikit-learn -> (sklearn,)."""
    name = repo.split("/")[1]
    return {"scikit-learn": ("sklearn",), "matplotlib": ("matplotlib", "mpl_toolkits")}.get(name, (name.replace("-", "_").lower(),))


def cmd_scan(a):
    sc = load_scanner(a.scanner)
    has_own = "own" in sc.scan.__code__.co_varnames
    tasks = load_tasks(a.cache)
    groups = collections.OrderedDict((g, collections.Counter()) for g in ("gold fix", "gold tests", "model resolved", "model unresolved"))
    sizes = collections.Counter()
    empty = missing = 0
    empty_by = collections.Counter()
    extra = collections.defaultdict(collections.Counter)
    own_dep = collections.Counter()
    all_dep = collections.Counter()
    shown = 0
    rows = []

    def tally(group, diff, scope, repo, tag):
        nonlocal shown
        found = sc.scan(diff, scope, own_package(repo)) if has_own else sc.scan(diff, scope)
        kinds = {k for k, *_ in found}
        sizes[group] += 1
        groups[group].update(kinds)
        groups[group]["(any check)"] += bool(kinds)
        groups[group]["(any but out-of-scope)"] += bool(kinds - {"out-of-scope"})
        for kind, path, line, text in found:
            if kind == "new-dependency" and line is not None:
                mod = re.match(r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w.]+))", text)
                all_dep[group] += 1
                if mod and (mod.group(1) or mod.group(2)).split(".")[0] in own_package(repo):
                    own_dep[group] += 1
            if a.show and kind == a.show and shown < 12:
                shown += 1
                print(f"  [{group}] {tag} {kind} {path}:{line} {text[:90]}")
        return kinds

    for iid, t in sorted(tasks.items()):
        scope = files_of(sc, t["patch"])
        k1 = tally("gold fix", t["patch"], scope, t["repo"], iid)
        k2 = tally("gold tests", t["test_patch"], [], t["repo"], iid)
        rows.append({"source": "gold", "instance_id": iid, "fix_checks": ";".join(sorted(k1)), "test_checks": ";".join(sorted(k2))})
    for run in RUNS:
        record, preds = load_run(a.cache, run)
        for iid in sorted(tasks):
            if iid not in preds:
                missing += 1
            p = preds.get(iid) or {}
            diff = p.get("model_patch") or ""
            resolved = bool((record.get(iid) or {}).get("resolved"))
            group = "model resolved" if resolved else "model unresolved"
            if not diff.strip():
                empty += 1
                empty_by[group] += 1
                sizes[group] += 1
                rows.append({"source": run, "instance_id": iid, "resolved": resolved, "fix_checks": "(empty)"})
                continue
            gold_files = files_of(sc, tasks[iid]["patch"])
            kinds = tally(group, diff, gold_files, tasks[iid]["repo"], f"{run}:{iid}")
            created = set(re.findall(r"^diff --git a/(\S+) b/\S+\nnew file mode", diff, re.M))
            for path in set(files_of(sc, diff)) - set(gold_files):
                extra[group]["new file" if path in created else "test path" if re.search(r"(^|/)tests?/|(^|/)test_", path)
                             else "existing source file"] += 1
            rows.append({"source": run, "instance_id": iid, "resolved": resolved, "fix_checks": ";".join(sorted(kinds))})

    print(f"SWE-bench Verified tasks: {len(tasks)}; runs: {len(RUNS)}; model patches: "
          f"{sizes['model resolved'] + sizes['model unresolved']:,} ({sizes['model resolved']:,} resolved, "
          f"{sizes['model unresolved']:,} unresolved)")
    print(f"empty submissions: {empty} ({empty_by['model resolved']} resolved, {empty_by['model unresolved']} unresolved); "
          f"patch.diff missing from the snapshot: {missing}")
    head = f"{'check':<24}" + "".join(f"{g:>18}" for g in groups)
    print(head)
    print(f"{'(patches)':<24}" + "".join(f"{sizes[g]:>18,}" for g in groups))
    for c in CHECKS + ["(any check)", "(any but out-of-scope)"]:
        print(f"{c:<24}" + "".join(f"{groups[g][c]:>9,} {100 * groups[g][c] / sizes[g]:>6.1f}%" for g in groups))
    print("files a model patch touched outside the gold fix's files:")
    for g in ("model resolved", "model unresolved"):
        print(f"  {g:<18}" + ", ".join(f"{k} {v:,}" for k, v in sorted(extra[g].items(), key=lambda kv: -kv[1])))
    print(f"scanner: {os.path.basename(a.scanner or 'reviewing-agent-diffs-checklist.py')}"
          f"{' (--own = the repository package)' if has_own else ' (no --own)'}")
    print("new-dependency hits on import lines that import the repository's own package:")
    for g in groups:
        if all_dep[g]:
            print(f"  {g:<18}{own_dep[g]:>6,} of {all_dep[g]:>6,} ({100 * own_dep[g] / all_dep[g]:.1f}%)")
    if a.csv:
        with open(a.csv, "w", newline="", encoding="utf-8") as fh:
            w = csv.DictWriter(fh, fieldnames=["source", "instance_id", "resolved", "fix_checks", "test_checks"])
            w.writeheader()
            w.writerows(rows)
    return 0


def main(argv):
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    sub = ap.add_subparsers(dest="cmd", required=True)
    f = sub.add_parser("fetch")
    f.add_argument("--cache", required=True)
    s = sub.add_parser("scan")
    s.add_argument("--cache", required=True)
    s.add_argument("--csv")
    s.add_argument("--show", help="print up to 12 hits of this check")
    s.add_argument("--scanner", help="path of another copy of reviewing-agent-diffs-checklist.py")
    a = ap.parse_args(argv)
    return cmd_fetch(a) if a.cmd == "fetch" else cmd_scan(a)


if __name__ == "__main__":
    sys.stdout.reconfigure(encoding="utf-8")
    sys.exit(main(sys.argv[1:]))