"""swe-bench-explained-what-the-numbers-measure.py - how big a gap between two SWE-bench Verified scores is real. What it does Reads the leaderboard's own per-task records (the SWE-bench/experiments repository at one pinned commit) and the 500 SWE-bench Verified tasks, the same inputs as swe-bench-verified-task-by-task.py, and prints: 1. each mini-swe-agent 2.0.0 run's score with a 95% Wilson interval (the 500 tasks read as a sample of tasks like them); 2. every pair of those runs compared on the tasks they split, with an exact two-sided McNemar test (a binomial test on the tasks one run resolved and the other did not), unadjusted and with Holm's correction; 3. how often each run ranks first when the 500 tasks are resampled with replacement (fixed seed); 4. across every Verified entry with a per-task record, which entries cannot be separated from the top score; 5. the share of the score that comes from django/django, and each run's Django and non-Django rates, with the non-Django tasks reweighted to Django's mix of difficulty labels. Inputs: network access to api.github.com (one request, unauthenticated), raw.githubusercontent.com and datasets-server.huggingface.co, or a --cache folder holding responses saved by an earlier run of this script or of swe-bench-verified-task-by-task.py (the two use the same folder layout). Standard library only (Python 3.9 or later). Run: python swe-bench-explained-what-the-numbers-measure.py # live python swe-bench-explained-what-the-numbers-measure.py --cache saved # read saved responses, save any it fetches What it does not do: it re-derives no verdict, runs no agent or test, and says nothing about how often a second attempt by the same model would differ (every run here made one attempt per task). A p-value here is about the tasks, not about the runs. """ from __future__ import annotations import argparse import json import math import os import random import re import sys import urllib.error import urllib.request REPO = "SWE-bench/experiments" PINNED = "40f164d5b8f1d249bf95a6df8b74b577fd8e519d" # main on 2026-09-03, the latest commit on 2026-10-08 ROWS = ("https://datasets-server.huggingface.co/rows?dataset=SWE-bench%2FSWE-bench_Verified" "&config=default&split=test&offset={off}&length=100") UA = "swe-bench-explained-what-the-numbers-measure/1.0 (research script)" DIFFICULTY = ["<15 min fix", "15 min - 1 hour", "1-4 hours", ">4 hours"] Z95 = 1.959963984540054 class Source: """Fetch a URL, or read it from the cache folder; save what was fetched when a cache folder is given.""" def __init__(self, cache: str | None): self.cache = cache def get(self, url: str, rel: str, optional: bool = False, accept: str | None = None) -> bytes | None: path = os.path.join(self.cache, rel.replace("/", os.sep)) if self.cache else None if path and os.path.exists(path): with open(path, "rb") as f: return f.read() if path and os.path.exists(path + ".404"): return None headers = {"User-Agent": UA} if accept: headers["Accept"] = accept try: with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=90) as r: body = r.read() except urllib.error.HTTPError as e: if e.code == 404 and optional: if path: os.makedirs(os.path.dirname(path), exist_ok=True) open(path + ".404", "w").close() return None raise if path: os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "wb") as f: f.write(body) return body def info_block(yaml_text: str) -> dict: """The scalar keys under the top-level `info:` key of an entry's metadata.yaml.""" out, top = {}, None for line in yaml_text.splitlines(): if line and not line.startswith(" ") and line.rstrip().endswith(":"): top = line.strip()[:-1] continue m = re.match(r"^ ([A-Za-z0-9_-]+):\s*(.*)$", line) if top == "info" and m and m.group(2) and not m.group(2).startswith("-"): out[m.group(1)] = m.group(2).strip().strip("'\"") return out def load(src: Source, sha: str): raw = f"https://raw.githubusercontent.com/{REPO}/{sha}/" listing = json.loads(src.get(f"https://api.github.com/repos/{REPO}/contents/evaluation/verified?ref={sha}", "contents_evaluation_verified.json", accept="application/vnd.github+json")) entries = {} for e in sorted(listing, key=lambda e: e["name"]): if e["type"] != "dir": continue d, base = e["name"], f"evaluation/verified/{e['name']}/" meta = src.get(raw + base + "metadata.yaml", f"verified/{d}/metadata.yaml", optional=True) rec, kind = src.get(raw + base + "per_instance_details.json", f"verified/{d}/per_instance_details.json", optional=True), "per_instance_details" if rec is None: rec, kind = src.get(raw + base + "results/results.json", f"verified/{d}/results/results.json", optional=True), "results" runs, resolved = None, None if rec is not None: j = json.loads(rec) if kind == "per_instance_details": runs = j resolved = {i for i, v in j.items() if v.get("resolved")} else: resolved = set(j.get("resolved", [])) entries[d] = {"info": info_block(meta.decode("utf-8")) if meta else {}, "runs": runs, "resolved": resolved} tasks = {} for off in range(0, 500, 100): page = json.loads(src.get(ROWS.format(off=off), f"hf_rows_{off:03d}.json")) for r in page["rows"]: row = r["row"] tasks[row["instance_id"]] = {"repo": row["repo"], "difficulty": row["difficulty"]} if len(tasks) != 500: sys.exit(f"expected 500 Verified tasks, read {len(tasks)}") return entries, tasks def wilson(k: int, n: int) -> tuple[float, float]: """95% Wilson score interval for k successes in n trials.""" p, z2 = k / n, Z95 * Z95 den = 1 + z2 / n mid = (p + z2 / (2 * n)) / den half = Z95 * math.sqrt(p * (1 - p) / n + z2 / (4 * n * n)) / den return mid - half, mid + half def mcnemar_exact(b: int, c: int) -> float: """Two-sided exact McNemar p-value: a binomial(b + c, 1/2) test on the split tasks.""" n, k = b + c, min(b, c) if n == 0: return 1.0 return min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) def holm(pvals: list[float], alpha: float = 0.05) -> int: """How many of the hypotheses Holm's step-down procedure rejects at family-wise level alpha.""" m = len(pvals) for r, p in enumerate(sorted(pvals)): if p > alpha / (m - r): return r return m def pct(x: float) -> str: return f"{100 * x:.1f}%" def main(argv=None) -> int: ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--cache", help="folder of saved responses (read first; anything fetched is saved there)") ap.add_argument("--sha", default=PINNED, help="SWE-bench/experiments commit to read (default: %(default).12s)") ap.add_argument("--harness", default="2.0.0", help="mini-swe-agent version to hold fixed (default: %(default)s)") ap.add_argument("--resamples", type=int, default=10000, help="bootstrap resamples (default: %(default)s)") ap.add_argument("--seed", type=int, default=20261105, help="bootstrap seed (default: %(default)s)") a = ap.parse_args(argv) if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") entries, tasks = load(Source(a.cache), a.sha) ids = sorted(tasks) n_tasks = len(ids) with_rec = {d: e["resolved"] for d, e in entries.items() if e["resolved"] is not None} # the runs held to one harness version whose record agrees with the entry's own metadata # (the same rule as swe-bench-verified-task-by-task.py: one harness version; a record that covers all 500 tasks # and agrees with the entry's own stated resolve rate; neither 0 nor 500 resolved) same = sorted(d for d, e in entries.items() if e["info"].get("mini-swe-agent_version") == a.harness) runs = {} for d in same: e = entries[d] if e["runs"] is None or set(e["runs"]) != set(ids): continue k, stated = len(e["resolved"] & set(ids)), e["info"].get("resolved") if stated is None or abs(100 * k / n_tasks - float(stated)) > 0.05 or k in (0, n_tasks): continue runs[e["info"].get("name", d)] = e["resolved"] & set(ids) names = sorted(runs, key=lambda n: (-len(runs[n]), n)) print(f"{REPO} at {a.sha[:12]}: {len(with_rec)} Verified entries with a per-task record; {n_tasks} tasks") print(f"mini-swe-agent {a.harness} entries: {len(same)}; runs used (a full record that agrees with its own " f"metadata): {len(names)}") print("\n1. One score on 500 tasks, with a 95% Wilson interval") print(f" {'run':26s} resolved score 95% interval width") widths = [] for n in names: k = len(runs[n]) lo, hi = wilson(k, n_tasks) widths.append(hi - lo) print(f" {n:26s} {k:6d} {pct(k / n_tasks):>6s} {100 * lo:4.1f}% to {100 * hi:4.1f}% {100 * (hi - lo):4.1f} pts") print(f" interval width: {100 * min(widths):.1f} to {100 * max(widths):.1f} points; one task = {100 / n_tasks:.1f} points") print("\n2. Every pair of runs, exact two-sided McNemar test on the tasks they split") pairs = [] for i, x in enumerate(names): for y in names[i + 1:]: b, c = len(runs[x] - runs[y]), len(runs[y] - runs[x]) pairs.append((x, y, len(runs[x]) - len(runs[y]), b, c, mcnemar_exact(b, c))) sig = [p for p in pairs if p[5] < 0.05] notsig = [p for p in pairs if p[5] >= 0.05] print(f" pairs: {len(pairs)}; separable at p < 0.05: {len(sig)}; " f"after Holm's correction for {len(pairs)} tests: {holm([p[5] for p in pairs])}") gmax_ns, gmin_s = max(p[2] for p in notsig), min(p[2] for p in sig) print(f" largest gap not separable: {gmax_ns} tasks ({100 * gmax_ns / n_tasks:.1f} points); " f"smallest gap separable: {gmin_s} tasks ({100 * gmin_s / n_tasks:.1f} points)") print(f" split tasks per pair: {min(p[3] + p[4] for p in pairs)} to {max(p[3] + p[4] for p in pairs)}") print(f" {'run A':26s} {'run B':26s} gap A only B only p") for x, y, g, b, c, p in sorted(pairs, key=lambda t: (t[2], t[0], t[1])): if g <= gmin_s: print(f" {x:26s} {y:26s} {g:3d} {b:6d} {c:6d} {p:6.3f}") print(f"\n3. Ranked first when the 500 tasks are resampled ({a.resamples:,} resamples, seed {a.seed}, ties split)") rng = random.Random(a.seed) pos = {t: j for j, t in enumerate(ids)} idx = {n: [pos[t] for t in runs[n]] for n in names} first = dict.fromkeys(names, 0.0) for _ in range(a.resamples): w = [0] * n_tasks for _ in range(n_tasks): w[int(rng.random() * n_tasks)] += 1 sc = {n: sum(w[j] for j in idx[n]) for n in names} best = max(sc.values()) top = [n for n in names if sc[n] == best] for n in top: first[n] += 1 / len(top) for n in names: share = first[n] / a.resamples print(f" {n:26s} {pct(share):>6s} {first[n]:8,.1f} of {a.resamples:,}") print("\n4. All entries with a per-task record: who cannot be separated from the top score") best = max(len(v) for v in with_rec.values()) tops = sorted(d for d, v in with_rec.items() if len(v) == best) print(f" most resolved: {best} ({', '.join(tops)})") if len(tops) > 1: b, c = len(with_rec[tops[0]] - with_rec[tops[1]]), len(with_rec[tops[1]] - with_rec[tops[0]]) print(f" the first two split {b + c} tasks, {b} to {c}") inseparable = sorted((d for d in with_rec if d not in tops and any( mcnemar_exact(len(with_rec[t] - with_rec[d]), len(with_rec[d] - with_rec[t])) >= 0.05 for t in tops)), key=lambda d: (-len(with_rec[d]), d)) rest = [len(with_rec[d]) for d in with_rec if d not in tops and d not in inseparable] print(f" not separable from the top at p < 0.05: {len(inseparable)} entries, " f"{min(len(with_rec[d]) for d in inseparable)} to {max(len(with_rec[d]) for d in inseparable)} resolved") for d in inseparable: print(f" {len(with_rec[d])} {d}") print(f" separable from the top: {len(rest)} entries, the highest of them {max(rest)} resolved") print("\n5. One repository's weight in the score") big = max({tasks[t]["repo"] for t in ids}, key=lambda r: sum(1 for t in ids if tasks[t]["repo"] == r)) inb = [t for t in ids if tasks[t]["repo"] == big] out = [t for t in ids if tasks[t]["repo"] != big] print(f" {big}: {len(inb)} of {n_tasks} tasks ({pct(len(inb) / n_tasks)}); the other " f"{len({tasks[t]['repo'] for t in out})} repositories: {len(out)}") for lab in DIFFICULTY: a_ = [t for t in inb if tasks[t]["difficulty"] == lab] b_ = [t for t in out if tasks[t]["difficulty"] == lab] ra = sum(1 for n in names for t in a_ if t in runs[n]) / (len(names) * len(a_)) if a_ else None rb = sum(1 for n in names for t in b_ if t in runs[n]) / (len(names) * len(b_)) if b_ else None print(f" {lab:16s} {big}: {len(a_):3d} tasks, all runs {pct(ra) if ra is not None else '-':>6s} " f"others: {len(b_):3d} tasks, all runs {pct(rb) if rb is not None else '-':>6s}") print(f" {'run':26s} score {big.split('/')[1]:>6s} others others at {big.split('/')[1]}'s label mix difference") diffs = [] order_out = sorted(names, key=lambda n: (-sum(1 for t in out if t in runs[n]), n)) for n in names: rin = sum(1 for t in inb if t in runs[n]) / len(inb) rout = sum(1 for t in out if t in runs[n]) / len(out) mix = 0.0 for lab in DIFFICULTY: a_ = [t for t in inb if tasks[t]["difficulty"] == lab] b_ = [t for t in out if tasks[t]["difficulty"] == lab] if a_: mix += sum(1 for t in b_ if t in runs[n]) / len(b_) * len(a_) / len(inb) diffs.append(rin - mix) print(f" {n:26s} {pct(len(runs[n]) / n_tasks):>6s} {pct(rin):>6s} {pct(rout):>6s} {pct(mix):>27s} " f"{100 * (rin - mix):+5.1f} pts") print(f" runs with a higher rate on {big} than on the others at its label mix: " f"{sum(1 for d in diffs if d > 0)} of {len(names)} ({100 * min(diffs):+.1f} to {100 * max(diffs):+.1f} pts)") print(" order on the other " + f"{len(out)} tasks: " + ", ".join( f"{n} {pct(sum(1 for t in out if t in runs[n]) / len(out))}" for n in order_out[:4]) + ", ...") return 0 if __name__ == "__main__": sys.exit(main())