"""time-to-green runner — time one small agent task from prompt to first green check. What it does 1. Looks up a task row (task_id) in the task CSV shipped with the article. 2. Starts a clock, launches the agent command with the task text as the prompt, waits for it to exit (or times out), and stops the clock. 3. Parses the agent's stdout as JSON when it can (claude -p --output-format json returns total_cost_usd, num_turns, duration_api_ms), then runs the task's green command. Green means "the named check exits 0". 4. Appends one row to the results CSV with the same columns as the task CSV. Inputs / flags --task T03 task id (required) --tasks path.csv task definitions (default: ../datasets/.csv) --out results.csv where rows are appended (default: time-to-green-results.csv) --agent-cmd "..." command template; {prompt} is replaced by the task text. Default: claude -p "{prompt}" --output-format json --max-turns 30 --max-budget-usd 2.00 --green-cmd "..." override the task's green command (for stand-in runs) --harness/--harness-version/--model labels written into the row --timeout 1800 seconds before the agent process is killed (outcome=timeout) How to run (real) python time-to-green-ten-small-tasks-method.py --task T01 --harness "Claude Code" \ --harness-version v2.1.261 --model claude-sonnet-5 How to run (stand-in, proves the timing and CSV path without any agent) python time-to-green-ten-small-tasks-method.py --task T01 \ --agent-cmd "python -c \"import json;print(json.dumps({'total_cost_usd':0.0,'num_turns':0}))\"" \ --green-cmd "python -m unittest build-a-cli-tool-with-an-agent-from-a-spec_test.py" Standard library only. Python 3.13. Windows and POSIX (commands run through the shell). """ from __future__ import annotations import argparse import csv import datetime as dt import json import os import subprocess import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) DEFAULT_TASKS = os.path.join(HERE, "..", "datasets", "time-to-green-ten-small-tasks-method.csv") COLUMNS = ["task_id", "task", "green_command", "harness", "harness_version", "model", "started_at", "green_at", "seconds_to_green", "turns", "tool_calls", "cost_usd", "outcome", "notes"] DEFAULT_AGENT = 'claude -p "{prompt}" --output-format json --max-turns 30 --max-budget-usd 2.00' def load_task(path: str, task_id: str) -> dict[str, str]: with open(path, newline="", encoding="utf-8") as fh: for row in csv.DictReader(fh): if row["task_id"] == task_id: return row raise SystemExit(f"no task {task_id!r} in {path}") def run_agent(cmd: str, timeout: float) -> tuple[dict, str, float]: """Run the agent command; return (parsed json or {}, outcome-or-empty, elapsed seconds).""" t0 = time.monotonic() try: proc = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) except subprocess.TimeoutExpired: return {}, "timeout", time.monotonic() - t0 elapsed = time.monotonic() - t0 payload: dict = {} text = proc.stdout.strip() if text: try: payload = json.loads(text.splitlines()[-1]) # json mode prints one object; stream mode's last line is the result except json.JSONDecodeError: payload = {} if proc.returncode != 0: return payload, "agent_error", elapsed return payload, "", elapsed def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--task", required=True) ap.add_argument("--tasks", default=DEFAULT_TASKS) ap.add_argument("--out", default="time-to-green-results.csv") ap.add_argument("--agent-cmd", default=DEFAULT_AGENT) ap.add_argument("--green-cmd") ap.add_argument("--harness", default="") ap.add_argument("--harness-version", default="") ap.add_argument("--model", default="") ap.add_argument("--timeout", type=float, default=1800) args = ap.parse_args(argv) task = load_task(args.tasks, args.task) prompt = task["task"].replace('"', "'") cmd = args.agent_cmd.replace("{prompt}", prompt) green_cmd = args.green_cmd or task["green_command"] started = dt.datetime.now(dt.timezone.utc) payload, outcome, elapsed = run_agent(cmd, args.timeout) check = subprocess.run(green_cmd, shell=True, capture_output=True, text=True) green = check.returncode == 0 finished = dt.datetime.now(dt.timezone.utc) if not outcome: outcome = "green" if green else "red" elif green: outcome += "+green" row = { "task_id": task["task_id"], "task": task["task"], "green_command": green_cmd, "harness": args.harness, "harness_version": args.harness_version, "model": args.model, "started_at": started.isoformat(timespec="seconds"), "green_at": finished.isoformat(timespec="seconds") if green else "", "seconds_to_green": f"{elapsed:.1f}" if green else "", "turns": payload.get("num_turns", ""), "tool_calls": "", "cost_usd": payload.get("total_cost_usd", ""), "outcome": outcome, "notes": f"agent_elapsed_s={elapsed:.1f}; green_exit={check.returncode}; " f"api_ms={payload.get('duration_api_ms', '')}", } new_file = not os.path.exists(args.out) with open(args.out, "a", newline="", encoding="utf-8") as fh: w = csv.DictWriter(fh, fieldnames=COLUMNS, lineterminator="\n") if new_file: w.writeheader() w.writerow(row) print(json.dumps({k: row[k] for k in ("task_id", "outcome", "seconds_to_green", "turns", "cost_usd", "notes")})) return 0 if green else 1 if __name__ == "__main__": sys.exit(main())