136 lines
5.2 KiB
Python
136 lines
5.2 KiB
Python
"""Compare two Harbor job directories and print a Markdown summary.
|
|
|
|
python -m harbor_coder.compare --baseline jobs/baseline --candidate jobs/candidate --labels main my-branch
|
|
|
|
Infrastructure errors are excluded: trials that raised before the agent
|
|
started, and trials whose chat failed for gateway or deployment reasons
|
|
(``CoderInfraError``). Other trials with no reward score 0.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import statistics
|
|
import sys
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
# Must match harbor_coder.agent.CoderInfraError; Harbor records the class name.
|
|
_INFRA_EXCEPTION = "CoderInfraError"
|
|
_BOOTSTRAP_SAMPLES = 10_000
|
|
|
|
|
|
def _summarize(job_dir: Path) -> dict:
|
|
by_task: dict[str, list[float]] = defaultdict(list)
|
|
cost = 0.0
|
|
infra = queued = 0
|
|
models: set[str] = set()
|
|
versions: set[str] = set()
|
|
# A trial dir without result.json crashed hard enough that Harbor never
|
|
# finalized it; surface it rather than silently shrinking the sample.
|
|
missing = [
|
|
d.name
|
|
for d in sorted(job_dir.iterdir())
|
|
if d.is_dir() and not d.name.startswith(".") and not (d / "result.json").exists()
|
|
]
|
|
for f in sorted(job_dir.glob("*/result.json")):
|
|
t = json.loads(f.read_text())
|
|
if (v := (t.get("agent_info") or {}).get("version")) and v != "unknown":
|
|
versions.add(v)
|
|
exc = t.get("exception_info") or {}
|
|
if exc and (
|
|
not (t.get("agent_execution") or {}).get("started_at") or exc.get("exception_type") == _INFRA_EXCEPTION
|
|
):
|
|
infra += 1
|
|
continue
|
|
rewards = (t.get("verifier_result") or {}).get("rewards") or {}
|
|
by_task[t.get("task_name") or "?"].append(float(rewards.get("reward", 0.0)))
|
|
agent = t.get("agent_result") or {}
|
|
cost += agent.get("cost_usd") or 0.0
|
|
meta = agent.get("metadata") or {}
|
|
if m := meta.get("model"):
|
|
models.add(m)
|
|
queued += bool(meta.get("queued_for_capacity"))
|
|
rewards = [r for rs in by_task.values() for r in rs]
|
|
return {
|
|
"by_task": by_task,
|
|
"mean": statistics.fmean(rewards) if rewards else 0.0,
|
|
"n": len(rewards),
|
|
"infra": infra,
|
|
"missing": missing,
|
|
"queued": queued,
|
|
"cost": cost,
|
|
"model": ", ".join(sorted(models)) or "-",
|
|
"version": ", ".join(sorted(versions)) or "-",
|
|
}
|
|
|
|
|
|
def paired_ci(
|
|
base: dict[str, list[float]], cand: dict[str, list[float]], n: int = _BOOTSTRAP_SAMPLES, seed: int = 0
|
|
) -> tuple[float, float, float, int] | None:
|
|
"""Mean per-task difference and its 95% bootstrap interval over tasks scored in both arms."""
|
|
tasks = sorted(set(base) & set(cand))
|
|
if not tasks:
|
|
return None
|
|
diffs = [statistics.fmean(cand[t]) - statistics.fmean(base[t]) for t in tasks]
|
|
rng = random.Random(seed)
|
|
means = sorted(statistics.fmean(rng.choices(diffs, k=len(diffs))) for _ in range(n))
|
|
return statistics.fmean(diffs), means[int(0.025 * n)], means[int(0.975 * n) - 1], len(tasks)
|
|
|
|
|
|
def _rate(xs: list[float]) -> str:
|
|
return f"{sum(1 for x in xs if x >= 1.0)}/{len(xs)}" if xs else "-"
|
|
|
|
|
|
def main() -> None:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--baseline", required=True, type=Path)
|
|
p.add_argument("--candidate", required=True, type=Path)
|
|
p.add_argument("--labels", nargs=2, default=["main", "branch"])
|
|
args = p.parse_args()
|
|
base, cand = _summarize(args.baseline), _summarize(args.candidate)
|
|
lb, lc = args.labels
|
|
ci = paired_ci(base["by_task"], cand["by_task"])
|
|
if ci is None:
|
|
verdict = "no data"
|
|
else:
|
|
delta, lo, hi, n_tasks = ci
|
|
interval = f"{delta:+.3f}, 95% CI [{lo:+.3f}, {hi:+.3f}] over {n_tasks} tasks"
|
|
if lo > 0 or hi < 0:
|
|
verdict = f"{lc} is {'better' if delta > 0 else 'worse'} than {lb}: {interval}"
|
|
else:
|
|
verdict = f"no significant difference: {interval}"
|
|
tasks = sorted(set(base["by_task"]) | set(cand["by_task"]))
|
|
lines = [
|
|
f"## Coder Agents eval: {lc} vs {lb}",
|
|
"",
|
|
f"**{verdict}**",
|
|
"",
|
|
f"| | {lb} | {lc} |",
|
|
"| --- | ---: | ---: |",
|
|
f"| Task success | {base['mean']:.3f} | {cand['mean']:.3f} |",
|
|
f"| coderd | {base['version']} | {cand['version']} |",
|
|
f"| Model | {base['model']} | {cand['model']} |",
|
|
f"| Trials scored | {base['n']} | {cand['n']} |",
|
|
f"| Infra errors (excluded) | {base['infra']} | {cand['infra']} |",
|
|
f"| Cost | ${base['cost']:.2f} | ${cand['cost']:.2f} |",
|
|
]
|
|
for key, label in (("missing", "Trials without result.json"), ("queued", "Trials queued for chat capacity")):
|
|
b, c = (len(s[key]) if isinstance(s[key], list) else s[key] for s in (base, cand))
|
|
if b or c:
|
|
lines.append(f"| {label} | {b} | {c} |")
|
|
lines += ["", f"| Task | {lb} | {lc} |", "| --- | ---: | ---: |"]
|
|
lines += [
|
|
f"| {t} | {_rate(base['by_task'].get(t, []))} | {_rate(cand['by_task'].get(t, []))} |" for t in tasks
|
|
]
|
|
for label, s in ((lb, base), (lc, cand)):
|
|
if s["missing"]:
|
|
print(f"warning: {label}: no result.json in {', '.join(s['missing'])}", file=sys.stderr)
|
|
print("\n".join(lines))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|