#!/usr/bin/env python3
"""score.py  v1.0  2026-09-17  MIT  Distru No Bullshit AI Course (08-moonshots/03)

Score a model's outputs against tasks.csv.

    python3 score.py tasks.csv outputs.csv

tasks.csv   columns: task_id, category, input, expected, match   (match = exact | numeric | contains)
outputs.csv columns: task_id, model_output [, confidence]         (confidence 0..1, optional)

Prints accuracy overall and per category. If a confidence column is present it also prints a
calibration table: rows are confidence bins, columns are how many answers landed in the bin, the
mean confidence, and the share that were actually right. A calibrated model has the last two
columns close to each other. Expected calibration error (ECE) is the count-weighted mean gap.

Tested 2026-09-17 with python3.12 against outputs.sample.csv. Standard library only.
"""
import csv
import sys
from collections import defaultdict


def norm(s):
    return " ".join(str(s).strip().lower().split())


def is_right(expected, got, match):
    if match == "numeric":
        try:
            return abs(float(str(got).replace(",", "").replace("$", "")) - float(expected)) < 1e-6
        except ValueError:
            return False
    if match == "contains":
        return norm(expected) in norm(got)
    return norm(expected) == norm(got)


def main(tasks_path, outputs_path, bins=5):
    with open(tasks_path, newline="") as f:
        tasks = {r["task_id"]: r for r in csv.DictReader(f)}
    with open(outputs_path, newline="") as f:
        outputs = list(csv.DictReader(f))
    has_conf = outputs and "confidence" in outputs[0] and any(o.get("confidence") for o in outputs)

    rows = []
    for o in outputs:
        t = tasks.get(o["task_id"])
        if t is None:
            print(f"warning: {o['task_id']} not in tasks.csv, skipped", file=sys.stderr)
            continue
        ok = is_right(t["expected"], o["model_output"], t.get("match", "exact"))
        conf = float(o["confidence"]) if has_conf and o.get("confidence") else None
        rows.append((t, o["model_output"], ok, conf))

    missing = set(tasks) - {r[0]["task_id"] for r in rows}
    for m in sorted(missing):
        print(f"warning: no output for {m}", file=sys.stderr)

    n = len(rows)
    right = sum(1 for r in rows if r[2])
    print(f"accuracy: {right}/{n} = {right / n:.0%}" if n else "no rows scored")

    by_cat = defaultdict(lambda: [0, 0])
    for t, _, ok, _ in rows:
        by_cat[t["category"]][0] += ok
        by_cat[t["category"]][1] += 1
    print("\nby category")
    for cat, (r, c) in sorted(by_cat.items()):
        print(f"  {cat:12s} {r}/{c}")

    print("\nmisses")
    for t, got, ok, conf in rows:
        if not ok:
            c = f"  conf={conf:.2f}" if conf is not None else ""
            print(f"  {t['task_id']}  expected={t['expected']!r}  got={got!r}{c}")

    if has_conf:
        print(f"\ncalibration ({bins} bins)")
        print(f"  {'bin':12s} {'n':>3s} {'mean conf':>10s} {'accuracy':>9s} {'gap':>6s}")
        ece = 0.0
        for b in range(bins):
            lo, hi = b / bins, (b + 1) / bins
            inb = [r for r in rows if r[3] is not None and (lo <= r[3] < hi or (b == bins - 1 and r[3] == 1.0))]
            if not inb:
                print(f"  {lo:.1f}-{hi:.1f}      {0:3d}          -         -      -")
                continue
            mc = sum(r[3] for r in inb) / len(inb)
            acc = sum(1 for r in inb if r[2]) / len(inb)
            ece += len(inb) / n * abs(mc - acc)
            print(f"  {lo:.1f}-{hi:.1f}      {len(inb):3d} {mc:10.2f} {acc:9.0%} {mc - acc:+6.2f}")
        print(f"  ECE = {ece:.3f}  (0 is perfect; a positive gap means overconfident)")
    return 0 if n else 1


if __name__ == "__main__":
    if len(sys.argv) != 3:
        print(__doc__)
        sys.exit(2)
    sys.exit(main(sys.argv[1], sys.argv[2]))