Bench Compare

git worktree list

grep -A1 '^\[\[bench\]\]' crates/shuck-benchmark/Cargo.toml | grep '^name'

SCRATCH=$(mktemp -d "${TMPDIR:-/tmp}/shuck-bench-compare.XXXXXX")
export CARGO_TARGET_DIR="$SCRATCH/target"
echo "Scratch: $SCRATCH"

mkdir -p "$SCRATCH/main-macro" "$SCRATCH/current-macro"

cd /path/to/main/worktree
env CARGO_TARGET_DIR="$SCRATCH/target" \
  cargo bench -p shuck-benchmark \
    --bench lexer --bench lexer_hot_path --bench parser --bench semantic --bench linter \
    -- --save-baseline=main --noplot \
  > "$SCRATCH/main-criterion.log" 2>&1

cd /path/to/current/worktree
env CARGO_TARGET_DIR="$SCRATCH/target" \
  cargo bench -p shuck-benchmark \
    --bench lexer --bench lexer_hot_path --bench parser --bench semantic --bench linter \
    -- --baseline=main --noplot \
  > "$SCRATCH/current-criterion.log" 2>&1

cd /path/to/worktree

# Clear stale bench exports to avoid mixing results
rm -f .cache/bench-*.json .cache/bench-*.md 2>/dev/null || true

# Build and verify deps
nix --extra-experimental-features 'nix-command flakes' develop --command \
  ./scripts/benchmarks/setup.sh > "$SCRATCH/{side}-macro-setup.log" 2>&1

# Run hyperfine comparisons
nix --extra-experimental-features 'nix-command flakes' develop --command \
  ./scripts/benchmarks/run.sh > "$SCRATCH/{side}-macro.log" 2>&1

# Copy exports to scratch
cp .cache/bench-*.json "$SCRATCH/{side}-macro/"

import json, pathlib

ROOT = pathlib.Path("$SCRATCH")
crit = ROOT / "target" / "criterion"
rows = []

for group in sorted(p for p in crit.iterdir() if p.is_dir()):
    for case in sorted(p for p in group.iterdir() if p.is_dir()):
        try:
            base = json.loads((case / "main" / "estimates.json").read_text())["mean"]["point_estimate"]
            new = json.loads((case / "new" / "estimates.json").read_text())["mean"]["point_estimate"]
            change = json.loads((case / "change" / "estimates.json").read_text())["mean"]["point_estimate"] * 100
        except FileNotFoundError:
            continue
        rows.append((group.name, case.name, base, new, change))

print("=== Criterion Summary (group-level) ===")
for group, case, base, new, change in rows:
    if case == "all":
        sign = "+" if change > 0 else ""
        print(f"  {group:30s}  {base:12.0f} → {new:12.0f} ns  ({sign}{change:.2f}%)")

print("\n=== Top Regressions ===")
for group, case, base, new, change in sorted(rows, key=lambda r: r[4], reverse=True)[:10]:
    print(f"  {group}/{case:20s}  {change:+.2f}%")

print("\n=== Top Improvements ===")
for group, case, base, new, change in sorted(rows, key=lambda r: r[4])[:10]:
    print(f"  {group}/{case:20s}  {change:+.2f}%")

for p in sorted((ROOT / "main-macro").glob("bench-*.json")):
    name = p.stem.removeprefix("bench-")
    main_results = json.loads(p.read_text())["results"]
    cur_results = json.loads((ROOT / "current-macro" / p.name).read_text())["results"]
    m_shuck = next(r for r in main_results if r["command"].startswith("shuck/"))
    c_shuck = next(r for r in cur_results if r["command"].startswith("shuck/"))
    change = (c_shuck["mean"] / m_shuck["mean"] - 1) * 100
    print(f"  {name:30s}  {m_shuck['mean']*1000:.1f} → {c_shuck['mean']*1000:.1f} ms  ({change:+.2f}%)")