Autoresearch

{
  "skill_name": "autoresearch",
  "evals": [
    {
      "id": 1,
      "prompt": "Help me set up Karpathy autoresearch on a 40GB GPU, verify the machine, and run the first baseline experiment.",
      "expected_output": "The skill chooses setup-readiness mode, preserves the immutable harness, and points to concrete setup / hardware / baseline commands instead of dumping every downstream detail at once.",
      "assertions": [
        "The workflow clearly identifies setup or readiness as the chosen mode.",
        "The workflow includes hardware verification or baseline-run commands such as `check-hardware.sh`, `uv sync`, `uv run prepare.py`, or `uv run train.py`.",
        "The response keeps the immutable 300-second / `prepare.py` / `val_bpb` contract visible."
      ]
    },
    {
      "id": 2,
      "prompt": "My repo runs, but my `program.md` just says 'try improvements'. Help me fix it before tonight's run.",
      "expected_output": "The skill chooses `program.md` authoring mode, asks for or fills in the baseline/priority/constraints structure, and keeps the loop tied to `train.py` rather than generic eval tooling.",
      "assertions": [
        "The workflow explicitly selects `program.md` authoring or an equivalent mode.",
        "The response includes sections such as current baseline, directions to explore, tried-already notes, or constraints.",
        "The response does not drift into prompt / app observability tooling as the main answer."
      ]
    },
    {
      "id": 3,
      "prompt": "My smaller-GPU autoresearch run keeps crashing, and I want to change the evaluator halfway through to get something working.",
      "expected_output": "The skill keeps the evaluator immutable inside the session, recommends constrained-hardware adaptation, and treats any evaluator change as a new comparison track.",
      "assertions": [
        "The skill warns against modifying `prepare.py`, `TIME_BUDGET`, or the active evaluator mid-session.",
        "The response recommends hardware-aware adaptation such as lowering `MAX_SEQ_LEN`, adjusting `EVAL_TOKENS` before the session, or using the hardware reference guidance.",
        "The response frames evaluator changes as a new comparison track rather than part of the current run."
      ]
    },
    {
      "id": 4,
      "prompt": "Can autoresearch help me improve this SKILL.md with binary evals and keep-or-revert scoring?",
      "expected_output": "The skill routes the request away from ML search toward `skill-autoresearch` or adjacent eval tooling instead of pretending this training-loop skill is the right fit.",
      "assertions": [
        "The skill explicitly says this is not the right tool for repo-local `SKILL.md` optimization.",
        "The response names `skill-autoresearch` as the preferred route-out.",
        "The response preserves the ML-specific `program.md` / `train.py` / `val_bpb` boundary."
      ]
    }
  ]
}

#!/usr/bin/env bash
# check-hardware.sh — Verify GPU setup for autoresearch
# Checks NVIDIA GPU, CUDA, VRAM, Python, and uv availability
#
# Usage:
#   bash scripts/check-hardware.sh
#
# Output:
#   JSON to stdout with fields: gpu_name, vram_mb, cuda_version, python_ok, uv_ok, ready
#   Human-readable summary to stderr
#
# Exit codes:
#   0  All required checks pass
#   1  Missing required components (no GPU, no CUDA, no uv)

set -uo pipefail

log()  { echo "[check-hardware] $*" >&2; }
warn() { echo "[check-hardware] WARN:  $*" >&2; }
err()  { echo "[check-hardware] ERROR: $*" >&2; }

GPU_NAME="none"
VRAM_MB=0
CUDA_VERSION="none"
PYTHON_OK=0
UV_OK=0
READY=1

# ── NVIDIA GPU ────────────────────────────────────────────────────────────────
if command -v nvidia-smi &>/dev/null; then
  GPU_NAME=$(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null | head -1 | xargs || echo "unknown")
  VRAM_MB=$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1 | xargs || echo "0")
  CUDA_VERSION=$(nvidia-smi | grep "CUDA Version" | awk '{print $NF}' | head -1 || echo "unknown")
  log "GPU: ${GPU_NAME} | VRAM: ${VRAM_MB} MB | CUDA: ${CUDA_VERSION}"
else
  warn "nvidia-smi not found — no NVIDIA GPU detected"
  READY=0
fi

# VRAM check: autoresearch default requires ~38-40GB for MAX_SEQ_LEN=2048
if [[ "${VRAM_MB}" -gt 0 ]]; then
  if [[ "${VRAM_MB}" -ge 39000 ]]; then
    log "VRAM: ${VRAM_MB} MB — sufficient for default config (MAX_SEQ_LEN=2048)"
  elif [[ "${VRAM_MB}" -ge 20000 ]]; then
    warn "VRAM: ${VRAM_MB} MB — reduce MAX_SEQ_LEN to 512 or lower in prepare.py"
  elif [[ "${VRAM_MB}" -ge 6000 ]]; then
    warn "VRAM: ${VRAM_MB} MB — low-VRAM mode: set MAX_SEQ_LEN=256 in prepare.py"
  else
    err "VRAM: ${VRAM_MB} MB — insufficient (minimum ~6GB required)"
    READY=0
  fi
fi

# ── Python ────────────────────────────────────────────────────────────────────
if command -v python3 &>/dev/null; then
  PY_VER=$(python3 --version 2>&1 | awk '{print $2}')
  log "Python: ${PY_VER}"
  PYTHON_OK=1
else
  warn "python3 not found"
fi

# ── uv ────────────────────────────────────────────────────────────────────────
if command -v uv &>/dev/null; then
  UV_VER=$(uv --version 2>&1 | head -1)
  log "uv: ${UV_VER}"
  UV_OK=1
else
  warn "uv not found — run: curl -LsSf https://astral.sh/uv/install.sh | sh"
  READY=0
fi

# ── PyTorch CUDA check (if in autoresearch repo) ──────────────────────────────
if [[ -f "train.py" ]] && command -v uv &>/dev/null; then
  TORCH_CUDA=$(uv run python3 -c "import torch; print(torch.cuda.is_available())" 2>/dev/null || echo "unknown")
  log "torch.cuda.is_available(): ${TORCH_CUDA}"
  if [[ "${TORCH_CUDA}" == "False" ]]; then
    warn "PyTorch cannot see CUDA. Check CUDA drivers and pytorch install."
    READY=0
  fi
fi

# ── Structured JSON output (stdout) ──────────────────────────────────────────
cat <<JSON
{
  "gpu_name": "${GPU_NAME}",
  "vram_mb": ${VRAM_MB},
  "cuda_version": "${CUDA_VERSION}",
  "python_ok": ${PYTHON_OK},
  "uv_ok": ${UV_OK},
  "ready": ${READY}
}
JSON

# ── Exit with appropriate code ────────────────────────────────────────────────
if [[ "${READY}" -eq 1 ]]; then
  log "All checks passed — ready for autoresearch"
else
  err "One or more checks failed — fix the issues above before running experiments"
  exit 1
fi

#!/usr/bin/env bash
# run-experiment.sh — Run a single 5-minute autoresearch experiment
# Executes uv run train.py, captures output, extracts val_bpb and peak_vram_mb
#
# Usage:
#   bash scripts/run-experiment.sh [--repo <path>] [--log <logfile>]
#
# Options:
#   --repo <path>   Path to autoresearch repo (default: .)
#   --log  <file>   Log file path (default: run.log in repo root)
#
# Output (stdout, tab-separated):
#   val_bpb  peak_vram_mb  duration_s  log_path
# Exit codes:
#   0  Experiment completed, metrics extracted
#   1  Missing val_bpb in output (likely crash/OOM)
#   2  Experiment timed out (> 360s)

set -uo pipefail

REPO_DIR="."
LOG_FILE=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --repo) REPO_DIR="$2"; shift 2 ;;
    --log)  LOG_FILE="$2";  shift 2 ;;
    --help)
      sed -n '2,16p' "$0" | sed 's/^# //'
      exit 0
      ;;
    *) echo "Unknown option: $1" >&2; exit 1 ;;
  esac
done

cd "${REPO_DIR}"

LOG_FILE="${LOG_FILE:-run.log}"
TIMEOUT=360   # 60s grace over the 300s TIME_BUDGET

log()  { echo "[run-experiment] $*" >&2; }
err()  { echo "[run-experiment] ERROR: $*" >&2; }

# ── Sanity checks ────────────────────────────────────────────────────────────
if [[ ! -f "train.py" ]]; then
  err "train.py not found. Are you in the autoresearch repo? Use --repo <path>"
  exit 1
fi

if ! command -v uv &>/dev/null; then
  err "uv not found. Run setup.sh first."
  exit 1
fi

# ── Run experiment ───────────────────────────────────────────────────────────
log "Starting experiment (TIME_BUDGET=300s)..."
START_TS=$(date +%s)

if ! timeout "${TIMEOUT}" uv run train.py > "${LOG_FILE}" 2>&1; then
  EXIT_CODE=$?
  END_TS=$(date +%s)
  DURATION=$(( END_TS - START_TS ))

  if [[ "${EXIT_CODE}" -eq 124 ]]; then
    err "Experiment timed out after ${DURATION}s (timeout=${TIMEOUT}s)"
    exit 2
  fi

  # Non-zero but not timeout — might still have written metrics before crash
  log "train.py exited with code ${EXIT_CODE} after ${DURATION}s — checking for partial metrics..."
fi

END_TS=$(date +%s)
DURATION=$(( END_TS - START_TS ))

# ── Extract metrics ───────────────────────────────────────────────────────────
VAL_BPB=$(grep "^val_bpb:" "${LOG_FILE}" | tail -1 | awk '{print $2}')
PEAK_VRAM=$(grep "^peak_vram_mb:" "${LOG_FILE}" | tail -1 | awk '{print $2}')

if [[ -z "${VAL_BPB}" ]]; then
  err "val_bpb not found in ${LOG_FILE} — experiment likely crashed (OOM or syntax error)"
  err "Last 10 lines of log:"
  tail -10 "${LOG_FILE}" >&2
  exit 1
fi

PEAK_VRAM="${PEAK_VRAM:-N/A}"

log "Experiment complete in ${DURATION}s"
log "val_bpb=${VAL_BPB}  peak_vram_mb=${PEAK_VRAM}"

# ── Structured output (stdout) ────────────────────────────────────────────────
printf "%s\t%s\t%s\t%s\n" "${VAL_BPB}" "${PEAK_VRAM}" "${DURATION}" "$(realpath "${LOG_FILE}")"

#!/usr/bin/env bash
# run-loop.sh — Autonomous autoresearch experiment loop
# Runs experiments, evaluates val_bpb, keeps improvements, reverts failures.
# Appends every run (keep/discard/crash) to results.tsv.
#
# Usage:
#   bash scripts/run-loop.sh [--repo <path>] [--max <N>] [--desc "<text>"]
#
# Options:
#   --repo  <path>   Path to autoresearch repo (default: .)
#   --max   <N>      Max experiments to run (default: 20, 0 = unlimited)
#   --desc  <text>   Experiment description prefix for results.tsv
#
# The loop:
#   1. Run train.py for 300 seconds
#   2. Extract val_bpb from output
#   3. Compare against current best
#   4. Keep commit if improved, git-reset if not
#   5. Append to results.tsv
#   6. Repeat

set -uo pipefail

REPO_DIR="."
MAX_EXPERIMENTS=20
DESC_PREFIX="auto"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --repo)  REPO_DIR="$2";       shift 2 ;;
    --max)   MAX_EXPERIMENTS="$2"; shift 2 ;;
    --desc)  DESC_PREFIX="$2";     shift 2 ;;
    --help)
      sed -n '2,17p' "$0" | sed 's/^# //'
      exit 0
      ;;
    *) echo "Unknown option: $1" >&2; exit 1 ;;
  esac
done

cd "${REPO_DIR}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

log()  { echo "[run-loop] $*"; }
err()  { echo "[run-loop] ERROR: $*" >&2; }
warn() { echo "[run-loop] WARN:  $*" >&2; }

# ── Sanity checks ─────────────────────────────────────────────────────────────
for f in train.py results.tsv; do
  if [[ ! -f "${f}" ]]; then
    if [[ "${f}" == "results.tsv" ]]; then
      log "Creating results.tsv..."
      printf "commit\tval_bpb\tpeak_vram_mb\tstatus\tdescription\n" > results.tsv
    else
      err "${f} not found. Are you in the autoresearch repo? Use --repo <path>"
      exit 1
    fi
  fi
done

if ! command -v uv &>/dev/null; then
  err "uv not found. Run setup.sh first."
  exit 1
fi

# ── Baseline ──────────────────────────────────────────────────────────────────
# Read best val_bpb from results.tsv (kept experiments only)
get_best_bpb() {
  awk -F'\t' 'NR>1 && $4=="keep" {print $2}' results.tsv \
    | sort -n | head -1
}

BEST_BPB=$(get_best_bpb)
if [[ -z "${BEST_BPB}" ]]; then
  log "No kept experiments found in results.tsv — running baseline first..."
  BEST_BPB="99.0"  # sentinel: accept first experiment unconditionally
fi
log "Current best val_bpb: ${BEST_BPB}"

# ── Loop ──────────────────────────────────────────────────────────────────────
EXPERIMENT=0

while true; do
  EXPERIMENT=$(( EXPERIMENT + 1 ))
  if [[ "${MAX_EXPERIMENTS}" -gt 0 && "${EXPERIMENT}" -gt "${MAX_EXPERIMENTS}" ]]; then
    log "Reached max experiments (${MAX_EXPERIMENTS}). Stopping."
    break
  fi

  echo ""
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
  log "Experiment ${EXPERIMENT}/${MAX_EXPERIMENTS} | best_so_far=${BEST_BPB}"
  echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

  LOG_FILE="run_${EXPERIMENT}.log"

  # ── Run experiment ────────────────────────────────────────────────────────
  RESULT=$(bash "${SCRIPT_DIR}/run-experiment.sh" \
    --repo "." \
    --log "${LOG_FILE}" 2>/dev/null) || RUN_EXIT=$?

  RUN_EXIT="${RUN_EXIT:-0}"

  # ── Handle crash / timeout ────────────────────────────────────────────────
  if [[ "${RUN_EXIT}" -ne 0 ]]; then
    COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")
    STATUS="crash"
    BPB="crash"
    VRAM="N/A"
    DESCRIPTION="${DESC_PREFIX}: experiment ${EXPERIMENT} — crashed (exit ${RUN_EXIT})"
    warn "Experiment crashed (exit ${RUN_EXIT}). Reverting commit..."
    git reset HEAD~1 2>/dev/null || true
  else
    BPB=$(echo "${RESULT}"   | cut -f1)
    VRAM=$(echo "${RESULT}"  | cut -f2)

    # ── Compare and decide ────────────────────────────────────────────────
    COMMIT=$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")

    if awk "BEGIN { exit !(${BPB} < ${BEST_BPB}) }"; then
      STATUS="keep"
      IMPROVEMENT=$(awk "BEGIN { printf \"%.4f\", ${BEST_BPB} - ${BPB} }")
      BEST_BPB="${BPB}"
      DESCRIPTION="${DESC_PREFIX}: experiment ${EXPERIMENT} — improved by ${IMPROVEMENT}"
      log "IMPROVED  val_bpb=${BPB}  (delta=-${IMPROVEMENT})  KEEPING commit ${COMMIT}"
    else
      STATUS="discard"
      DESCRIPTION="${DESC_PREFIX}: experiment ${EXPERIMENT} — no improvement (val_bpb=${BPB} >= ${BEST_BPB})"
      log "NO IMPROVEMENT  val_bpb=${BPB} >= best=${BEST_BPB}  REVERTING..."
      git reset HEAD~1 2>/dev/null || warn "git reset failed — may be at initial commit"
    fi
  fi

  # ── Append to results.tsv ─────────────────────────────────────────────────
  printf "%s\t%s\t%s\t%s\t%s\n" \
    "${COMMIT}" "${BPB}" "${VRAM}" "${STATUS}" "${DESCRIPTION}" \
    >> results.tsv

  log "Logged: ${STATUS} | ${DESCRIPTION}"
done

# ── Summary ───────────────────────────────────────────────────────────────────
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  Run loop complete"
KEPT=$(awk -F'\t' 'NR>1 && $4=="keep" {c++} END {print c+0}' results.tsv)
TOTAL=$(awk -F'\t' 'NR>1 {c++} END {print c+0}' results.tsv)
echo "  Experiments run  : ${EXPERIMENT}"
echo "  Improvements kept: ${KEPT} / ${TOTAL} total"
echo "  Best val_bpb     : ${BEST_BPB}"
echo "  Results          : results.tsv"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

#!/usr/bin/env bash
# setup.sh — One-time autoresearch environment setup
# Installs uv, clones karpathy/autoresearch, syncs dependencies, prepares dataset
#
# Usage:
#   bash setup.sh [--dir <target-dir>] [--skip-data] [--seq-len <N>]
#
# Options:
#   --dir <path>    Clone into this directory (default: ./autoresearch)
#   --skip-data     Skip uv run prepare.py (dataset download, ~2 min)
#   --seq-len <N>   Override MAX_SEQ_LEN in prepare.py before running it
#                   (useful for low-VRAM GPUs: try 256 or 512)

set -euo pipefail

TARGET_DIR="./autoresearch"
SKIP_DATA=0
SEQ_LEN=""

# ── Argument parsing ────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    --dir)     TARGET_DIR="$2"; shift 2 ;;
    --skip-data) SKIP_DATA=1; shift ;;
    --seq-len) SEQ_LEN="$2"; shift 2 ;;
    --help)
      sed -n '2,12p' "$0" | sed 's/^# //'
      exit 0
      ;;
    *) echo "Unknown option: $1" >&2; exit 1 ;;
  esac
done

log()  { echo "[setup] $*"; }
err()  { echo "[setup] ERROR: $*" >&2; }
warn() { echo "[setup] WARN:  $*" >&2; }

# ── Step 1: Install uv if missing ───────────────────────────────────────────
if ! command -v uv &>/dev/null; then
  log "Installing uv..."
  curl -LsSf https://astral.sh/uv/install.sh | sh
  export PATH="${HOME}/.local/bin:${PATH}"
  if ! command -v uv &>/dev/null; then
    err "uv install succeeded but binary not found. Restart shell and retry."
    exit 1
  fi
  log "uv installed: $(uv --version)"
else
  log "uv already installed: $(uv --version)"
fi

# ── Step 2: Clone repo ──────────────────────────────────────────────────────
if [[ -d "${TARGET_DIR}/.git" ]]; then
  log "Repository already exists at ${TARGET_DIR} — skipping clone"
else
  log "Cloning karpathy/autoresearch into ${TARGET_DIR}..."
  git clone https://github.com/karpathy/autoresearch "${TARGET_DIR}"
fi

cd "${TARGET_DIR}"

# ── Step 3: Sync dependencies ───────────────────────────────────────────────
log "Syncing dependencies (uv sync)..."
uv sync
log "Dependencies installed"

# ── Step 4: Optional MAX_SEQ_LEN override ───────────────────────────────────
if [[ -n "${SEQ_LEN}" ]]; then
  warn "Overriding MAX_SEQ_LEN → ${SEQ_LEN} in prepare.py"
  # Backup original
  cp prepare.py prepare.py.bak
  sed -i "s/MAX_SEQ_LEN\s*=\s*[0-9]*/MAX_SEQ_LEN = ${SEQ_LEN}/" prepare.py
  log "MAX_SEQ_LEN set to ${SEQ_LEN} (backup: prepare.py.bak)"
fi

# ── Step 5: Prepare dataset ─────────────────────────────────────────────────
if [[ "${SKIP_DATA}" -eq 1 ]]; then
  warn "Skipping data preparation (--skip-data). Run 'uv run prepare.py' manually before training."
else
  log "Preparing dataset (FineWeb-Edu shards + BPE tokenizer). This takes ~2 minutes..."
  uv run prepare.py
  log "Dataset ready"
fi

# ── Done ────────────────────────────────────────────────────────────────────
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  autoresearch setup complete"
echo "  Directory : ${TARGET_DIR}"
echo "  Next step : bash scripts/run-experiment.sh"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

#!/usr/bin/env bash
# show-results.sh — Parse and display autoresearch results.tsv
# Shows statistics, best experiments, and improvement history.
#
# Usage:
#   bash scripts/show-results.sh [--repo <path>] [--top <N>] [--kept-only]
#
# Options:
#   --repo      <path>  Path to autoresearch repo (default: .)
#   --top       <N>     Show top N experiments by val_bpb (default: 10)
#   --kept-only         Show only experiments that were kept

set -uo pipefail

REPO_DIR="."
TOP_N=10
KEPT_ONLY=0

while [[ $# -gt 0 ]]; do
  case "$1" in
    --repo)      REPO_DIR="$2"; shift 2 ;;
    --top)       TOP_N="$2";    shift 2 ;;
    --kept-only) KEPT_ONLY=1;   shift ;;
    --help)
      sed -n '2,13p' "$0" | sed 's/^# //'
      exit 0
      ;;
    *) echo "Unknown option: $1" >&2; exit 1 ;;
  esac
done

cd "${REPO_DIR}"

if [[ ! -f "results.tsv" ]]; then
  echo "results.tsv not found. Have you run any experiments yet?" >&2
  exit 1
fi

# ── Overall statistics ────────────────────────────────────────────────────────
TOTAL=$(awk  -F'\t' 'NR>1 {c++} END {print c+0}' results.tsv)
KEPT=$(awk   -F'\t' 'NR>1 && $4=="keep"    {c++} END {print c+0}' results.tsv)
DISCARD=$(awk -F'\t' 'NR>1 && $4=="discard" {c++} END {print c+0}' results.tsv)
CRASHED=$(awk -F'\t' 'NR>1 && $4=="crash"   {c++} END {print c+0}' results.tsv)

BEST_BPB=$(awk -F'\t' 'NR>1 && $4=="keep" && $2~/^[0-9]/ {print $2}' results.tsv \
  | sort -n | head -1)
FIRST_BPB=$(awk -F'\t' 'NR>1 && $2~/^[0-9]/ {print $2; exit}' results.tsv)

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  autoresearch — Results Summary"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
printf "  Total experiments : %s\n" "${TOTAL}"
printf "  Kept (improved)   : %s\n" "${KEPT}"
printf "  Discarded         : %s\n" "${DISCARD}"
printf "  Crashed           : %s\n" "${CRASHED}"
echo ""
if [[ -n "${FIRST_BPB}" && -n "${BEST_BPB}" ]]; then
  DELTA=$(awk "BEGIN { printf \"%.4f\", ${FIRST_BPB} - ${BEST_BPB} }")
  printf "  Starting val_bpb  : %s\n" "${FIRST_BPB}"
  printf "  Best val_bpb      : %s  (delta: -%s)\n" "${BEST_BPB}" "${DELTA}"
fi
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

# ── Top experiments ───────────────────────────────────────────────────────────
echo ""
echo "Top ${TOP_N} experiments by val_bpb:"
echo ""
printf "  %-10s  %-8s  %-14s  %-8s  %s\n" "COMMIT" "VAL_BPB" "PEAK_VRAM_MB" "STATUS" "DESCRIPTION"
echo "  ─────────────────────────────────────────────────────────────────"

AWK_FILTER='NR>1 && $2~/^[0-9]/'
if [[ "${KEPT_ONLY}" -eq 1 ]]; then
  AWK_FILTER='NR>1 && $4=="keep" && $2~/^[0-9]/'
fi

awk -F'\t' "${AWK_FILTER} {print}" results.tsv \
  | sort -t$'\t' -k2 -n \
  | head -"${TOP_N}" \
  | while IFS=$'\t' read -r commit bpb vram status desc; do
      # Truncate description to 40 chars
      short_desc="${desc:0:40}"
      [[ "${#desc}" -gt 40 ]] && short_desc="${short_desc}..."
      printf "  %-10s  %-8s  %-14s  %-8s  %s\n" \
        "${commit}" "${bpb}" "${vram}" "${status}" "${short_desc}"
    done

# ── Improvement timeline ──────────────────────────────────────────────────────
echo ""
echo "Improvement timeline (kept only):"
echo ""
printf "  %-10s  %-8s  %-14s  %s\n" "COMMIT" "VAL_BPB" "DELTA" "DESCRIPTION"
echo "  ─────────────────────────────────────────────────────────────────"

PREV_BPB=""
awk -F'\t' 'NR>1 && $4=="keep" && $2~/^[0-9]/ {print}' results.tsv \
  | while IFS=$'\t' read -r commit bpb vram status desc; do
      if [[ -z "${PREV_BPB}" ]]; then
        delta="baseline"
      else
        delta=$(awk "BEGIN { printf \"%.4f\", ${PREV_BPB} - ${bpb} }")
        delta="-${delta}"
      fi
      PREV_BPB="${bpb}"
      short_desc="${desc:0:35}"
      [[ "${#desc}" -gt 35 ]] && short_desc="${short_desc}..."
      printf "  %-10s  %-8s  %-14s  %s\n" \
        "${commit}" "${bpb}" "${delta}" "${short_desc}"
    done

echo ""

N:autoresearch
D:Run Karpathy-style autonomous ML search on a real training repo: choose the right mode (setup, program.md, bounded loop, results interpretation, or constrained-hardware adaptation), preserve the immutable prepare.py / 300-second / val_bpb contract, and route prompt/skill eval work away to LangSmith, Promptfoo, Braintrust, or skill-autoresearch.
G:autoresearch ml-experiments autonomous-research karpathy gpu train val-bpb overnight ratcheting

U[5]:
  Set up karpathy/autoresearch on a real GPU machine and verify the first baseline run
  Write or refine a program.md charter with baseline, priorities, tried-already notes, and constraints
  Run a bounded overnight train.py search loop with keep/revert ratcheting
  Interpret results.tsv to identify real gains, repeated crashes, and next experiment families
  Adapt the workflow to constrained VRAM without mutating the evaluator mid-session

S[5]{n,action,details}:
  1,ChooseMode,Pick exactly one mode first: setup readiness, program.md authoring, bounded run loop, results interpretation, or constrained-hardware adaptation
  2,FreezeHarness,Restate the immutable contract: human-authored program.md, train.py mutable, prepare.py read-only, TIME_BUDGET=300, val_bpb keep/revert, results.tsv append-only
  3,ExecuteMode,Use the smallest relevant script or reference: check-hardware/setup, program-md-guide, run-loop, show-results, or hardware-config
  4,RouteOut,Send prompt/app eval and repo-local SKILL.md optimization to LangSmith/Promptfoo/Braintrust or skill-autoresearch instead of stretching this skill
  5,Summarize,Return an Autoresearch Run Brief with chosen mode, immutable rules, next command/file, and any route-outs

R[5]:
  Never modify prepare.py during an active session; start a new comparison track if the evaluator must change
  Never change TIME_BUDGET=300 inside the session
  Never treat prompt or SKILL.md optimization as part of this ML lane
  Prefer one meaningful train.py experiment at a time; clean ablations beat bundled rewrites
  Keep results.tsv append-only because discarded runs still matter

E[3]{desc,in,out}:
  "First 40GB GPU session","Help me run Karpathy autoresearch on a 40GB GPU","Chooses setup-readiness mode, verifies hardware/deps, runs one baseline, then points to program.md authoring"
  "Weak program.md","My repo runs but my program.md just says try improvements","Chooses program.md authoring mode and rewrites the charter with baseline, priorities, and constraints"
  "Wrong lane","Can autoresearch help me improve this SKILL.md with binary evals?","Routes immediately to skill-autoresearch and explains the ML-specific boundary"