Test Skill

{
  "$schema": "https://json.schemastore.org/claude-code-marketplace.json",
  "name": "auto-empirical-research-skills",
  "version": "1.1.0",
  "description": "First-party, install-ready Claude Code plugins from the Auto-Empirical-Research-Skills collection (Stanford REAP x CoPaper.AI). Add this marketplace to one-command-install the empirical-research skill stacks (AER manuscript pipeline + explicit Python / Stata / R analysis pipelines); the wider catalog of 50+ third-party skills is browsable in the repo and consumed via copy-into-.claude/skills or --plugin-dir.",
  "owner": {
    "name": "Bryce Wang",
    "email": "brycew6m@stanford.edu",
    "url": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills"
  },
  "plugins": [
    {
      "name": "aer-skills",
      "source": "./skills/50-brycewang-aer-skills",
      "version": "1.0.0",
      "description": "Nine-skill stack for top-5 economics manuscripts (AER / AER: Insights / AEJ): topic selection, modern causal identification (DiD / IV / RDD / SCM / Bartik), referee-anticipating robustness, Keith-Head-style introductions, AER booktabs tables, AEA Data and Code Availability deposits (openICPSR-ready), submission preflight, and R&R rebuttal letters. Ships Stata / R / Python templates and classic-AER-paper exemplars.",
      "author": { "name": "Bryce Wang", "email": "brycew6m@stanford.edu" },
      "license": "MIT",
      "homepage": "https://github.com/brycewang-stanford/AER-skills",
      "repository": "https://github.com/brycewang-stanford/AER-skills",
      "category": "academic",
      "keywords": [
        "economics", "aer", "american-economic-review", "aej", "causal-inference",
        "difference-in-differences", "instrumental-variables", "regression-discontinuity",
        "synthetic-control", "replication", "academic-writing", "empirical-research"
      ]
    },
    {
      "name": "empirical-analysis-python",
      "source": "./plugins/empirical-analysis-python",
      "version": "1.0.0",
      "description": "Explicit 8-step empirical-analysis pipeline in the traditional Python econometrics stack (pandas + statsmodels + linearmodels + pyfixest + rdrobust + econml + causalml): cleaning -> variable construction -> Table 1 -> diagnostics -> estimation (OLS/IV/DID/RDD/PSM/SCM/DML/Causal Forest) -> robustness -> mechanism/heterogeneity/mediation -> publication-ready tables & figures. Includes epidemiology and ML-causal modes.",
      "author": { "name": "Bryce Wang", "email": "brycew6m@stanford.edu" },
      "license": "CC-BY-SA-4.0",
      "homepage": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "repository": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "category": "academic",
      "keywords": [
        "econometrics", "causal-inference", "python", "difference-in-differences",
        "instrumental-variables", "regression-discontinuity", "double-machine-learning",
        "empirical-research", "replication"
      ]
    },
    {
      "name": "empirical-analysis-stata",
      "source": "./plugins/empirical-analysis-stata",
      "version": "1.0.0",
      "description": "Explicit 8-step empirical-analysis pipeline in the traditional Stata ecosystem (reghdfe + ivreg2 + csdid + did_imputation + sdid + rdrobust + synth + psmatch2 + teffects + esttab + coefplot): from use/import to .tex/.rtf tables and .pdf figures, with the full modern DID toolkit (bacondecomp + honestdid + rwolf + ritest). Includes epidemiology and ML-causal modes. The first choice for referee-level Stata replication packs.",
      "author": { "name": "Bryce Wang", "email": "brycew6m@stanford.edu" },
      "license": "CC-BY-SA-4.0",
      "homepage": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "repository": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "category": "academic",
      "keywords": [
        "econometrics", "causal-inference", "stata", "reghdfe", "difference-in-differences",
        "instrumental-variables", "regression-discontinuity", "synthetic-control",
        "empirical-research", "replication"
      ]
    },
    {
      "name": "empirical-analysis-r",
      "source": "./plugins/empirical-analysis-r",
      "version": "1.0.0",
      "description": "Explicit 8-step empirical-analysis pipeline in the modern tidyverse + econometrics R stack (dplyr + haven + fixest + did + synthdid + rdrobust + MatchIt + WeightIt + grf + DoubleML + mediation + marginaleffects + modelsummary). All eight steps fit in a single .qmd; `quarto render` produces a unified PDF/HTML/Word reproducibility report. Includes epidemiology and ML-causal modes.",
      "author": { "name": "Bryce Wang", "email": "brycew6m@stanford.edu" },
      "license": "CC-BY-SA-4.0",
      "homepage": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "repository": "https://github.com/brycewang-stanford/Auto-Empirical-Research-Skills",
      "category": "academic",
      "keywords": [
        "econometrics", "causal-inference", "r", "tidyverse", "fixest",
        "difference-in-differences", "regression-discontinuity", "quarto",
        "empirical-research", "reproducible-research"
      ]
    }
  ]
}

version: 2
updates:
  - package-ecosystem: "github-actions"
    directory: "/"
    schedule:
      interval: "weekly"
      day: "monday"
      time: "09:00"
      timezone: "Etc/UTC"
    open-pull-requests-limit: 5
    commit-message:
      prefix: "ci"
      include: "scope"
    groups:
      github-actions:
        patterns:
          - "*"

name: Bug report
description: Report a broken link, stale entry, bad install command, or repository quality issue.
title: "[Bug]: "
labels:
  - bug
body:
  - type: textarea
    id: problem
    attributes:
      label: What is wrong?
      description: Include the file path, link, or command that failed.
      placeholder: "docs/05-统计分析与因果推断.md links to ..."
    validations:
      required: true
  - type: textarea
    id: expected
    attributes:
      label: Expected behavior
      description: What should the entry, link, or command do instead?
    validations:
      required: true
  - type: input
    id: environment
    attributes:
      label: Environment
      description: Optional. Agent, OS, Python/R/Stata version, or browser.
      placeholder: "Claude Code, macOS, Python 3.11"

blank_issues_enabled: true
contact_links:
  - name: CoPaper.AI
    url: https://copaper.ai
    about: Try the end-to-end empirical research assistant.

name: Skill submission
description: Suggest an open-source, self-contained skill for the catalog.
title: "[Skill]: "
labels:
  - skills
  - triage
body:
  - type: input
    id: source
    attributes:
      label: Source repository
      description: Link to the upstream repository or folder.
      placeholder: "https://github.com/owner/repo/tree/main/skills/example"
    validations:
      required: true
  - type: input
    id: license
    attributes:
      label: License
      description: Name the upstream license and note any restrictions.
      placeholder: "MIT"
    validations:
      required: true
  - type: textarea
    id: function
    attributes:
      label: What does it do?
      description: One concise paragraph explaining the workflow and research stage.
    validations:
      required: true
  - type: dropdown
    id: category
    attributes:
      label: Best category
      options:
        - "01 - Topic Selection & Research Design"
        - "02 - Literature Search & Review"
        - "03 - Paper Reading & Analysis"
        - "04 - Data Collection & Cleaning"
        - "05 - Statistical Analysis & Causal Inference"
        - "06 - Paper Writing"
        - "07 - Paper Revision & Polishing"
        - "08 - Citation Management & Typesetting"
        - "09 - Replication & Reproducible Research"
        - "10 - Peer Review Response & Defense"
    validations:
      required: true
  - type: checkboxes
    id: scope
    attributes:
      label: Scope checks
      options:
        - label: The core skill is open source and independently runnable.
          required: true
        - label: The core workflow does not require a paid/proprietary API or account.
          required: true
        - label: I checked for obvious malicious code, credential exfiltration, and prompt injection.
          required: true

name: Check external links

on:
  schedule:
    - cron: "0 7 * * 1"
  workflow_dispatch: {}

permissions:
  contents: read

jobs:
  links:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Check maintained-doc external links
        run: make external-links

      - name: Upload link-check report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: external-link-check
          path: catalog/external-link-check.json

name: Check tools links

# Network-bound drift guard for tools/tools.json. Pings each tool's primary URL
# (and homepage, as a non-fatal warning) so a maintainer pass can re-verify or
# retire entries whose upstream moved. Not part of the blocking validate gate.

on:
  schedule:
    - cron: "0 8 1 * *"  # 08:00 UTC on the 1st of each month
  workflow_dispatch: {}

permissions:
  contents: read

jobs:
  tools-links:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Check tools-catalog links
        run: make tools-links

      - name: Upload tools link-check report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: tools-link-check
          path: catalog/tools-link-check.json

name: Quality (evals + benchmark + tests)

# Complements validate-catalog.yml (which checks catalog freshness). This job
# enforces the eval/benchmark/test layer. All steps are stdlib-only; no pip.
on:
  push:
    branches: [main]
  pull_request:
  workflow_dispatch:

permissions:
  contents: read

jobs:
  quality:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.9", "3.12"]
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
          submodules: recursive
      - uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Catalog enrichment freshness
        run: python3 scripts/build-catalog-enrich.py --check

      - name: Python tooling compatibility
        run: make python-compat

      - name: Lint eval-harness scenarios
        run: >-
          python3 eval-harness/run_evals.py
          --min-scenarios 17
          --min-auto-checks 80
          --expect-categories causal-identification,reproducibility,citation-hygiene,runtime-safety,research-integrity,writing-compliance,writing-style

      - name: Unit tests (stdlib unittest)
        run: python3 -m unittest discover -s tests -p "test_*.py"

      - name: Benchmark (reference pipeline + strict grading)
        run: |
          python3 benchmark/check_benchmark.py --lint
          python3 benchmark/reference_pipeline.py --check
          python3 benchmark/check_benchmark.py --strict --fail-on-partial --fail-on-orphan-results

      - name: Grade example candidates (smoke)
        run: >-
          python3 eval-harness/run_evals.py
          --grade eval-harness/candidates/_example
          --expect-graded 9
          --expect-fail-required statspai-weak-iv
          --expect-graded-categories causal-identification,reproducibility,citation-hygiene,runtime-safety,research-integrity
          --fail-on-orphans
          --fail-on-partial
          --no-write

name: OpenSSF Scorecard

on:
  push:
    branches:
      - main
  schedule:
    - cron: "0 8 * * 1"
  workflow_dispatch: {}

permissions:
  contents: read

jobs:
  scorecard:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read
      security-events: write
      id-token: write
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Run OpenSSF Scorecard
        uses: ossf/scorecard-action@v2.4.3
        with:
          results_file: scorecard-results.sarif
          results_format: sarif
          publish_results: true

      - name: Upload SARIF
        uses: github/codeql-action/upload-sarif@v4
        with:
          sarif_file: scorecard-results.sarif

name: Sync AER-skills from upstream

# Weekly mirror of brycewang-stanford/AER-skills (whole repo, excluding .git
# and .gitignore) into skills/50-brycewang-aer-skills/. Opens a PR on drift;
# does nothing on parity. No untrusted user input (issue/PR/commit fields) is
# interpolated into any run step, so command-injection attack surface is zero.

on:
  schedule:
    - cron: "0 6 * * 1"
  workflow_dispatch: {}

permissions:
  contents: write
  pull-requests: write

jobs:
  sync:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
          submodules: recursive

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Run sync script
        id: sync
        run: |
          set +e
          bash scripts/sync-aer-skills.sh
          rc=$?
          set -e
          if [ "$rc" -eq 0 ]; then
            echo "drift=false" >> "$GITHUB_OUTPUT"
          elif [ "$rc" -eq 1 ]; then
            echo "drift=true" >> "$GITHUB_OUTPUT"
          else
            echo "Sync script failed with exit code $rc" >&2
            exit "$rc"
          fi

      # Regenerate catalog/docs from the synced tree, then run the same full
      # gate a human PR must pass (`make check`), so an upstream snapshot that
      # breaks repo invariants cannot be merged unnoticed. PRs opened by
      # GITHUB_TOKEN do not trigger CI, so this in-job gate is the only
      # pre-merge signal.
      - name: Regenerate catalog and run full gate
        id: gate
        if: steps.sync.outputs.drift == 'true'
        run: |
          set +e
          make catalog && make check
          echo "status=$?" >> "$GITHUB_OUTPUT"

      - name: Open PR on drift
        if: steps.sync.outputs.drift == 'true'
        uses: peter-evans/create-pull-request@v8
        with:
          branch: chore/sync-aer-skills
          commit-message: "chore(skills/aer-skills): sync from upstream"
          title: "chore(skills/aer-skills): sync from upstream AER-skills"
          body: |
            Automated sync from
            [`brycewang-stanford/AER-skills`](https://github.com/brycewang-stanford/AER-skills).

            **Local full gate (`make catalog && make check`):** ${{ steps.gate.outputs.status == '0' && '✅ passed — review the diff and merge if intended' || '❌ FAILED — do NOT merge until fixed; see the run log (PR labelled needs-fix)' }}

            Triggered by `.github/workflows/sync-aer-skills.yml`.
          labels: ${{ steps.gate.outputs.status == '0' && 'sync,skills' || 'sync,skills,needs-fix' }}

      - name: Fail the run if the synced tree breaks the gate
        if: steps.sync.outputs.drift == 'true' && steps.gate.outputs.status != '0'
        run: |
          echo "::error::Synced upstream fails 'make check' (exit ${{ steps.gate.outputs.status }}). A PR was opened and labelled needs-fix; do not merge until the gate passes."
          exit 1

name: Sync StatsPAI skill from upstream

# Weekly mirror of brycewang-stanford/StatsPAI:StatsPAI_full_data_analysis_skill/
# into skills/00-Full-empirical-analysis-skill_StatsPAI/. Opens a PR on drift;
# does nothing on parity. No untrusted user input (issue/PR/commit fields) is
# interpolated into any run step, so command-injection attack surface is zero.

on:
  schedule:
    - cron: "0 6 * * 1"
  workflow_dispatch: {}

permissions:
  contents: write
  pull-requests: write

jobs:
  sync:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
          submodules: recursive

      - uses: actions/setup-python@v6
        with:
          python-version: "3.12"

      - name: Run sync script
        id: sync
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set +e
          bash scripts/sync-statspai-skill.sh
          rc=$?
          set -e
          if [ "$rc" -eq 0 ]; then
            echo "drift=false" >> "$GITHUB_OUTPUT"
          elif [ "$rc" -eq 1 ]; then
            echo "drift=true" >> "$GITHUB_OUTPUT"
          else
            echo "Sync script failed with exit code $rc" >&2
            exit "$rc"
          fi

      # Regenerate catalog/docs from the synced tree, then run the same full
      # gate a human PR must pass (`make check`). This is what catches an
      # upstream snapshot that silently clobbers local invariants (e.g. the
      # SkillOpt execution-gate terms enforced by tests/test_skillopt_gates.py)
      # before it can be merged. PRs opened by GITHUB_TOKEN do not trigger CI,
      # so this in-job gate is the only pre-merge signal.
      - name: Regenerate catalog and run full gate
        id: gate
        if: steps.sync.outputs.drift == 'true'
        run: |
          set +e
          make catalog && make check
          echo "status=$?" >> "$GITHUB_OUTPUT"

      - name: Open PR on drift
        if: steps.sync.outputs.drift == 'true'
        uses: peter-evans/create-pull-request@v8
        with:
          branch: chore/sync-statspai-skill
          commit-message: "chore(skills/statspai): sync from upstream"
          title: "chore(skills/statspai): sync from upstream StatsPAI_skill/"
          body: |
            Automated sync from
            [`brycewang-stanford/StatsPAI:StatsPAI_skill/`](https://github.com/brycewang-stanford/StatsPAI/tree/main/StatsPAI_skill).

            **Local full gate (`make catalog && make check`):** ${{ steps.gate.outputs.status == '0' && '✅ passed — review the diff and merge if intended' || '❌ FAILED — do NOT merge until fixed; see the run log (PR labelled needs-fix)' }}

            Triggered by `.github/workflows/sync-statspai-skill.yml`.
          labels: ${{ steps.gate.outputs.status == '0' && 'sync,skills' || 'sync,skills,needs-fix' }}

      - name: Fail the run if the synced tree breaks the gate
        if: steps.sync.outputs.drift == 'true' && steps.gate.outputs.status != '0'
        run: |
          echo "::error::Synced upstream fails 'make check' (exit ${{ steps.gate.outputs.status }}). A PR was opened and labelled needs-fix; do not merge until the gate passes."
          exit 1

name: Validate catalog and repo structure

on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch: {}

permissions:
  contents: read

jobs:
  validate:
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false
          submodules: recursive

      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Validate repository structure and generated artifacts
        run: make validate

社科实证 Agent Skills 收集整理/
/Users/brycewang/Documents/GitHub/Awesome-Agent-Skills-for-Empirical-Research/社科实证 Agent Skills 收集整理/
test-notebooks/
社媒运营/

# macOS
.DS_Store
**/.DS_Store

# Python / local tooling
__pycache__/
*.py[cod]
.pytest_cache/
.ruff_cache/
.mypy_cache/
.venv/
venv/
catalog/external-link-check.json

# Local secrets
.env
.env.*
!.env.example


stata-skills/

CLAUDE.md

# Local, network-free pre-commit hooks for AERS quality gates.
# Install once:  pip install pre-commit && pre-commit install
# Run on demand: pre-commit run --all-files
#
# All hooks are `language: system` and call the repo's own stdlib-only tools, so
# there is nothing to download and they work offline.
repos:
  - repo: local
    hooks:
      - id: aers-eval-harness-lint
        name: AERS eval-harness scenarios lint
        entry: python3 eval-harness/run_evals.py
        language: system
        pass_filenames: false
        files: ^eval-harness/
      - id: aers-unit-tests
        name: AERS unit tests
        entry: python3 -m unittest discover -s tests -p "test_*.py"
        language: system
        pass_filenames: false
        files: ^(tests/|eval-harness/|benchmark/|scripts/)
      - id: aers-python-compat
        name: AERS Python tooling compile
        entry: make python-compat
        language: system
        pass_filenames: false
        files: ^(tests/|eval-harness/|benchmark/|scripts/|Makefile)
      - id: aers-benchmark
        name: AERS benchmark (strict)
        entry: bash -c 'python3 benchmark/check_benchmark.py --lint >/dev/null && python3 benchmark/reference_pipeline.py --check >/dev/null && python3 benchmark/check_benchmark.py --strict --fail-on-partial --fail-on-orphan-results >/dev/null'
        language: system
        pass_filenames: false
        files: ^benchmark/
      - id: aers-tracked-file-hygiene
        name: AERS tracked-file hygiene
        entry: python3 scripts/check-repo-hygiene.py
        language: system
        pass_filenames: false
        files: ^(.DS_Store|.*(__pycache__|\.py[cod]|\.pytest_cache|\.ruff_cache|\.mypy_cache)|scripts/check-repo-hygiene\.py)
      - id: aers-repo-validate
        name: AERS repo validate (links + frontmatter)
        entry: python3 scripts/validate-repo.py
        language: system
        pass_filenames: false
        files: ^(skills/|docs/|README|catalog/)

interface:
  display_name: "AERS Catalog Router"
  short_description: "Route empirical-research skills from AERS"
  default_prompt: "Use $auto-empirical-research-skills to choose the right empirical-research skill for my task."

results/
candidates/_*/

{
  "task": "bad-control-recovery",
  "method": "y~d (naive) vs y~d+x (good pre-treatment control) vs y~d+x+m (bad post-treatment mediator control)",
  "n": 120,
  "true_total": 2.5,
  "naive_effect": 2.5,
  "good_control_effect": 2.5,
  "bad_control_effect": 0.5
}

{
  "task": "bayesian-recovery",
  "method": "Conjugate Normal-Normal posterior mean: weakly-informative vs overconfident miscalibrated prior",
  "n": 20,
  "data_mean": 2.0,
  "posterior_weak": 2.0,
  "posterior_strong": 5.5556
}

{
  "task": "did-staggered-recovery",
  "method": "Group-time DID with not-yet-treated controls; TWFE diagnostic reported",
  "n": 600,
  "true_att": 2.9091,
  "twfe_att": 1.4545,
  "cs_att": 2.9091
}

{
  "task": "dml-recovery",
  "method": "Cross-fitted partialling-out (DML) vs naive OLS of outcome on treatment only",
  "n": 120,
  "true_theta": 1.5,
  "dml_theta": 1.5,
  "naive_theta": 3.4284
}

{
  "task": "event-study-recovery",
  "method": "Event-study with unit+time FE and treated x relative-time dummies vs naive treated-only before/after",
  "n": 240,
  "true_att": 3.0,
  "es_att": 3.0,
  "es_pre_max": 0.0,
  "naive_before_after": 5.4
}

{
  "task": "card-iv-recovery",
  "method": "OLS vs 2SLS (nearc4 instrument), manual two-stage",
  "n": 3010,
  "ols_return": 0.0747,
  "iv_return": 0.1315,
  "first_stage_coef": 0.3199,
  "first_stage_F": 13.26
}

{
  "task": "lalonde-recovery",
  "method": "OLS regression adjustment (full controls incl. re74, re75)",
  "n_treated": 185,
  "n_control": 429,
  "naive_att": -635.0,
  "adjusted_att": 1548.2,
  "balance": {
    "age": -0.242,
    "educ": 0.045,
    "black": 1.668,
    "hispan": -0.277,
    "married": -0.719,
    "nodegree": 0.235,
    "re74": -0.596,
    "re75": -0.287
  }
}

{
  "task": "panel-fe-recovery",
  "method": "Two-way (unit+time) fixed effects vs pooled OLS that ignores unit heterogeneity",
  "n": 240,
  "true_att": 2.0,
  "twoway_fe_att": 2.0,
  "pooled_att": 7.0667
}

{
  "task": "rdd-recovery",
  "method": "Sharp RD: local-linear at the cutoff vs global common-slope OLS vs naive across-cutoff mean difference",
  "n": 101,
  "bandwidth": 0.5,
  "true_tau": 3.0,
  "naive_jump": 5.51,
  "global_att": 2.94,
  "local_att": 3.0
}

{
  "task": "survival-recovery",
  "method": "Kaplan-Meier survival (handles censoring) vs naive proportion that treats censored as failures",
  "n": 160,
  "true_surv_treat": 0.65,
  "true_surv_control": 0.3,
  "km_surv_treat": 0.651,
  "km_surv_control": 0.2986,
  "naive_surv_treat": 0.525,
  "naive_surv_control": 0.2375
}

{
  "task": "synthetic-control-recovery",
  "method": "Synthetic control (simplex-fit donor weights on the pre-period) vs equal-weight donor average",
  "n": 8,
  "true_effect": 3.0,
  "sc_effect": 3.0,
  "naive_effect": 4.8167
}

#!/usr/bin/env python3
"""Grade candidate results against AERS benchmark tasks.

A "candidate" is a results.json produced by any pipeline (the reference pipeline,
the repo's demo outputs, or a real agent run). For each task the checker
recomputes the data-derived golds from the dataset itself, so a candidate cannot
pass by reporting fabricated numbers. Literature constants (the LaLonde
experimental benchmark, the Card canonical values) come from the task specs.

Usage:
    python3 benchmark/reference_pipeline.py            # produce reference candidates
    python3 benchmark/check_benchmark.py --lint        # validate task + candidate metadata only
    python3 benchmark/check_benchmark.py               # grade all tasks
    python3 benchmark/check_benchmark.py --task card-iv-recovery
    python3 benchmark/check_benchmark.py --strict      # nonzero exit on required fail
    python3 benchmark/check_benchmark.py --strict --fail-on-partial --fail-on-orphan-results
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from numbers import Real
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "scripts"))
import toml_compat as tomllib  # noqa: E402

sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
import lalonde  # noqa: E402
import card  # noqa: E402
import simdid  # noqa: E402
import rdd  # noqa: E402
import badcontrol  # noqa: E402
import panelfe  # noqa: E402
import eventstudy  # noqa: E402
import dml  # noqa: E402
import survival  # noqa: E402
import bayesian  # noqa: E402
import synth  # noqa: E402

TASKS_DIR = Path(__file__).resolve().parent / "tasks"
CANDIDATES_DIR = Path(__file__).resolve().parent / "candidates"
RESULTS_DIR = Path(__file__).resolve().parent / "results"

SUPPORTED_TASK_IDS = {
    "bad-control-recovery",
    "bayesian-recovery",
    "card-iv-recovery",
    "did-staggered-recovery",
    "dml-recovery",
    "event-study-recovery",
    "lalonde-recovery",
    "panel-fe-recovery",
    "rdd-recovery",
    "survival-recovery",
    "synthetic-control-recovery",
}
CANDIDATE_DIR_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
KNOWN_CHECKS = {
    "adjusted_recovery",
    "biased_away",
    "closer_to_truth",
    "cross_check",
    "first_stage_min",
    "imbalance_count",
    "iv_gt_ols",
    "naive_sign",
    "near_benchmark",
    "recovers_truth",
    "value_near",
}

REQUIRED_TASK_FIELDS = ("id", "title", "data", "reference_candidate", "gold")
REQUIRED_GOLD_FIELDS = ("id", "description", "check", "required", "weight")
TASK_STRING_FIELDS = {
    "lalonde-recovery": ("treatment", "outcome"),
}
TASK_NUMERIC_FIELDS = {
    "lalonde-recovery": ("experimental_att", "experimental_tol"),
}
CHECK_REQUIRED_FIELDS = {
    "adjusted_recovery": ("min_swing",),
    "biased_away": ("field", "truth_key", "min_gap"),
    "closer_to_truth": ("near_field", "far_field", "truth_key"),
    "cross_check": (),
    "first_stage_min": ("min_f",),
    "imbalance_count": ("min_covariates", "smd_threshold"),
    "iv_gt_ols": (),
    "naive_sign": ("expected_sign",),
    "near_benchmark": (),
    "recovers_truth": ("field", "truth_key", "tol"),
    "value_near": ("field", "expected", "tol"),
}
CHECK_STRING_FIELDS = {
    "biased_away": ("field", "truth_key"),
    "closer_to_truth": ("near_field", "far_field", "truth_key"),
    "naive_sign": ("expected_sign",),
    "recovers_truth": ("field", "truth_key"),
    "value_near": ("field",),
}
CHECK_NUMERIC_FIELDS = {
    "adjusted_recovery": ("min_swing",),
    "biased_away": ("min_gap",),
    "first_stage_min": ("min_f",),
    "imbalance_count": ("smd_threshold",),
    "recovers_truth": ("tol",),
    "value_near": ("expected", "tol"),
}
CROSS_CHECK_NUMERIC_FIELDS = {
    "bad-control-recovery": ("tol",),
    "card-iv-recovery": ("tol", "f_tol"),
    "did-staggered-recovery": ("tol",),
    "lalonde-recovery": ("naive_tol", "smd_tol"),
    "rdd-recovery": ("tol",),
}
CANDIDATE_NUMERIC_FIELDS = {
    "bad-control-recovery": ("true_total", "naive_effect", "good_control_effect", "bad_control_effect"),
    "card-iv-recovery": ("ols_return", "iv_return", "first_stage_F", "first_stage_coef"),
    "did-staggered-recovery": ("true_att", "twfe_att", "cs_att"),
    "lalonde-recovery": ("naive_att", "adjusted_att"),
    "rdd-recovery": ("true_tau", "naive_jump", "global_att", "local_att"),
    "panel-fe-recovery": ("true_att", "twoway_fe_att", "pooled_att"),
    "event-study-recovery": ("true_att", "es_att", "es_pre_max", "naive_before_after"),
    "dml-recovery": ("true_theta", "dml_theta", "naive_theta"),
    "survival-recovery": (
        "true_surv_treat", "true_surv_control",
        "km_surv_treat", "km_surv_control",
        "naive_surv_treat", "naive_surv_control",
    ),
    "bayesian-recovery": ("data_mean", "posterior_weak", "posterior_strong"),
    "synthetic-control-recovery": ("true_effect", "sc_effect", "naive_effect"),
}
CANDIDATE_NUMERIC_MAP_FIELDS = {
    "lalonde-recovery": ("balance",),
}


def _is_number(value: object) -> bool:
    return isinstance(value, Real) and not isinstance(value, bool)


def validate_candidate_dir_name(value: object, label: str = "candidate") -> list[str]:
    if not isinstance(value, str) or not value:
        return [f"{label} must be a non-empty string"]
    if not CANDIDATE_DIR_RE.fullmatch(value):
        return [
            f"{label} '{value}' must be a single directory name matching {CANDIDATE_DIR_RE.pattern}"
        ]
    return []


def validate_repo_relative_file(value: object, label: str) -> list[str]:
    if not isinstance(value, str) or not value:
        return [f"{label} must be a non-empty string"]
    raw_path = Path(value)
    if raw_path.is_absolute():
        return [f"{label} '{value}' must be repo-relative, not absolute"]
    root = ROOT.resolve()
    resolved = (ROOT / raw_path).resolve(strict=False)
    try:
        resolved.relative_to(root)
    except ValueError:
        return [f"{label} '{value}' must stay inside the repository"]
    if not resolved.exists():
        return [f"{label} path does not exist: {value}"]
    if not resolved.is_file():
        return [f"{label} path must be a file: {value}"]
    return []


def orphan_result_files(results_dir: Path, task_ids: set[str]) -> list[Path]:
    """Generated result JSON files whose stem no longer matches a benchmark task."""
    if not results_dir.exists():
        return []
    return sorted(path for path in results_dir.glob("*.json") if path.stem not in task_ids)


def validate_task(task: dict, task_path: Path) -> list[str]:
    """Validate a benchmark task spec before scoring candidates."""
    problems = []
    if not isinstance(task, dict):
        return [f"{task_path} must contain a TOML table"]

    for field in REQUIRED_TASK_FIELDS:
        if field not in task:
            problems.append(f"missing top-level field '{field}'")

    task_id = task.get("id")
    if not isinstance(task_id, str) or not task_id:
        problems.append("field 'id' must be a non-empty string")
    elif task_id != task_path.stem:
        problems.append(f"field 'id' must match file stem '{task_path.stem}'")
    elif task_id not in SUPPORTED_TASK_IDS:
        problems.append(f"unsupported task id '{task_id}'")

    for field in ("title", "data", "reference_candidate"):
        value = task.get(field)
        if field in task and (not isinstance(value, str) or not value.strip()):
            problems.append(f"field '{field}' must be a non-empty string")

    if "reference_candidate" in task:
        problems.extend(validate_candidate_dir_name(task.get("reference_candidate"), "reference_candidate"))

    if isinstance(task_id, str):
        for field in TASK_STRING_FIELDS.get(task_id, ()):
            value = task.get(field)
            if not isinstance(value, str) or not value.strip():
                problems.append(f"field '{field}' must be a non-empty string")
        for field in TASK_NUMERIC_FIELDS.get(task_id, ()):
            value = task.get(field)
            if not _is_number(value):
                problems.append(f"field '{field}' must be numeric")

    if "data" in task:
        problems.extend(validate_repo_relative_file(task.get("data"), "data"))

    gold = task.get("gold")
    if not isinstance(gold, list) or not gold:
        problems.append("field 'gold' must be a non-empty list")
        return problems

    seen_gold_ids = set()
    for idx, item in enumerate(gold):
        prefix = f"gold[{idx}]"
        if not isinstance(item, dict):
            problems.append(f"{prefix} must be a table")
            continue

        for field in REQUIRED_GOLD_FIELDS:
            if field not in item:
                problems.append(f"{prefix} missing field '{field}'")

        gid = item.get("id")
        if not isinstance(gid, str) or not gid:
            problems.append(f"{prefix} field 'id' must be a non-empty string")
        elif gid in seen_gold_ids:
            problems.append(f"{prefix} duplicate id '{gid}'")
        else:
            seen_gold_ids.add(gid)

        description = item.get("description")
        if "description" in item and (not isinstance(description, str) or not description.strip()):
            problems.append(f"{prefix} field 'description' must be a non-empty string")

        required = item.get("required")
        if "required" in item and not isinstance(required, bool):
            problems.append(f"{prefix} field 'required' must be a boolean")

        weight = item.get("weight")
        if "weight" in item and (not _is_number(weight) or weight <= 0):
            problems.append(f"{prefix} field 'weight' must be a positive number")

        check = item.get("check")
        if not isinstance(check, str) or not check:
            problems.append(f"{prefix} field 'check' must be a non-empty string")
            continue
        if check not in KNOWN_CHECKS:
            problems.append(f"{prefix} unknown check '{check}'")
            continue

        for field in CHECK_REQUIRED_FIELDS[check]:
            if field not in item:
                problems.append(f"{prefix} check '{check}' missing field '{field}'")

        if check == "naive_sign" and item.get("expected_sign") not in {"negative", "positive"}:
            problems.append(f"{prefix} expected_sign must be 'negative' or 'positive'")
        if check == "imbalance_count" and not isinstance(item.get("min_covariates"), int):
            problems.append(f"{prefix} min_covariates must be an integer")

        for field in CHECK_STRING_FIELDS.get(check, ()):
            value = item.get(field)
            if field in item and (not isinstance(value, str) or not value.strip()):
                problems.append(f"{prefix} field '{field}' must be a non-empty string")
        for field in CHECK_NUMERIC_FIELDS.get(check, ()):
            value = item.get(field)
            if field in item and not _is_number(value):
                problems.append(f"{prefix} field '{field}' must be numeric")

        if check == "cross_check" and isinstance(task_id, str):
            for field in CROSS_CHECK_NUMERIC_FIELDS.get(task_id, ()):
                value = item.get(field)
                if field not in item:
                    problems.append(f"{prefix} check 'cross_check' missing field '{field}'")
                elif not _is_number(value):
                    problems.append(f"{prefix} field '{field}' must be numeric")

    return problems


def validate_candidate(task: dict, candidate: dict, candidate_path: Path) -> list[str]:
    """Validate candidate result metadata before comparing reported numbers."""
    problems = []
    if not isinstance(candidate, dict):
        return [f"{candidate_path} must contain a JSON object"]

    candidate_task = candidate.get("task")
    if not isinstance(candidate_task, str) or not candidate_task:
        problems.append("candidate field 'task' must be a non-empty string")
    elif candidate_task != task["id"]:
        problems.append(f"candidate task '{candidate_task}' does not match benchmark task '{task['id']}'")

    task_id = task.get("id")
    for field in CANDIDATE_NUMERIC_FIELDS.get(task_id, ()):
        if field in candidate and not _is_number(candidate[field]):
            problems.append(f"candidate field '{field}' must be numeric")

    for field in CANDIDATE_NUMERIC_MAP_FIELDS.get(task_id, ()):
        if field not in candidate:
            continue
        value = candidate[field]
        if not isinstance(value, dict):
            problems.append(f"candidate field '{field}' must be an object of numeric values")
            continue
        for key, item in value.items():
            if not isinstance(key, str) or not key:
                problems.append(f"candidate field '{field}' has a non-string or empty key")
            if not _is_number(item):
                problems.append(f"candidate field '{field}.{key}' must be numeric")

    return problems


def compute_truth(task: dict) -> dict:
    """Recompute data-derived golds from the dataset for the given task."""
    if task["id"] == "lalonde-recovery":
        data = ROOT / task["data"]
        rows = lalonde.load(data)
        return {
            "n": len(rows),
            "naive_att": lalonde.naive_att(rows, task["treatment"], task["outcome"]),
            "smd": lalonde.smd_table(rows, task["treatment"]),
        }
    if task["id"] == "card-iv-recovery":
        data = ROOT / task["data"]
        rows = card.load(data)
        coef, f = card.first_stage(rows)
        return {
            "n": len(rows),
            "ols_return": card.ols_return(rows),
            "iv_return": card.iv_return(rows),
            "first_stage_F": f,
            "first_stage_coef": coef,
        }
    if task["id"] == "did-staggered-recovery":
        data = ROOT / task["data"]
        rows = simdid.load(data)
        return {
            "n": len(rows),
            "true_att": simdid.true_att(rows),
            "twfe_att": simdid.twfe_att(rows),
            "cs_att": simdid.cs_att(rows),
        }
    if task["id"] == "rdd-recovery":
        data = ROOT / task["data"]
        rows = rdd.load(data)
        return {
            "n": len(rows),
            "true_tau": rdd.true_tau(rows),
            "naive_jump": rdd.naive_jump(rows),
            "global_att": rdd.global_att(rows),
            "local_att": rdd.local_att(rows),
        }
    if task["id"] == "bad-control-recovery":
        data = ROOT / task["data"]
        rows = badcontrol.load(data)
        return {
            "n": len(rows),
            "true_total": badcontrol.true_total(rows),
            "naive_effect": badcontrol.naive_effect(rows),
            "good_control_effect": badcontrol.good_control_effect(rows),
            "bad_control_effect": badcontrol.bad_control_effect(rows),
        }
    if task["id"] == "panel-fe-recovery":
        data = ROOT / task["data"]
        rows = panelfe.load(data)
        return {
            "n": len(rows),
            "true_att": panelfe.true_att(rows),
            "twoway_fe_att": panelfe.twoway_fe_att(rows),
            "pooled_att": panelfe.pooled_att(rows),
        }
    if task["id"] == "event-study-recovery":
        data = ROOT / task["data"]
        rows = eventstudy.load(data)
        return {
            "n": len(rows),
            "true_att": eventstudy.true_att(rows),
            "es_att": eventstudy.es_att(rows),
            "es_pre_max": eventstudy.es_pre_max(rows),
            "naive_before_after": eventstudy.naive_before_after(rows),
        }
    if task["id"] == "dml-recovery":
        data = ROOT / task["data"]
        rows = dml.load(data)
        return {
            "n": len(rows),
            "true_theta": dml.true_theta(rows),
            "dml_theta": dml.dml_theta(rows),
            "naive_theta": dml.naive_theta(rows),
        }
    if task["id"] == "survival-recovery":
        data = ROOT / task["data"]
        rows = survival.load(data)
        return {
            "n": len(rows),
            "true_surv_treat": survival.true_survival(rows, 1),
            "true_surv_control": survival.true_survival(rows, 0),
            "km_surv_treat": survival.km_survival(rows, 1),
            "km_surv_control": survival.km_survival(rows, 0),
            "naive_surv_treat": survival.naive_survival(rows, 1),
            "naive_surv_control": survival.naive_survival(rows, 0),
        }
    if task["id"] == "bayesian-recovery":
        data = ROOT / task["data"]
        rows = bayesian.load(data)
        return {
            "n": len(rows),
            "data_mean": bayesian.data_mean(rows),
            "posterior_weak": bayesian.posterior_weak(rows),
            "posterior_strong": bayesian.posterior_strong(rows),
        }
    if task["id"] == "synthetic-control-recovery":
        data = ROOT / task["data"]
        rows = synth.load(data)
        return {
            "n": len(rows),
            "true_effect": synth.true_effect(rows),
            "sc_effect": synth.sc_effect(rows),
            "naive_effect": synth.naive_effect(rows),
        }
    raise ValueError(f"unknown task {task['id']}")


def grade(task: dict, candidate: dict, truth: dict) -> list[dict]:
    out = []
    for g in task["gold"]:
        gid, check = g["id"], g["check"]
        passed, detail = False, ""

        # --- shared / lalonde checks ---
        if check == "imbalance_count":
            thr = g["smd_threshold"]
            n_big = sum(1 for v in candidate.get("balance", {}).values() if abs(v) > thr)
            passed = n_big >= g["min_covariates"]
            detail = f"{n_big} covariates with |SMD|>{thr} (need >= {g['min_covariates']})"

        elif check == "naive_sign":
            v = candidate.get("naive_att")
            want = g["expected_sign"]
            passed = v is not None and ((v < 0) if want == "negative" else (v > 0))
            detail = f"naive_att = {v} (want {want})"

        elif check == "adjusted_recovery":
            naive, adj = candidate.get("naive_att"), candidate.get("adjusted_att")
            if naive is not None and adj is not None:
                swing = adj - naive
                passed = adj > 0 and swing >= g["min_swing"]
                detail = f"adjusted {adj} vs naive {naive} (swing {swing:+.0f}, need +{g['min_swing']:.0f} & positive)"
            else:
                detail = "missing naive_att or adjusted_att"

        elif check == "near_benchmark":
            adj = candidate.get("adjusted_att")
            bench, tol = task["experimental_att"], task["experimental_tol"]
            if adj is not None:
                passed = abs(adj - bench) <= tol
                detail = f"adjusted {adj} vs benchmark {bench} (tol {tol}; |diff|={abs(adj-bench):.0f})"
            else:
                detail = "missing adjusted_att"

        # --- card checks ---
        elif check == "value_near":
            v, exp, tol = candidate.get(g["field"]), g["expected"], g["tol"]
            if v is not None:
                passed = abs(v - exp) <= tol
                detail = f"{g['field']} {v} vs {exp} (tol {tol}; |diff|={abs(v-exp):.4f})"
            else:
                detail = f"missing {g['field']}"

        elif check == "iv_gt_ols":
            ols, iv = candidate.get("ols_return"), candidate.get("iv_return")
            if ols is not None and iv is not None:
                passed = iv > 0 and iv > ols
                detail = f"iv {iv:.4f} {'>' if iv > ols else '<='} ols {ols:.4f}"
            else:
                detail = "missing ols_return or iv_return"

        elif check == "first_stage_min":
            f = candidate.get("first_stage_F")
            if f is not None:
                passed = f >= g["min_f"]
                detail = f"first-stage F {f} (need >= {g['min_f']})"
            else:
                detail = "missing first_stage_F"

        # --- staggered-DID checks ---
        elif check == "recovers_truth":
            field, truth_key, tol = g["field"], g["truth_key"], g["tol"]
            v, exp = candidate.get(field), truth[truth_key]
            if v is not None:
                passed = abs(v - exp) <= tol
                detail = f"{field} {v} vs true {exp:.4f} (tol {tol}; |diff|={abs(v-exp):.4f})"
            else:
                detail = f"missing {field}"

        elif check == "biased_away":
            field, truth_key = g["field"], g["truth_key"]
            v, exp = candidate.get(field), truth[truth_key]
            if v is not None:
                gap = abs(v - exp)
                passed = gap >= g["min_gap"]
                detail = f"{field} {v} vs true {exp:.4f} (gap {gap:.4f})"
            else:
                detail = f"missing {field}"

        elif check == "closer_to_truth":
            near, far = candidate.get(g["near_field"]), candidate.get(g["far_field"])
            exp = truth[g["truth_key"]]
            if near is not None and far is not None:
                near_gap, far_gap = abs(near - exp), abs(far - exp)
                passed = near_gap < far_gap
                detail = f"{g['near_field']} gap {near_gap:.4f} vs {g['far_field']} gap {far_gap:.4f}"
            else:
                detail = f"missing {g['near_field']} or {g['far_field']}"

        # --- shared anti-fabrication cross-check ---
        elif check == "cross_check":
            problems = []
            if task["id"] == "lalonde-recovery":
                rn = candidate.get("naive_att")
                if rn is None or abs(rn - truth["naive_att"]) > g["naive_tol"]:
                    problems.append(f"naive_att {rn} vs true {truth['naive_att']:.1f}")
                rbal = candidate.get("balance", {})
                for k, tv in truth["smd"].items():
                    rv = rbal.get(k)
                    if rv is None or abs(rv - tv) > g["smd_tol"]:
                        problems.append(f"SMD[{k}] {rv} vs true {tv:.3f}")
            elif task["id"] == "card-iv-recovery":
                for field in ("ols_return", "iv_return"):
                    rv = candidate.get(field)
                    if rv is None or abs(rv - truth[field]) > g["tol"]:
                        problems.append(f"{field} {rv} vs true {truth[field]:.4f}")
                rf = candidate.get("first_stage_F")
                if rf is None or abs(rf - truth["first_stage_F"]) > g["f_tol"]:
                    problems.append(f"first_stage_F {rf} vs true {truth['first_stage_F']:.2f}")
            elif task["id"] == "did-staggered-recovery":
                for field in ("twfe_att", "cs_att"):
                    rv = candidate.get(field)
                    if rv is None or abs(rv - truth[field]) > g["tol"]:
                        problems.append(f"{field} {rv} vs true {truth[field]:.4f}")
                rt = candidate.get("true_att")
                if rt is not None and abs(rt - truth["true_att"]) > g["tol"]:
                    problems.append(f"true_att {rt} vs true {truth['true_att']:.4f}")
            elif task["id"] == "rdd-recovery":
                for field in ("naive_jump", "global_att", "local_att"):
                    rv = candidate.get(field)
                    if rv is None or abs(rv - truth[field]) > g["tol"]:
                        problems.append(f"{field} {rv} vs true {truth[field]:.4f}")
                rt = candidate.get("true_tau")
                if rt is not None and abs(rt - truth["true_tau"]) > g["tol"]:
                    problems.append(f"true_tau {rt} vs true {truth['true_tau']:.4f}")
            elif task["id"] == "bad-control-recovery":
                for field in ("naive_effect", "good_control_effect", "bad_control_effect"):
                    rv = candidate.get(field)
                    if rv is None or abs(rv - truth[field]) > g["tol"]:
                        problems.append(f"{field} {rv} vs true {truth[field]:.4f}")
                rt = candidate.get("true_total")
                if rt is not None and abs(rt - truth["true_total"]) > g["tol"]:
                    problems.append(f"true_total {rt} vs true {truth['true_total']:.4f}")
            passed = not problems
            detail = "reported numbers match data" if passed else "; ".join(problems[:3])

        else:
            detail = f"unknown check {check}"

        out.append({"id": gid, "required": g.get("required", False),
                    "weight": g.get("weight", 1), "passed": passed, "detail": detail})
    return out


def exit_code_for_failures(
    required_failures: list[str],
    optional_failures: list[str],
    strict: bool,
    fail_on_partial: bool,
) -> int:
    if required_failures and strict:
        return 1
    if optional_failures and fail_on_partial:
        return 1
    return 0


def grade_task(
    task_path: Path,
    candidate_override: str | None,
    strict: bool,
    fail_on_partial: bool,
) -> tuple[int, list[str]]:
    with task_path.open("rb") as fh:
        task = tomllib.load(fh)
    task_id = task.get("id", task_path.stem) if isinstance(task, dict) else task_path.stem
    task_problems = validate_task(task, task_path)
    if task_problems:
        print(f"[{task_id}] invalid benchmark task {task_path}", file=sys.stderr)
        for problem in task_problems:
            print(f"  - {problem}", file=sys.stderr)
        return 1, [task_id]

    cand_dir = candidate_override or task.get("reference_candidate", "")
    candidate_dir_problems = validate_candidate_dir_name(cand_dir, "candidate")
    if candidate_dir_problems:
        print(f"[{task['id']}] invalid candidate directory name", file=sys.stderr)
        for problem in candidate_dir_problems:
            print(f"  - {problem}", file=sys.stderr)
        return 1, [task["id"]]

    results_json = (CANDIDATES_DIR / cand_dir / "results.json") if cand_dir else None
    if not results_json or not results_json.exists():
        print(f"[{task['id']}] no candidate results.json at {results_json}", file=sys.stderr)
        print("  Run: python3 benchmark/reference_pipeline.py", file=sys.stderr)
        return 1, [task["id"]]
    try:
        candidate = json.loads(results_json.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        print(f"[{task['id']}] invalid candidate JSON {results_json}", file=sys.stderr)
        print(f"  - {exc.msg} at line {exc.lineno}, column {exc.colno}", file=sys.stderr)
        return 1, [task["id"]]

    candidate_problems = validate_candidate(task, candidate, results_json)
    if candidate_problems:
        print(f"[{task['id']}] invalid candidate {results_json}", file=sys.stderr)
        for problem in candidate_problems:
            print(f"  - {problem}", file=sys.stderr)
        return 1, [task["id"]]

    truth = compute_truth(task)
    graded = grade(task, candidate, truth)
    earned = sum(g["weight"] for g in graded if g["passed"])
    possible = sum(g["weight"] for g in graded)
    req_fail = [g["id"] for g in graded if g["required"] and not g["passed"]]
    opt_fail = [g["id"] for g in graded if not g["required"] and not g["passed"]]

    print(f"Benchmark: {task['id']}  (candidate: {results_json.parent.name}, N={truth['n']})")
    print("-" * 72)
    for g in graded:
        mark = "PASS" if g["passed"] else "FAIL"
        req = "*" if g["required"] else " "
        print(f"  [{mark}]{req} {g['id']:32s} {g['detail']}")
    print(f"Score: {earned}/{possible}  |  required failures: {req_fail or 'none'}")
    if opt_fail:
        print(f"Optional failures: {opt_fail}")
    print()

    RESULTS_DIR.mkdir(parents=True, exist_ok=True)
    (RESULTS_DIR / f"{task['id']}.json").write_text(
        json.dumps({"task": task["id"], "candidate": results_json.parent.name,
                    "earned": earned, "possible": possible,
                    "required_failures": req_fail,
                    "optional_failures": opt_fail,
                    "items": graded}, indent=2) + "\n",
        encoding="utf-8")
    return exit_code_for_failures(req_fail, opt_fail, strict, fail_on_partial), req_fail


def lint_task(task_path: Path, candidate_override: str | None = None) -> list[str]:
    with task_path.open("rb") as fh:
        task = tomllib.load(fh)
    task_id = task.get("id", task_path.stem) if isinstance(task, dict) else task_path.stem
    problems = [f"{task_id}: {problem}" for problem in validate_task(task, task_path)]
    if problems:
        return problems

    cand_dir = candidate_override or task.get("reference_candidate", "")
    problems.extend(
        f"{task['id']}: {problem}"
        for problem in validate_candidate_dir_name(cand_dir, "candidate")
    )
    if problems:
        return problems

    results_json = CANDIDATES_DIR / cand_dir / "results.json"
    if not results_json.exists():
        return [f"{task['id']}: missing candidate results.json at {results_json.relative_to(ROOT)}"]
    try:
        candidate = json.loads(results_json.read_text(encoding="utf-8"))
    except json.JSONDecodeError as exc:
        return [
            f"{task['id']}: invalid candidate JSON {results_json.relative_to(ROOT)} "
            f"({exc.msg} at line {exc.lineno}, column {exc.colno})"
        ]
    return [
        f"{task['id']}: {problem}"
        for problem in validate_candidate(task, candidate, results_json)
    ]


def lint_tasks(task_paths: list[Path], candidate_override: str | None = None) -> int:
    problems: list[str] = []
    for task_path in task_paths:
        problems.extend(lint_task(task_path, candidate_override))
    if problems:
        print(f"Benchmark lint found {len(problems)} problem(s):", file=sys.stderr)
        for problem in problems:
            print(f"  - {problem}", file=sys.stderr)
        return 1
    print(f"Benchmark lint passed for {len(task_paths)} task(s).")
    return 0


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Grade AERS benchmark candidates")
    ap.add_argument("--task", help="grade only this task id (default: all)")
    ap.add_argument("--candidate", help="override candidate dir name under benchmark/candidates/")
    ap.add_argument(
        "--lint",
        action="store_true",
        help="validate task specs and reference candidate metadata without scoring",
    )
    ap.add_argument("--strict", action="store_true")
    ap.add_argument(
        "--fail-on-partial",
        action="store_true",
        help="nonzero exit when any non-required gold fails",
    )
    ap.add_argument(
        "--fail-on-orphan-results",
        action="store_true",
        help="nonzero exit when benchmark/results/ contains JSON for no current task",
    )
    args = ap.parse_args(argv)

    all_tasks = sorted(TASKS_DIR.glob("*.toml"))
    task_ids = {path.stem for path in all_tasks}
    orphan_results = orphan_result_files(RESULTS_DIR, task_ids)
    if orphan_results:
        print(
            f"Found {len(orphan_results)} orphan benchmark result file(s):",
            file=sys.stderr,
        )
        for path in orphan_results:
            print(f"  - {path.relative_to(ROOT)}", file=sys.stderr)
        if args.fail_on_orphan_results:
            return 1

    tasks = all_tasks
    if args.task:
        tasks = [t for t in tasks if t.stem == args.task]
        if not tasks:
            print(f"No task '{args.task}'", file=sys.stderr)
            return 1

    if args.lint:
        return lint_tasks(tasks, args.candidate)

    rc = 0
    for t in tasks:
        code, _ = grade_task(t, args.candidate, args.strict, args.fail_on_partial)
        rc = rc or code
    return rc


if __name__ == "__main__":
    raise SystemExit(main())

x,d,m,y,y0,y1
0,0,-0.6,-1.2,-1.2,1.3
0,1,0.7,1.9,-0.6,1.9
1,0,0.5,2.0,2.0,4.5
1,1,1.8,5.1,2.6,5.1
2,0,1.6,5.2,5.2,7.7
2,1,1.4,5.3,2.8,5.3
0,0,-0.3,-0.6,-0.6,1.9
0,1,1.0,2.5,0.0,2.5
1,0,0.8,2.6,2.6,5.1
1,1,2.1,5.7,3.2,5.7
2,0,0.4,2.8,2.8,5.3
2,1,1.7,5.9,3.4,5.9
0,0,0.0,0.0,0.0,2.5
0,1,1.3,3.1,0.6,3.1
1,0,1.1,3.2,3.2,5.7
1,1,0.9,3.3,0.8,3.3
2,0,0.7,3.4,3.4,5.9
2,1,2.0,6.5,4.0,6.5
0,0,0.3,0.6,0.6,3.1
0,1,1.6,3.7,1.2,3.7
1,0,-0.1,0.8,0.8,3.3
1,1,1.2,3.9,1.4,3.9
2,0,1.0,4.0,4.0,6.5
2,1,2.3,7.1,4.6,7.1
0,0,0.6,1.2,1.2,3.7
0,1,0.4,1.3,-1.2,1.3
1,0,0.2,1.4,1.4,3.9
1,1,1.5,4.5,2.0,4.5
2,0,1.3,4.6,4.6,7.1
2,1,2.6,7.7,5.2,7.7
0,0,-0.6,-1.2,-1.2,1.3
0,1,0.7,1.9,-0.6,1.9
1,0,0.5,2.0,2.0,4.5
1,1,1.8,5.1,2.6,5.1
2,0,1.6,5.2,5.2,7.7
2,1,1.4,5.3,2.8,5.3
0,0,-0.3,-0.6,-0.6,1.9
0,1,1.0,2.5,0.0,2.5
1,0,0.8,2.6,2.6,5.1
1,1,2.1,5.7,3.2,5.7
2,0,0.4,2.8,2.8,5.3
2,1,1.7,5.9,3.4,5.9
0,0,0.0,0.0,0.0,2.5
0,1,1.3,3.1,0.6,3.1
1,0,1.1,3.2,3.2,5.7
1,1,0.9,3.3,0.8,3.3
2,0,0.7,3.4,3.4,5.9
2,1,2.0,6.5,4.0,6.5
0,0,0.3,0.6,0.6,3.1
0,1,1.6,3.7,1.2,3.7
1,0,-0.1,0.8,0.8,3.3
1,1,1.2,3.9,1.4,3.9
2,0,1.0,4.0,4.0,6.5
2,1,2.3,7.1,4.6,7.1
0,0,0.6,1.2,1.2,3.7
0,1,0.4,1.3,-1.2,1.3
1,0,0.2,1.4,1.4,3.9
1,1,1.5,4.5,2.0,4.5
2,0,1.3,4.6,4.6,7.1
2,1,2.6,7.7,5.2,7.7
0,0,-0.6,-1.2,-1.2,1.3
0,1,0.7,1.9,-0.6,1.9
1,0,0.5,2.0,2.0,4.5
1,1,1.8,5.1,2.6,5.1
2,0,1.6,5.2,5.2,7.7
2,1,1.4,5.3,2.8,5.3
0,0,-0.3,-0.6,-0.6,1.9
0,1,1.0,2.5,0.0,2.5
1,0,0.8,2.6,2.6,5.1
1,1,2.1,5.7,3.2,5.7
2,0,0.4,2.8,2.8,5.3
2,1,1.7,5.9,3.4,5.9
0,0,0.0,0.0,0.0,2.5
0,1,1.3,3.1,0.6,3.1
1,0,1.1,3.2,3.2,5.7
1,1,0.9,3.3,0.8,3.3
2,0,0.7,3.4,3.4,5.9
2,1,2.0,6.5,4.0,6.5
0,0,0.3,0.6,0.6,3.1
0,1,1.6,3.7,1.2,3.7
1,0,-0.1,0.8,0.8,3.3
1,1,1.2,3.9,1.4,3.9
2,0,1.0,4.0,4.0,6.5
2,1,2.3,7.1,4.6,7.1
0,0,0.6,1.2,1.2,3.7
0,1,0.4,1.3,-1.2,1.3
1,0,0.2,1.4,1.4,3.9
1,1,1.5,4.5,2.0,4.5
2,0,1.3,4.6,4.6,7.1
2,1,2.6,7.7,5.2,7.7
0,0,-0.6,-1.2,-1.2,1.3
0,1,0.7,1.9,-0.6,1.9
1,0,0.5,2.0,2.0,4.5
1,1,1.8,5.1,2.6,5.1
2,0,1.6,5.2,5.2,7.7
2,1,1.4,5.3,2.8,5.3
0,0,-0.3,-0.6,-0.6,1.9
0,1,1.0,2.5,0.0,2.5
1,0,0.8,2.6,2.6,5.1
1,1,2.1,5.7,3.2,5.7
2,0,0.4,2.8,2.8,5.3
2,1,1.7,5.9,3.4,5.9
0,0,0.0,0.0,0.0,2.5
0,1,1.3,3.1,0.6,3.1
1,0,1.1,3.2,3.2,5.7
1,1,0.9,3.3,0.8,3.3
2,0,0.7,3.4,3.4,5.9
2,1,2.0,6.5,4.0,6.5
0,0,0.3,0.6,0.6,3.1
0,1,1.6,3.7,1.2,3.7
1,0,-0.1,0.8,0.8,3.3
1,1,1.2,3.9,1.4,3.9
2,0,1.0,4.0,4.0,6.5
2,1,2.3,7.1,4.6,7.1
0,0,0.6,1.2,1.2,3.7
0,1,0.4,1.3,-1.2,1.3
1,0,0.2,1.4,1.4,3.9
1,1,1.5,4.5,2.0,4.5
2,0,1.3,4.6,4.6,7.1
2,1,2.6,7.7,5.2,7.7

id,y
1,0.1
2,0.3
3,0.5
4,0.7
5,0.9
6,1.1
7,1.3
8,1.5
9,1.7
10,1.9
11,2.1
12,2.3
13,2.5
14,2.7
15,2.9
16,3.1
17,3.3
18,3.5
19,3.7
20,3.9

id,d,y,x1,x2,x3,x4
1,1.54,6.91,1.0,3.0,-0.5,2.0
2,4.16,15.44,2.0,6.0,0.5,1.0
3,3.28,11.72,3.0,2.0,1.5,0.0
4,4.3,15.85,4.0,5.0,-1.5,2.0
5,0.42,1.63,0.0,1.0,-0.5,1.0
6,3.04,10.16,1.0,4.0,0.5,0.0
7,2.16,7.64,2.0,0.0,1.5,2.0
8,3.18,10.57,3.0,3.0,-1.5,1.0
9,4.72,17.48,4.0,6.0,-0.5,0.0
10,0.84,4.46,0.0,2.0,0.5,2.0
11,3.46,12.99,1.0,5.0,1.5,1.0
12,0.98,3.67,2.0,1.0,-1.5,0.0
13,3.6,13.4,3.0,4.0,-0.5,2.0
14,2.72,9.68,4.0,0.0,0.5,1.0
15,2.34,7.71,0.0,3.0,1.5,0.0
16,3.36,11.84,1.0,6.0,-1.5,2.0
17,2.48,8.12,2.0,2.0,-0.5,1.0
18,4.02,15.03,3.0,5.0,0.5,0.0
19,3.14,12.51,4.0,1.0,1.5,2.0
20,1.16,4.94,0.0,4.0,-1.5,1.0
21,0.28,1.22,1.0,0.0,-0.5,0.0
22,2.9,10.95,2.0,3.0,0.5,2.0
23,5.52,19.48,3.0,6.0,1.5,1.0
24,3.04,10.16,4.0,2.0,-1.5,0.0
25,2.66,9.39,0.0,5.0,-0.5,2.0
26,1.78,5.67,1.0,1.0,0.5,1.0
27,3.32,12.58,2.0,4.0,1.5,0.0
28,0.84,4.46,3.0,0.0,-1.5,2.0
29,3.46,12.99,4.0,3.0,-0.5,1.0
30,3.08,11.02,0.0,6.0,0.5,0.0
31,2.2,8.5,1.0,2.0,1.5,2.0
32,3.22,11.43,2.0,5.0,-1.5,1.0
33,2.34,7.71,3.0,1.0,-0.5,0.0
34,4.96,17.44,4.0,4.0,0.5,2.0
35,1.08,3.22,0.0,0.0,1.5,1.0
36,1.02,4.53,1.0,3.0,-1.5,0.0
37,3.64,14.26,2.0,6.0,-0.5,2.0
38,2.76,10.54,3.0,2.0,0.5,1.0
39,5.38,19.07,4.0,5.0,1.5,0.0
40,-0.1,0.45,0.0,1.0,-1.5,2.0
41,2.52,8.98,1.0,4.0,-0.5,1.0
42,1.64,5.26,2.0,0.0,0.5,0.0
43,4.26,14.99,3.0,3.0,1.5,2.0
44,5.28,17.92,4.0,6.0,-1.5,1.0
45,0.32,2.08,0.0,2.0,-0.5,0.0
46,2.94,11.81,1.0,5.0,0.5,2.0
47,2.06,8.09,2.0,1.0,1.5,1.0
48,3.08,11.02,3.0,4.0,-1.5,0.0
49,2.2,8.5,4.0,0.0,-0.5,2.0
50,1.82,6.53,0.0,3.0,0.5,1.0
51,4.44,15.06,1.0,6.0,1.5,0.0
52,1.96,6.94,2.0,2.0,-1.5,2.0
53,4.58,15.47,3.0,5.0,-0.5,1.0
54,2.62,10.13,4.0,1.0,0.5,0.0
55,2.24,9.36,0.0,4.0,1.5,2.0
56,-0.24,0.04,1.0,0.0,-1.5,1.0
57,2.38,8.57,2.0,3.0,-0.5,0.0
58,5.0,18.3,3.0,6.0,0.5,2.0
59,4.12,14.58,4.0,2.0,1.5,1.0
60,2.14,7.01,0.0,5.0,-1.5,0.0
61,1.26,4.49,1.0,1.0,-0.5,2.0
62,3.88,13.02,2.0,4.0,0.5,1.0
63,1.92,7.68,3.0,0.0,1.5,0.0
64,2.94,11.81,4.0,3.0,-1.5,2.0
65,2.56,9.84,0.0,6.0,-0.5,1.0
66,1.68,6.12,1.0,2.0,0.5,0.0
67,4.3,15.85,2.0,5.0,1.5,2.0
68,1.82,6.53,3.0,1.0,-1.5,1.0
69,4.44,15.06,4.0,4.0,-0.5,0.0
70,0.56,2.04,0.0,0.0,0.5,2.0
71,3.18,10.57,1.0,3.0,1.5,1.0
72,3.12,11.88,2.0,6.0,-1.5,0.0
73,2.24,9.36,3.0,2.0,-0.5,2.0
74,4.86,17.89,4.0,5.0,0.5,1.0
75,0.98,3.67,0.0,1.0,1.5,0.0
76,2.0,7.8,1.0,4.0,-1.5,2.0
77,1.12,4.08,2.0,0.0,-0.5,1.0
78,3.74,12.61,3.0,3.0,0.5,0.0
79,6.36,22.34,4.0,6.0,1.5,2.0
80,0.88,2.52,0.0,2.0,-1.5,1.0
81,2.42,9.43,1.0,5.0,-0.5,0.0
82,1.54,6.91,2.0,1.0,0.5,2.0
83,4.16,15.44,3.0,4.0,1.5,1.0
84,1.68,6.12,4.0,0.0,-1.5,0.0
85,1.3,5.35,0.0,3.0,-0.5,2.0
86,3.92,13.88,1.0,6.0,0.5,1.0
87,3.04,10.16,2.0,2.0,1.5,0.0
88,4.06,14.29,3.0,5.0,-1.5,2.0
89,3.18,10.57,4.0,1.0,-0.5,1.0
90,1.72,6.98,0.0,4.0,0.5,0.0
91,0.84,4.46,1.0,0.0,1.5,2.0
92,1.86,7.39,2.0,3.0,-1.5,1.0
93,4.48,15.92,3.0,6.0,-0.5,0.0
94,3.6,13.4,4.0,2.0,0.5,2.0
95,3.22,11.43,0.0,5.0,1.5,1.0
96,0.74,2.11,1.0,1.0,-1.5,0.0
97,3.36,11.84,2.0,4.0,-0.5,2.0
98,2.48,8.12,3.0,0.0,0.5,1.0
99,4.02,15.03,4.0,3.0,1.5,0.0
100,2.04,8.66,0.0,6.0,-1.5,2.0
101,1.16,4.94,1.0,2.0,-0.5,1.0
102,3.78,13.47,2.0,5.0,0.5,0.0
103,2.9,10.95,3.0,1.0,1.5,2.0
104,3.92,13.88,4.0,4.0,-1.5,1.0
105,0.04,-0.34,0.0,0.0,-0.5,0.0
106,2.66,9.39,1.0,3.0,0.5,2.0
107,5.28,17.92,2.0,6.0,1.5,1.0
108,1.72,6.98,3.0,2.0,-1.5,0.0
109,4.34,16.71,4.0,5.0,-0.5,2.0
110,0.46,2.49,0.0,1.0,0.5,1.0
111,3.08,11.02,1.0,4.0,1.5,0.0
112,0.6,2.9,2.0,0.0,-1.5,2.0
113,3.22,11.43,3.0,3.0,-0.5,1.0
114,5.84,19.96,4.0,6.0,0.5,0.0
115,1.96,6.94,0.0,2.0,1.5,2.0
116,2.98,9.87,1.0,5.0,-1.5,1.0
117,1.02,4.53,2.0,1.0,-0.5,0.0
118,3.64,14.26,3.0,4.0,0.5,2.0
119,2.76,10.54,4.0,0.0,1.5,1.0
120,0.78,2.97,0.0,3.0,-1.5,0.0