Improvement Evaluator

[
  {
    "type": "accuracy",
    "succeeded": true,
    "context": {
      "dimension": "accuracy",
      "scores": {
        "coverage": 0.9,
        "accuracy": 0.7428571428571429,
        "efficiency": 1.0,
        "reliability": 1.0,
        "security": 0.8333333333333334,
        "trigger_quality": 1.0
      }
    },
    "timestamp": "2026-04-05T15:23:39Z",
    "hit_count": 1
  }
]

#!/usr/bin/env python3
"""Judge implementations for task evaluation.

Three judge types:
- ContainsJudge: deterministic keyword check
- PytestJudge: run pytest on AI output
- LLMRubricJudge: LLM scores against rubric (mock mode available)
"""

from __future__ import annotations

import json
import subprocess
import tempfile
from abc import ABC, abstractmethod
from pathlib import Path
from typing import Any


class BaseJudge(ABC):
    """Abstract base for all judge types."""

    @abstractmethod
    def evaluate(self, output: str, task: dict) -> dict:
        """Returns {passed: bool, details: str, score: float}"""


class ContainsJudge(BaseJudge):
    """Check that output contains all expected keywords (case-insensitive)."""

    def evaluate(self, output: str, task: dict) -> dict:
        expected: list[str] = task["judge"]["expected"]
        output_lower = output.lower()
        found = [kw for kw in expected if kw.lower() in output_lower]
        missing = [kw for kw in expected if kw.lower() not in output_lower]
        passed = len(missing) == 0
        score = len(found) / len(expected) if expected else 0.0
        return {
            "passed": passed,
            "details": f"Found: {found}, Missing: {missing}",
            "score": round(score, 4),
        }


class PytestJudge(BaseJudge):
    """Write AI output to temp file, run pytest with the specified test file.

    SECURITY: test_file must start with "fixtures/" to prevent path traversal.
    """

    def evaluate(self, output: str, task: dict) -> dict:
        test_file = task["judge"]["test_file"]
        # Security: validate test_file is within fixtures/
        if not test_file.startswith("fixtures/"):
            return {
                "passed": False,
                "details": f"SECURITY: test_file must start with 'fixtures/', got: {test_file}",
                "score": 0.0,
            }

        skill_root = Path(__file__).resolve().parents[1]
        test_path = skill_root / "tests" / test_file
        # Security: resolve symlinks and prevent traversal (e.g. fixtures/../../)
        resolved = test_path.resolve()
        fixtures_root = (skill_root / "tests" / "fixtures").resolve()
        if not str(resolved).startswith(str(fixtures_root)):
            return {
                "passed": False,
                "details": "SECURITY: path traversal detected",
                "score": 0.0,
            }
        if not test_path.exists():
            return {
                "passed": False,
                "details": f"Test file not found: {test_path}",
                "score": 0.0,
            }

        tmpdir = None
        try:
            tmpdir = tempfile.mkdtemp(prefix="evaluator_pytest_")
            output_file = Path(tmpdir) / "ai_output.txt"
            output_file.write_text(output, encoding="utf-8")

            result = subprocess.run(
                ["python3", "-m", "pytest", str(test_path), "-v",
                 "--tb=short", f"--rootdir={tmpdir}"],
                capture_output=True,
                text=True,
                timeout=60,
                env={
                    **__import__("os").environ,
                    "AI_OUTPUT_FILE": str(output_file),
                },
            )
            passed = result.returncode == 0
            details = result.stdout[-500:] if result.stdout else result.stderr[-500:]
            return {
                "passed": passed,
                "details": details.strip(),
                "score": 1.0 if passed else 0.0,
            }
        except subprocess.TimeoutExpired:
            return {
                "passed": False,
                "details": "pytest timed out after 60s",
                "score": 0.0,
            }
        except Exception as exc:
            return {
                "passed": False,
                "details": f"pytest execution error: {exc}",
                "score": 0.0,
            }
        finally:
            if tmpdir:
                import shutil
                shutil.rmtree(tmpdir, ignore_errors=True)


class LLMRubricJudge(BaseJudge):
    """Score AI output against a rubric via LLM (or mock).

    Pass threshold comes from task["judge"]["pass_threshold"].
    """

    def __init__(self, mock: bool = False):
        self.mock = mock

    def evaluate(self, output: str, task: dict) -> dict:
        rubric = task["judge"].get("rubric", "")
        pass_threshold = task["judge"].get("pass_threshold", 0.7)

        if self.mock:
            score = 0.8
            return {
                "passed": score >= pass_threshold,
                "details": f"[mock] score={score}, threshold={pass_threshold}",
                "score": score,
            }

        # Real LLM evaluation via claude -p
        prompt = (
            f"You are a judge evaluating AI output quality.\n\n"
            f"## Rubric\n{rubric}\n\n"
            f"## AI Output\n{output[:3000]}\n\n"
            f"Score the output from 0.0 to 1.0 based on the rubric.\n"
            f"Respond with ONLY a JSON object: {{\"score\": <float>, \"reasoning\": \"<str>\"}}"
        )
        try:
            result = subprocess.run(
                ["claude", "-p", "--output-format", "json"],
                input=prompt,
                capture_output=True,
                text=True,
                timeout=120,
            )
            if result.returncode != 0:
                return {
                    "passed": False,
                    "details": f"claude -p failed: {result.stderr[:300]}",
                    "score": 0.0,
                }
            # Parse claude JSON output to get the text result
            try:
                claude_output = json.loads(result.stdout)
                text_result = claude_output.get("result", result.stdout)
            except (json.JSONDecodeError, TypeError):
                text_result = result.stdout

            # Parse the judge's JSON response from the text
            parsed = json.loads(text_result) if isinstance(text_result, str) else text_result
            score = float(parsed.get("score", 0.0))
            reasoning = parsed.get("reasoning", "")
            return {
                "passed": score >= pass_threshold,
                "details": f"score={score}, threshold={pass_threshold}, reasoning={reasoning}",
                "score": score,
            }
        except json.JSONDecodeError:
            return {
                "passed": False,
                "details": "Failed to parse LLM judge response as JSON",
                "score": 0.0,
            }
        except subprocess.TimeoutExpired:
            return {
                "passed": False,
                "details": "LLM judge timed out after 120s",
                "score": 0.0,
            }
        except FileNotFoundError:
            return {
                "passed": False,
                "details": "claude CLI not found",
                "score": 0.0,
            }


def get_judge(judge_config: dict, mock: bool = False) -> BaseJudge:
    """Factory function to create the appropriate judge."""
    judge_type = judge_config["type"]
    if judge_type == "contains":
        return ContainsJudge()
    if judge_type == "pytest":
        return PytestJudge()
    if judge_type == "llm-rubric":
        return LLMRubricJudge(mock=mock)
    raise ValueError(f"Unknown judge type: {judge_type}")

#!/usr/bin/env python3
"""Improvement Evaluator: run task suites to measure Skill execution effectiveness.

Sits between the discriminator (scoring) and gate (quality gate) stages.
Runs real tasks with the Skill under test, compares candidate vs baseline pass rates.

Usage:
    python3 scripts/evaluate.py \
        --input ranking.json \
        --candidate-id c1 \
        --task-suite tasks.yaml \
        --state-root /tmp/state \
        [--pass-k 1] \
        [--baseline-cache-dir /tmp/cache] \
        [--output path.json] \
        [--eval-threshold 6.0] \
        [--mock]
"""

from __future__ import annotations

import argparse
import hashlib
import json
import logging
import shutil
import sys
import time
from pathlib import Path
from typing import Any

import yaml

_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))

_SCRIPTS_DIR = str(Path(__file__).resolve().parent)
if _SCRIPTS_DIR not in sys.path:
    sys.path.insert(0, _SCRIPTS_DIR)

from lib.common import (
    SCHEMA_VERSION,
    read_json,
    utc_now_iso,
    write_json,
)

from task_runner import TaskRunner, TaskResult

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------

BASELINE_CACHE_TTL_DAYS = 7
BASELINE_ABORT_THRESHOLD = 0.2  # abort if baseline < 20%
VALID_JUDGE_TYPES = {"contains", "pytest", "llm-rubric"}


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Run task suite to evaluate Skill execution effectiveness.",
    )
    parser.add_argument("--input", help="Path to ranking artifact JSON (from discriminator)")
    parser.add_argument("--candidate-id", help="ID of the candidate to evaluate")
    parser.add_argument("--standalone", action="store_true",
                        help="Run task suite directly without ranking artifact (standalone mode)")
    parser.add_argument("--task-suite", required=True, help="Path to task suite YAML")
    parser.add_argument("--state-root", required=True, help="State root directory")
    parser.add_argument("--pass-k", type=int, default=1, help="Number of attempts per task (pass@k)")
    parser.add_argument("--baseline-cache-dir", help="Directory for baseline result caching")
    parser.add_argument("--output", help="Output path for evaluation artifact JSON")
    parser.add_argument("--eval-threshold", type=float, default=6.0, help="Minimum discriminator score to evaluate")
    parser.add_argument("--mock", action="store_true", help="Use mock execution (no claude CLI needed)")
    parser.add_argument("--skill-path", help="Path to SKILL.md or skill directory (standalone mode: prepend to prompts)")
    return parser.parse_args(argv)


# ---------------------------------------------------------------------------
# Preflight checks
# ---------------------------------------------------------------------------


def preflight_check(task_suite_path: Path, mock: bool = False) -> None:
    """Validate prerequisites before running evaluation."""
    # Check claude CLI availability (skip in mock mode)
    if not mock:
        if shutil.which("claude") is None:
            raise AssertionError(
                "claude CLI not found in PATH. Install it or use --mock for testing."
            )

    # Validate task suite file exists
    if not task_suite_path.exists():
        raise AssertionError(f"Task suite not found: {task_suite_path}")

    # Validate YAML schema
    suite = _load_yaml(task_suite_path)
    _validate_suite_schema(suite)


def _load_yaml(path: Path) -> dict:
    """Load and return YAML content."""
    with path.open("r", encoding="utf-8") as f:
        return yaml.safe_load(f)


def _validate_suite_schema(suite: dict) -> None:
    """Validate task suite YAML schema."""
    assert "skill_id" in suite and suite["skill_id"], "skill_id is required and must be non-empty"
    assert "version" in suite and suite["version"] == "1.0", "version must be '1.0'"
    assert "tasks" in suite and isinstance(suite["tasks"], list), "tasks must be a list"
    assert len(suite["tasks"]) > 0, "tasks list must not be empty"

    seen_ids: set[str] = set()
    for i, task in enumerate(suite["tasks"]):
        assert "id" in task, f"task[{i}] missing 'id'"
        assert task["id"] not in seen_ids, f"duplicate task id: {task['id']}"
        seen_ids.add(task["id"])
        assert "prompt" in task and task["prompt"], f"task[{i}] missing or empty 'prompt'"
        assert "judge" in task, f"task[{i}] missing 'judge'"
        judge = task["judge"]
        assert "type" in judge, f"task[{i}] judge missing 'type'"
        assert judge["type"] in VALID_JUDGE_TYPES, (
            f"task[{i}] unknown judge type: {judge['type']}"
        )

        if judge["type"] == "contains":
            assert "expected" in judge and isinstance(judge["expected"], list), (
                f"task[{i}] contains judge needs 'expected' list"
            )
            assert len(judge["expected"]) > 0, f"task[{i}] 'expected' list must not be empty"
        elif judge["type"] == "pytest":
            assert "test_file" in judge, f"task[{i}] pytest judge needs 'test_file'"
            assert judge["test_file"].startswith("fixtures/"), (
                f"task[{i}] test_file must start with 'fixtures/'"
            )
        elif judge["type"] == "llm-rubric":
            assert "rubric" in judge and judge["rubric"], (
                f"task[{i}] llm-rubric judge needs non-empty 'rubric'"
            )


# ---------------------------------------------------------------------------
# Task suite loading
# ---------------------------------------------------------------------------


def load_task_suite(path: Path) -> dict:
    """Load and validate a task suite YAML file."""
    suite = _load_yaml(path)
    _validate_suite_schema(suite)
    return suite


# ---------------------------------------------------------------------------
# Candidate / baseline execution
# ---------------------------------------------------------------------------


def extract_candidate_skill(ranking_artifact: dict, candidate_id: str) -> dict:
    """Find the candidate by ID in the ranking artifact."""
    for candidate in ranking_artifact.get("scored_candidates", []):
        if candidate.get("id") == candidate_id:
            return candidate
    raise ValueError(f"Candidate '{candidate_id}' not found in ranking artifact")


def get_skill_content(candidate: dict, target: dict) -> str:
    """Extract the SKILL.md content for a candidate.

    Candidates may have their own 'skill_content' or 'content' field.
    If not, read the original SKILL.md from the target path.
    """
    # Check candidate for inline content
    for key in ("skill_content", "content", "body"):
        if key in candidate and candidate[key]:
            return candidate[key]

    # Fallback: read original SKILL.md from target
    target_path = Path(target.get("path", ""))
    skill_md = target_path / "SKILL.md" if target_path.is_dir() else target_path
    if skill_md.exists():
        return skill_md.read_text(encoding="utf-8")

    raise ValueError(f"Cannot find SKILL.md content for candidate or at {target_path}")


def get_baseline_skill_content(target: dict) -> str:
    """Read the original (baseline) SKILL.md from the target path."""
    target_path = Path(target.get("path", ""))
    skill_md = target_path / "SKILL.md" if target_path.is_dir() else target_path
    if skill_md.exists():
        return skill_md.read_text(encoding="utf-8")
    raise ValueError(f"Baseline SKILL.md not found at {target_path}")


def run_task_suite(
    runner: TaskRunner,
    skill_content: str,
    tasks: list[dict],
    pass_k: int = 1,
) -> list[dict]:
    """Run all tasks in a suite and return per-task results."""
    results = []
    for task in tasks:
        task_result = runner.run(skill_content, task, pass_k=pass_k)
        results.append({
            "task_id": task["id"],
            "passed": task_result.passed,
            "score": task_result.judge_output.get("score", 0.0),
            "details": task_result.judge_output.get("details", ""),
            "duration_ms": task_result.duration_ms,
            "error": task_result.error,
        })
    return results


def compute_pass_rate(results: list[dict]) -> float:
    """Compute pass rate from task results."""
    if not results:
        return 0.0
    passed = sum(1 for r in results if r["passed"])
    return round(passed / len(results), 4)


# ---------------------------------------------------------------------------
# Baseline caching
# ---------------------------------------------------------------------------


def _baseline_cache_key(skill_content: str, suite_path: str) -> str:
    """Generate a cache key from skill content and suite path."""
    h = hashlib.sha256()
    h.update(skill_content.encode("utf-8"))
    h.update(suite_path.encode("utf-8"))
    return h.hexdigest()[:16]


def load_baseline_cache(cache_dir: Path, cache_key: str) -> dict | None:
    """Load cached baseline results if fresh enough."""
    cache_file = cache_dir / f"baseline_{cache_key}.json"
    if not cache_file.exists():
        return None

    cached = read_json(cache_file)
    # Check TTL
    created_at = cached.get("created_at", "")
    if not created_at:
        return None

    from datetime import datetime, timezone
    try:
        created = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
        age_days = (datetime.now(timezone.utc) - created).days
        if age_days > BASELINE_CACHE_TTL_DAYS:
            return None
    except (ValueError, TypeError):
        return None

    return cached


def save_baseline_cache(cache_dir: Path, cache_key: str, data: dict) -> None:
    """Save baseline results to cache."""
    cache_dir.mkdir(parents=True, exist_ok=True)
    cache_file = cache_dir / f"baseline_{cache_key}.json"
    write_json(cache_file, data)


# ---------------------------------------------------------------------------
# Result computation
# ---------------------------------------------------------------------------


def compute_results(
    candidate_rate: float,
    baseline_rate: float,
) -> dict:
    """Compute delta and verdict.

    Verdict is "pass" if candidate pass_rate >= baseline pass_rate.
    """
    delta = round(candidate_rate - baseline_rate, 4)
    verdict = "pass" if delta >= 0 or candidate_rate >= baseline_rate else "fail"
    return {
        "execution_pass_rate": candidate_rate,
        "baseline_pass_rate": baseline_rate,
        "delta": delta,
        "verdict": verdict,
    }


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main(argv: list[str] | None = None) -> int:
    args = parse_args(argv)
    logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

    state_root = Path(args.state_root).expanduser().resolve()
    state_root.mkdir(parents=True, exist_ok=True)
    task_suite_path = Path(args.task_suite).expanduser().resolve()

    # Preflight
    try:
        preflight_check(task_suite_path, mock=args.mock)
    except AssertionError as exc:
        logger.error("Preflight failed: %s", exc)
        return 1

    # --- Standalone mode: run task suite directly without ranking artifact ---
    if args.standalone:
        suite = load_task_suite(task_suite_path)
        tasks = suite["tasks"]
        runner = TaskRunner(mock=args.mock)
        run_id = f"standalone-{suite.get('skill_id', 'unknown')}"

        # Load skill content if --skill-path provided
        skill_content = ""
        if args.skill_path:
            sp = Path(args.skill_path).expanduser().resolve()
            skill_md = sp / "SKILL.md" if sp.is_dir() else sp
            if skill_md.exists():
                skill_content = skill_md.read_text(encoding="utf-8")
                logger.info("Loaded SKILL.md from %s (%d chars)", skill_md, len(skill_content))
            else:
                logger.warning("SKILL.md not found at %s, running without skill content", skill_md)

        logger.info("Running %d tasks in standalone mode...", len(tasks))
        results = run_task_suite(runner, skill_content, tasks, pass_k=args.pass_k)
        pass_rate = compute_pass_rate(results)

        output_path = _resolve_output(args.output, state_root, run_id)
        artifact = {
            "schema_version": SCHEMA_VERSION,
            "lane": "standalone",
            "run_id": run_id,
            "stage": "evaluated",
            "status": "success",
            "created_at": utc_now_iso(),
            "task_suite": str(task_suite_path),
            "skill_id": suite.get("skill_id", ""),
            "pass_k": args.pass_k,
            "evaluation": {
                "total_tasks": len(results),
                "passed": sum(1 for r in results if r["passed"]),
                "failed": sum(1 for r in results if not r["passed"]),
                "pass_rate": pass_rate,
                "verdict": "pass" if pass_rate >= 0.5 else "fail",
            },
            "task_results": results,
            "truth_anchor": str(output_path),
        }
        write_json(output_path, artifact)
        print(str(output_path))

        # Pretty print summary
        logger.info("=== Standalone Evaluation Results ===")
        logger.info("Skill: %s", suite.get("skill_id", "unknown"))
        logger.info("Tasks: %d passed / %d total (%.0f%%)",
                     artifact["evaluation"]["passed"], len(results), pass_rate * 100)
        for r in results:
            status = "PASS" if r["passed"] else "FAIL"
            logger.info("  [%s] %s", status, r.get("task_id", "?"))
        return 0

    # --- Pipeline mode: requires --input and --candidate-id ---
    if not args.input or not args.candidate_id:
        logger.error("--input and --candidate-id required in pipeline mode (use --standalone for direct evaluation)")
        return 1

    # Load inputs
    ranking_artifact = read_json(Path(args.input).expanduser().resolve())
    run_id = ranking_artifact["run_id"]
    target = ranking_artifact.get("target", {})

    # Find candidate
    try:
        candidate = extract_candidate_skill(ranking_artifact, args.candidate_id)
    except ValueError as exc:
        logger.error(str(exc))
        return 1

    # Check if candidate score meets threshold
    candidate_score = candidate.get("score", 0.0)
    if candidate_score < args.eval_threshold:
        logger.info(
            "Candidate score %.1f below threshold %.1f, skipping evaluation",
            candidate_score, args.eval_threshold,
        )
        output_path = _resolve_output(args.output, state_root, run_id)
        artifact = _build_artifact(
            run_id=run_id,
            target=target,
            candidate_id=args.candidate_id,
            verdict="skipped",
            reason=f"score {candidate_score} < threshold {args.eval_threshold}",
            output_path=output_path,
        )
        write_json(output_path, artifact)
        print(str(output_path))
        return 0

    # Load task suite
    suite = load_task_suite(task_suite_path)
    tasks = suite["tasks"]

    runner = TaskRunner(mock=args.mock)

    # --- Run candidate ---
    try:
        candidate_skill = get_skill_content(candidate, target)
    except ValueError as exc:
        logger.error("Cannot get candidate skill content: %s", exc)
        return 1

    logger.info("Running %d tasks for candidate '%s'...", len(tasks), args.candidate_id)
    candidate_results = run_task_suite(runner, candidate_skill, tasks, pass_k=args.pass_k)
    candidate_rate = compute_pass_rate(candidate_results)
    logger.info("Candidate pass rate: %.2f", candidate_rate)

    # --- Run baseline ---
    baseline_results: list[dict] | None = None
    baseline_rate = 0.0

    cache_dir = Path(args.baseline_cache_dir).expanduser().resolve() if args.baseline_cache_dir else None

    try:
        baseline_skill = get_baseline_skill_content(target)
    except ValueError:
        logger.warning("Baseline SKILL.md not found; using 0.0 as baseline")
        baseline_skill = None

    if baseline_skill:
        # Check cache
        cached = None
        cache_key = ""
        if cache_dir:
            cache_key = _baseline_cache_key(baseline_skill, str(task_suite_path))
            cached = load_baseline_cache(cache_dir, cache_key)

        if cached:
            baseline_rate = cached["pass_rate"]
            baseline_results = cached["results"]
            logger.info("Baseline loaded from cache: %.2f", baseline_rate)
        else:
            logger.info("Running %d tasks for baseline...", len(tasks))
            baseline_results = run_task_suite(runner, baseline_skill, tasks, pass_k=args.pass_k)
            baseline_rate = compute_pass_rate(baseline_results)
            logger.info("Baseline pass rate: %.2f", baseline_rate)

            # Save to cache
            if cache_dir and cache_key:
                save_baseline_cache(cache_dir, cache_key, {
                    "pass_rate": baseline_rate,
                    "results": baseline_results,
                    "created_at": utc_now_iso(),
                })

        # Abort if baseline is broken
        if baseline_rate < BASELINE_ABORT_THRESHOLD:
            logger.error(
                "Baseline pass rate %.2f < %.2f threshold. Task suite may be broken.",
                baseline_rate, BASELINE_ABORT_THRESHOLD,
            )
            output_path = _resolve_output(args.output, state_root, run_id)
            artifact = _build_artifact(
                run_id=run_id,
                target=target,
                candidate_id=args.candidate_id,
                verdict="error",
                reason=f"baseline pass rate {baseline_rate} < {BASELINE_ABORT_THRESHOLD}",
                output_path=output_path,
            )
            write_json(output_path, artifact)
            print(str(output_path))
            return 1

    # --- Compute results ---
    evaluation = compute_results(candidate_rate, baseline_rate)

    # --- Build output artifact ---
    output_path = _resolve_output(args.output, state_root, run_id)
    artifact = {
        "schema_version": SCHEMA_VERSION,
        "lane": ranking_artifact.get("lane", "generic-skill"),
        "run_id": run_id,
        "stage": "evaluated",
        "status": "success",
        "created_at": utc_now_iso(),
        "source_ranking_artifact": args.input,
        "candidate_id": args.candidate_id,
        "candidate_score": candidate_score,
        "task_suite": str(task_suite_path),
        "task_suite_skill_id": suite.get("skill_id", ""),
        "pass_k": args.pass_k,
        "evaluation": evaluation,
        "candidate_results": candidate_results,
        "baseline_results": baseline_results,
        "target": target,
        "next_step": "gate_decision",
        "next_owner": "gate",
        "truth_anchor": str(output_path),
    }
    write_json(output_path, artifact)
    print(str(output_path))
    return 0


def _resolve_output(output_arg: str | None, state_root: Path, run_id: str) -> Path:
    """Resolve the output file path."""
    if output_arg:
        return Path(output_arg).expanduser().resolve()
    evaluations_dir = state_root / "evaluations"
    evaluations_dir.mkdir(parents=True, exist_ok=True)
    return evaluations_dir / f"{run_id}.json"


def _build_artifact(
    *,
    run_id: str,
    target: dict,
    candidate_id: str,
    verdict: str,
    reason: str,
    output_path: Path,
) -> dict:
    """Build a minimal evaluation artifact for skip/error cases."""
    return {
        "schema_version": SCHEMA_VERSION,
        "lane": "generic-skill",
        "run_id": run_id,
        "stage": "evaluated",
        "status": verdict,
        "created_at": utc_now_iso(),
        "candidate_id": candidate_id,
        "evaluation": {
            "execution_pass_rate": 0.0,
            "baseline_pass_rate": 0.0,
            "delta": 0.0,
            "verdict": verdict,
            "reason": reason,
        },
        "target": target,
        "next_step": "gate_decision" if verdict == "skipped" else "abort",
        "next_owner": "gate" if verdict == "skipped" else "human",
        "truth_anchor": str(output_path),
    }


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Task execution engine for the improvement evaluator.

Runs a single task with SKILL.md content prepended to the prompt,
evaluates the output with the configured judge, and returns a TaskResult.
"""

from __future__ import annotations

import json
import shutil
import subprocess
import tempfile
import time
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

import sys

_SKILL_ROOT = Path(__file__).resolve().parents[1]
_INTERFACES_DIR = str(_SKILL_ROOT / "interfaces")
if _INTERFACES_DIR not in sys.path:
    sys.path.insert(0, _INTERFACES_DIR)

from judges import get_judge


@dataclass
class TaskResult:
    """Result of running a single task."""
    passed: bool
    judge_output: dict
    raw_output: str
    cost_usd: float = 0.0
    duration_ms: int = 0
    error: str = ""


class TaskRunner:
    """Runs a single task with SKILL.md context."""

    def __init__(self, mock: bool = False, timeout: int = 300):
        self.mock = mock
        self.timeout = timeout

    def run(self, skill_content: str, task: dict, pass_k: int = 1) -> TaskResult:
        """Run a task pass_k times and return the best result.

        Args:
            skill_content: SKILL.md text to prepend to the prompt
            task: Task dict with id, prompt, judge config
            pass_k: Number of attempts; passes if any attempt passes
        """
        prompt = self._build_prompt(skill_content, task)
        timeout = task.get("timeout_seconds", self.timeout)

        best_result: TaskResult | None = None
        for attempt in range(pass_k):
            start = time.monotonic()
            try:
                if self.mock:
                    raw_output = self._mock_execute(prompt)
                else:
                    raw_output = self._execute_claude(prompt, timeout)
            except Exception as exc:
                duration_ms = int((time.monotonic() - start) * 1000)
                result = TaskResult(
                    passed=False,
                    judge_output={"passed": False, "details": str(exc), "score": 0.0},
                    raw_output="",
                    duration_ms=duration_ms,
                    error=str(exc),
                )
                if best_result is None:
                    best_result = result
                continue

            duration_ms = int((time.monotonic() - start) * 1000)

            # Evaluate with judge
            judge = get_judge(task["judge"], mock=self.mock)
            judge_output = judge.evaluate(raw_output, task)

            result = TaskResult(
                passed=judge_output.get("passed", False),
                judge_output=judge_output,
                raw_output=raw_output[:2000],  # truncate for storage
                duration_ms=duration_ms,
            )

            if result.passed:
                return result
            if best_result is None or result.judge_output.get("score", 0) > best_result.judge_output.get("score", 0):
                best_result = result

        return best_result  # type: ignore[return-value]

    def _build_prompt(self, skill_content: str, task: dict) -> str:
        """Prepend SKILL.md content to the task prompt."""
        return (
            f"You have been given the following skill documentation to guide your work:\n\n"
            f"---BEGIN SKILL.MD---\n{skill_content}\n---END SKILL.MD---\n\n"
            f"Task: {task['prompt']}"
        )

    def _execute_claude(self, prompt: str, timeout: int) -> str:
        """Call claude -p subprocess and return the text output."""
        tmpdir = None
        try:
            tmpdir = tempfile.mkdtemp(prefix="evaluator_task_")
            prompt_file = Path(tmpdir) / "prompt.txt"
            prompt_file.write_text(prompt, encoding="utf-8")

            result = subprocess.run(
                ["claude", "-p", "--output-format", "json"],
                input=prompt,
                capture_output=True,
                text=True,
                timeout=timeout,
            )

            if result.returncode != 0:
                raise RuntimeError(f"claude -p exited {result.returncode}: {result.stderr[:300]}")

            # Parse claude JSON output
            try:
                parsed = json.loads(result.stdout)
                return parsed.get("result", result.stdout)
            except (json.JSONDecodeError, TypeError):
                return result.stdout

        except subprocess.TimeoutExpired:
            raise RuntimeError(f"claude -p timed out after {timeout}s")
        finally:
            if tmpdir:
                shutil.rmtree(tmpdir, ignore_errors=True)

    def _mock_execute(self, prompt: str) -> str:
        """Return a mock response for testing without claude CLI."""
        # Return a generic response that includes common keywords
        # for basic ContainsJudge tests
        return (
            "Based on the skill documentation, here is my analysis:\n\n"
            "The quality tier is POWERFUL based on the high accuracy and coverage scores.\n"
            "This indicates strong performance across multiple dimensions.\n\n"
            "Regarding the Pareto front, if accuracy regressed significantly, "
            "we should reject the change as it represents a regression in a key metric. "
            "The coverage improvement does not compensate for the accuracy loss.\n"
            "hello world"
        )

{
  "platform": "ios",
  "from_tag": "0.12.36-ios",
  "to_tag": "0.12.37-ios",
  "commits": [
    {"hash": "345409b29", "message": "fix:修复富文本点击", "files": ["platforms/ios/source/components/RichTextViewHandler.mm"]},
    {"hash": "55dbd8140", "message": "feat: podfile", "files": ["platforms/ios/NanoCompose.podspec"]},
    {"hash": "3449e931b", "message": "fix: 种类错误", "files": ["nano_compose/foundation/components/text.cpp"]},
    {"hash": "7b58ce1e7", "message": "feat: 复用场景下需要回归行高", "files": ["nano_compose/runtime/layout/layout_stage.cpp"]},
    {"hash": "6afb45729", "message": "feat(ios): 新增统一模版获取接口 fetchTemplateWithInfo:fallbackInfo:", "files": ["platforms/ios/source/base/NCEngine.mm", "platforms/ios/source/base/NCEngine.h"]},
    {"hash": "b0e9545a0", "message": "fix: 修复holder可能出现访问野指针的问题", "files": ["nano_compose/runtime/node/origin_node.cpp"]},
    {"hash": "a9ccc62c2", "message": "refactor(ios): 抽取 ceilAlignSize 统一像素对齐 + Scene1 返回 cachedSize", "files": ["platforms/ios/source/render/NCUIHelper.mm"]},
    {"hash": "9c4d18481", "message": "fix(ios): updateLayoutMetrics 中 cachedSize 使用 ceil 像素对齐", "files": ["platforms/ios/source/render/NCLayoutNode.mm"]},
    {"hash": "72838f0b5", "message": "fix: compose_func解析响应式丢失问题", "files": ["nano_compose/binary/bytes_reader.cpp"]}
  ],
  "stats": {
    "total_commits": 9,
    "total_files": 11,
    "categories": {"features": 3, "bugfixes": 5, "refactoring": 1}
  }
}

skill_id: "changelog-gen"
version: "1.0"
description: "Evaluates changelog-gen skill's ability to produce correct, well-structured, platform-isolated release notes"

tasks:
  # === 结构正确性 ===
  - id: "cg-structure-01"
    description: "Release notes must have standard sections: 版本概述, 新增功能, 问题修复"
    prompt: |
      Based on the following commit data for NanoCompose iOS platform release v0.12.37-ios,
      generate release notes in Chinese following the changelog-gen skill format.

      Commit data:
      - fix:修复富文本点击 (RichTextViewHandler.mm)
      - feat: 复用场景下需要回归行高 (layout_stage.cpp)
      - feat(ios): 新增统一模版获取接口 fetchTemplateWithInfo:fallbackInfo: (NCEngine.mm/h)
      - fix: 修复holder可能出现访问野指针的问题 (origin_node.cpp)
      - refactor(ios): 抽取 ceilAlignSize 统一像素对齐 (NCUIHelper.mm)
      - fix(ios): updateLayoutMetrics 中 cachedSize 使用 ceil 像素对齐 (NCLayoutNode.mm)
      - fix: compose_func解析响应式丢失问题 (bytes_reader.cpp)

      Output markdown release notes.
    judge:
      type: "contains"
      expected:
        - "版本概述"
        - "新增功能"
        - "修复"
    timeout_seconds: 120

  # === 平台隔离 ===
  - id: "cg-isolation-01"
    description: "iOS release notes must NOT contain Android/HarmonyOS keywords"
    prompt: |
      Generate iOS platform release notes for NanoCompose v0.12.37-ios based on these commits:
      - fix:修复富文本点击 (platforms/ios/source/components/RichTextViewHandler.mm)
      - feat(ios): 新增统一模版获取接口 (platforms/ios/source/base/NCEngine.mm)
      - fix: 修复holder访问野指针 (nano_compose/runtime/node/origin_node.cpp)
      - fix: compose_func解析响应式丢失 (nano_compose/binary/bytes_reader.cpp)

      IMPORTANT: This is iOS platform only. Do not mention other platforms.
      Output markdown release notes.
    judge:
      type: "contains"
      expected:
        - "iOS"
    timeout_seconds: 120

  - id: "cg-isolation-02"
    description: "iOS release notes must not leak Android/OHOS keywords"
    prompt: |
      Generate iOS platform release notes for NanoCompose v0.12.37-ios.
      Commits:
      - feat(ios): 新增统一模版获取接口 fetchTemplateWithInfo:fallbackInfo:
      - fix: 修复holder可能出现访问野指针的问题 (Core C++ layer)
      - refactor(ios): 抽取 ceilAlignSize 统一像素对齐

      This is iOS-only release notes. Generate in Chinese.
    judge:
      type: "llm-rubric"
      rubric: |
        The release notes are for iOS platform ONLY. Check:
        1. Must NOT contain "Kotlin", "Java", "Android", "ArkTS", "HarmonyOS", "OHOS", "鸿蒙" keywords
        2. Core (C++) changes should be described as "底层核心"/"公共层"/"框架核心", not "Android核心" or "OHOS核心"
        3. Should use iOS-specific terminology (ObjC, Swift, UIKit, etc.) where appropriate
        Score 1.0 if all checks pass, 0.0 if any platform leakage found.
      pass_threshold: 0.8
    timeout_seconds: 120

  # === 分类准确性 ===
  - id: "cg-categorize-01"
    description: "fix: commits should appear in bugfix section, not feature section"
    prompt: |
      Generate release notes categorizing these commits:
      - fix:修复富文本点击
      - fix: 修复holder可能出现访问野指针的问题
      - fix(ios): updateLayoutMetrics 中 cachedSize 使用 ceil 像素对齐
      - fix: compose_func解析响应式丢失问题
      - feat(ios): 新增统一模版获取接口 fetchTemplateWithInfo:fallbackInfo:
      - feat: 复用场景下需要回归行高
      - refactor(ios): 抽取 ceilAlignSize 统一像素对齐

      Separate into: Features, Bugfixes, Refactoring sections.
      Output in Chinese markdown.
    judge:
      type: "contains"
      expected:
        - "修复"
        - "富文本"
        - "野指针"
    timeout_seconds: 120

  # === Commit 链接格式 ===
  - id: "cg-commit-link-01"
    description: "Each change should reference commit hash"
    prompt: |
      Generate release notes for these commits. Include commit hash references.
      - 345409b29 fix:修复富文本点击
      - 6afb45729 feat(ios): 新增统一模版获取接口 fetchTemplateWithInfo:fallbackInfo:
      - b0e9545a0 fix: 修复holder可能出现访问野指针的问题

      Format: include commit hash in parentheses or as link after each item.
    judge:
      type: "contains"
      expected:
        - "345409b"
        - "6afb457"
        - "b0e9545"
    timeout_seconds: 120

  # === 错误检测 ===
  - id: "cg-review-bad-01"
    description: "Should identify platform leakage in bad release notes"
    prompt: |
      Review the following iOS release notes for quality issues:

      ```markdown
      # Release Notes v0.12.37-ios
      ## Changes
      - Fixed some bugs in text rendering
      - Added new Kotlin features for Android platform
      - Improved HarmonyOS ArkTS performance
      - Updated Java dependencies
      - fix: 修复了一些问题
      - feat: 新增了一些功能
      ## Stats
      10 commits, 15 files changed
      ```

      Identify all quality problems in these release notes.
    judge:
      type: "contains"
      expected:
        - "Kotlin"
        - "Android"
    timeout_seconds: 120

  # === 模块分类 ===
  - id: "cg-module-01"
    description: "Files should be classified into correct modules"
    prompt: |
      Classify these changed files into NanoCompose modules:
      - nano_compose/runtime/layout/layout_stage.cpp
      - nano_compose/runtime/node/origin_node.cpp
      - nano_compose/binary/bytes_reader.cpp
      - platforms/ios/source/base/NCEngine.mm
      - platforms/ios/source/render/NCUIHelper.mm
      - platforms/ios/source/components/RichTextViewHandler.mm

      Use these module names: LayoutStage, Node, Binary, iOS-Base, iOS-Render, iOS-Components
    judge:
      type: "contains"
      expected:
        - "LayoutStage"
        - "Node"
        - "Binary"
        - "iOS-Base"
    timeout_seconds: 60

skill_id: "deslop"
version: "1.0"
description: "Evaluates deslop skill's ability to remove AI patterns from text while preserving substance and adding human voice"

tasks:
  # === Tier 1 模式识别与删除 ===
  - id: "ds-tier1-removal"
    description: "Must remove Tier 1 AI patterns: AI vocabulary, significance inflation, copula avoidance"
    prompt: |
      Rewrite the following paragraph to remove AI-generated patterns. Apply the deslop workflow.

      "The NanoCompose framework serves as a groundbreaking solution that delves into the intricate
      landscape of cross-platform rendering. This pivotal initiative showcases how modern technology
      can foster innovation, highlighting the enduring interplay between performance and usability.
      The framework stands as a testament to the team's commitment to excellence, underscoring
      its vital role in the evolving ecosystem of mobile development."

      Output the rewritten version only, no explanations.
    judge:
      type: "contains"
      expected:
        # These AI words must be removed
        - "is"  # copula should be restored from "serves as"
      # Verify via llm-rubric that AI words are gone
    timeout_seconds: 120

  - id: "ds-tier1-verify-removal"
    description: "Rewritten text must NOT contain Tier 1 AI vocabulary words"
    prompt: |
      Rewrite this paragraph to remove all AI-generated patterns:

      "The NanoCompose framework serves as a groundbreaking solution that delves into the intricate
      landscape of cross-platform rendering. This pivotal initiative showcases how modern technology
      can foster innovation, highlighting the enduring interplay between performance and usability."

      Output the rewritten version only.
    judge:
      type: "llm-rubric"
      rubric: |
        Check the rewritten text for Tier 1 AI vocabulary. Score 0.0 if ANY of these words remain:
        delve, tapestry, landscape (abstract), interplay, intricate, groundbreaking, pivotal,
        showcase, underscore, foster, garnering, enduring, testament, serves as, stands as.
        Score 0.5 if 1-2 remain. Score 1.0 if none remain.
        Also check: is the core meaning preserved? If the rewrite lost the original facts, score 0.0.
      pass_threshold: 0.8
    timeout_seconds: 120

  # === 中文特有模式 ===
  - id: "ds-chinese-patterns"
    description: "Must fix Chinese-specific AI patterns: 四字堆砌, 被动过多, 总分总"
    prompt: |
      用 deslop 方法重写以下中文段落，去除 AI 味：

      "我们很高兴地宣布，这套革命性的架构通过高效协同、智能赋能、敏捷迭代、持续优化等最佳实践，
      已被成功实施并在生产环境中得到验证。下面从三个方面进行阐述。第一，性能优化已被全面推进。
      第二，用户体验已被显著提升。第三，系统稳定性已被大幅增强。综上所述，该架构代表了该领域的重要进展。"

      只输出重写后的版本。
    judge:
      type: "llm-rubric"
      rubric: |
        Check the rewritten Chinese text:
        1. No "四字堆砌" (4-char buzzword chains like 高效协同/智能赋能/敏捷迭代/持续优化) — score 0 if >2 remain
        2. Passive voice reduced (已被X → 主动语态) — score 0 if "已被" appears more than once
        3. No "总分总" pattern (下面从X个方面/综上所述) — score 0 if these frames remain
        4. No "革命性"/"最佳实践" buzzwords — score 0 if these remain
        5. Core claims preserved (performance, UX, stability improvements mentioned)
        Score 1.0 if all pass, 0.5 if 3-4 pass, 0.0 if fewer.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === 两次 Pass 执行 ===
  - id: "ds-two-pass"
    description: "Output must show evidence of two-pass workflow (Pass 1 + remaining tells + Pass 2)"
    prompt: |
      Apply the full deslop workflow to this text. You MUST show both passes:

      "Additionally, this innovative approach leverages cutting-edge AI technology to seamlessly
      enhance developer productivity. It's not just about automation; it's about empowering teams
      to unlock their full potential. The future looks bright as we continue our journey toward
      excellence in software engineering."

      Show: Pass 1 rewrite, remaining AI tells, then Pass 2 final version.
    judge:
      type: "contains"
      expected:
        - "Pass 1"
        - "Pass 2"
    timeout_seconds: 120

  # === 评分输出格式 ===
  - id: "ds-score-format"
    description: "Must output original score and final score in the specified format"
    prompt: |
      Apply deslop to this text and output with scores:

      "We are thrilled to announce our revolutionary platform that delves into the vibrant
      landscape of cloud computing. This groundbreaking initiative serves as a testament to
      our unwavering commitment to fostering innovation and showcasing excellence."

      Follow the deslop output format with scores.
    judge:
      type: "contains"
      expected:
        - "/10"
    timeout_seconds: 120

  # === 防过度矫正 ===
  - id: "ds-preserve-substance"
    description: "Must NOT remove technical substance while removing AI patterns"
    prompt: |
      Apply deslop to this technical paragraph. Preserve ALL technical details:

      "The ExprVm executes compiled binary templates with P95 latency under 3ms for simple templates.
      The dual-node architecture (OriginNode for immutable prototypes, LayoutNode for mutable layout)
      achieves zero-copy references. The three-stage pipeline (ExpandTree → Layout → Render) processes
      complex templates in under 16ms. This groundbreaking approach serves as a testament to the
      team's innovative engineering, showcasing the pivotal role of performance optimization."

      Remove AI patterns but keep every technical fact. Output the rewritten version.
    judge:
      type: "contains"
      expected:
        - "P95"
        - "3ms"
        - "OriginNode"
        - "LayoutNode"
        - "ExpandTree"
        - "16ms"
    timeout_seconds: 120

  # === 禁用词检测 ===
  - id: "ds-banned-words"
    description: "Must remove all banned words from the disallow list"
    prompt: |
      Apply deslop to remove AI patterns from this text:

      "This paradigm-shifting solution empowers developers with cutting-edge tools that
      seamlessly integrate into existing workflows. By leveraging synergy between teams and
      unlocking the full potential of the platform, we drive innovation at unprecedented scale.
      Our best practices ensure a transformative experience."

      Output the rewritten version only.
    judge:
      type: "llm-rubric"
      rubric: |
        Check that ALL of these banned words/phrases have been removed or replaced:
        - paradigm / paradigm-shifting
        - empowers / empowering
        - cutting-edge
        - seamlessly / seamless
        - leveraging / leverage
        - synergy
        - unlocking / unlock potential
        - drive innovation
        - best practices
        - transformative
        Score 1.0 if all removed. Score 0.5 if 1-2 remain. Score 0.0 if 3+ remain.
        The replacement should use specific, concrete language instead.
      pass_threshold: 0.8
    timeout_seconds: 120

skill_id: "improvement-gate"
version: "1.0"
description: "Evaluates improvement-gate skill's ability to make correct keep/reject decisions, enforce 6-layer gate ordering, and handle edge cases"

tasks:
  # === 6-Layer Gate Ordering ===
  - id: "ig-layer-ordering"
    description: "Must list the 6 gate layers in the correct order"
    prompt: |
      List the 6 gate layers of the improvement-gate in order from first to last.
      For each layer, state its name and one-sentence pass condition.
    judge:
      type: "contains"
      expected:
        - "SchemaGate"
        - "CompileGate"
        - "LintGate"
        - "RegressionGate"
        - "ReviewGate"
        - "HumanReviewGate"
    timeout_seconds: 60

  # === Keep Decision Scenario ===
  - id: "ig-keep-decision"
    description: "Must correctly identify a keep-eligible candidate"
    prompt: |
      Given this candidate, determine the gate decision (keep/reject/revert/pending_promote):

      - recommendation: accept_for_execution
      - category: "reference"
      - risk_level: "low"
      - target_path: "skills/my-skill/references/patterns.md"
      - execution status: "success"
      - All 6 gate layers passed
      - No human review needed

      What is the gate decision and why?
    judge:
      type: "contains"
      expected:
        - "keep"
    timeout_seconds: 60

  # === Reject Decision Scenario ===
  - id: "ig-reject-on-layer-fail"
    description: "Must reject when a required gate layer fails"
    prompt: |
      A candidate has these gate layer results:
      - SchemaGate: PASSED
      - CompileGate: FAILED (Python syntax error in modified file)
      - LintGate: not reached
      - RegressionGate: not reached
      - ReviewGate: not reached
      - HumanReviewGate: not reached

      The candidate's recommendation was "accept_for_execution" and risk_level "low".
      What is the gate decision? Does the recommendation matter here?
    judge:
      type: "llm-rubric"
      rubric: |
        1. Must state the decision is "revert" (if file was modified) or "reject" (if no file modified). Score 0.0 if it says "keep" or "pending_promote".
        2. Must explain that CompileGate is a required layer, so failure overrides the recommendation. Score 0.0 if it says recommendation still matters.
        3. Must note that subsequent layers are not reached after a required layer fails. Score 0.5 if this is missing but decision is correct.
        Score 1.0 if all correct. Score 0.5 if decision correct but explanation incomplete.
      pass_threshold: 0.7
    timeout_seconds: 60

  # === 5% Regression Tolerance ===
  - id: "ig-regression-tolerance"
    description: "Must correctly apply 5% tolerance for Pareto dimension regression"
    prompt: |
      A candidate's evaluator evidence shows:
      - accuracy dimension: baseline 0.85, candidate 0.82 (3.5% drop)
      - coverage dimension: baseline 0.70, candidate 0.78 (11.4% improvement)

      The RegressionGate checks "No Pareto dimension regressed beyond 5%."
      Does this candidate pass the RegressionGate? Show your reasoning.
    judge:
      type: "llm-rubric"
      rubric: |
        1. Must state that the accuracy dropped by 3.5% which is within the 5% tolerance, so it PASSES the RegressionGate. Score 0.0 if it says the candidate fails RegressionGate.
        2. Must mention the 5% threshold explicitly. Score 0.5 if reasoning is correct but threshold not mentioned.
        3. Should note the coverage improvement as a positive signal. Score 0.8 if passes but doesn't mention coverage.
        Score 1.0 if all correct. Score 0.0 if wrong pass/fail decision.
      pass_threshold: 0.7
    timeout_seconds: 60

  # === Pending Promote Escalation ===
  - id: "ig-pending-promote"
    description: "Must escalate to pending_promote when human review is needed"
    prompt: |
      A candidate passed all 6 mechanical gate layers. However:
      - category: "workflow"
      - risk_level: "medium"
      - recommendation: "accept_for_execution"

      The HumanReviewGate determined this needs human review (needs_human=True).
      What is the final gate decision? What happens to the file change?
    judge:
      type: "contains"
      expected:
        - "pending_promote"
    timeout_seconds: 60

  # === CLI Usage ===
  - id: "ig-cli-commands"
    description: "Must know the correct CLI commands for gate validation and review management"
    prompt: |
      Show the exact CLI commands for these three operations:
      1. Run gate validation on ranking.json and execution.json, output to receipt.json
      2. List pending human reviews from /tmp/state
      3. Approve a review with ID "REQ_001" with reason "low risk documentation change"
    judge:
      type: "contains"
      expected:
        - "gate.py"
        - "--ranking"
        - "--execution"
        - "review.py"
        - "--list"
        - "--complete"
        - "REQ_001"
        - "approve"
    timeout_seconds: 60

  # === Revert vs Reject Distinction ===
  - id: "ig-revert-vs-reject"
    description: "Must distinguish between revert (file modified) and reject (no file modified)"
    prompt: |
      Two candidates both failed the ReviewGate (panel disputed + LLM judge reject):

      Candidate A: execution result shows modified=true, has a rollback_pointer with backup_path
      Candidate B: execution result shows modified=false (executor returned unsupported)

      What is the gate decision for each? What happens with the backup file for Candidate A?
    judge:
      type: "llm-rubric"
      rubric: |
        1. Candidate A must get "revert" decision (file was modified, needs rollback). Score 0.0 if wrong.
        2. Candidate B must get "reject" decision (no file was modified). Score 0.0 if wrong.
        3. Must mention that Candidate A's backup file is restored to the target path. Score 0.5 if decision correct but rollback not mentioned.
        4. Must NOT say both candidates get the same decision. Score 0.0 if it treats them identically.
        Score 1.0 if all correct.
      pass_threshold: 0.7
    timeout_seconds: 60

skill_id: "prompt-hardening"
version: "1.0"
description: "Evaluates prompt-hardening skill's ability to audit prompts, apply 16 hardening patterns, and produce actionable hardening recommendations"

tasks:
  # === P1 Triple Reinforcement Detection ===
  - id: "ph-p1-triple-reinforcement"
    description: "Must identify missing P1 triple reinforcement and recommend MUST/NEVER + example + I REPEAT"
    prompt: |
      Audit this agent prompt rule for hardening gaps. Apply the prompt-hardening methodology.

      ```
      ## File Handling
      Try to avoid creating temporary files. If you do create them, clean them up afterward.
      Prefer using pipes for data transformation.
      ```

      List which hardening patterns (P1-P16) are missing and show the hardened version.
    judge:
      type: "contains"
      expected:
        - "P1"
        - "MUST"
    timeout_seconds: 120

  # === P5 Anti-Reasoning Detection ===
  - id: "ph-p5-anti-reasoning"
    description: "Must detect need for P5 anti-reasoning block when model could rationalize violations"
    prompt: |
      This rule is repeatedly violated by the AI model, which rationalizes that "the user probably wants me to skip this step to save time." Apply prompt-hardening to fix it.

      ```
      Before modifying any source file, run the linter first.
      ```

      Show the hardened version with the specific patterns applied.
    judge:
      type: "llm-rubric"
      rubric: |
        Check the hardened output for P5 anti-reasoning block:
        1. Does it explicitly anticipate the model's rationalization (e.g., "if you find yourself thinking X, STOP")? Score 0.0 if no anti-reasoning block present.
        2. Does it include MUST/NEVER keywords (P1)? Score 0.0 if missing.
        3. Does it pre-empt the specific excuse about "saving time" or "user wants to skip"? Score 0.5 if generic, 1.0 if specific.
        Score 1.0 if all three are present with specific anti-reasoning. Score 0.5 if P5 is present but generic.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Audit Output Format ===
  - id: "ph-audit-output-format"
    description: "Audit must produce 16-item checklist with pass/fail indicators and a score denominator of 16"
    prompt: |
      Perform a prompt hardening audit on this SOUL.md excerpt. Output the 16-point checklist.

      ```
      # Agent Instructions

      You are a code review assistant. Review pull requests and provide feedback.
      Focus on code quality, security, and performance.

      MUST NOT approve PRs with failing tests.

      When reviewing:
      - Check for common security vulnerabilities
      - Verify test coverage
      - Look for performance anti-patterns
      ```
    judge:
      type: "contains"
      expected:
        - "P1"
        - "P2"
        - "P5"
        - "P13"
        - "/16"
    timeout_seconds: 120

  # === CLI Usage Accuracy ===
  - id: "ph-cli-reference"
    description: "Must reference the correct audit.sh CLI command when suggesting audit"
    prompt: |
      A user says: "I have a SOUL.md at ~/projects/my-agent/SOUL.md and I want to check how hardened it is. What command should I run?"

      Provide the exact command.
    judge:
      type: "contains"
      expected:
        - "audit.sh"
        - "SOUL.md"
    timeout_seconds: 60

  # === Pattern Selection for Scenarios ===
  - id: "ph-pattern-selection"
    description: "Must recommend correct pattern combinations for specific violation scenarios"
    prompt: |
      Given these three violation scenarios, recommend which prompt-hardening patterns (P1-P16) to apply for each. Reference the pattern names and explain why.

      Scenario A: The model repeatedly ignores a MUST rule about running tests before committing.
      Scenario B: The model uses grep instead of the Grep tool, despite instructions.
      Scenario C: After 20+ turns in a conversation, the model forgets to update COMPACT_STATE.

      For each scenario, list the primary pattern(s) and the expected reliability improvement.
    judge:
      type: "llm-rubric"
      rubric: |
        Evaluate pattern recommendations:
        - Scenario A (repeated MUST violation): Must recommend P1 (triple reinforcement) + P13 (code-level). Score 0 if neither mentioned.
        - Scenario B (wrong tool usage): Must recommend P2 (tool forcing: Use X NOT Y) or P3 (exhaustive negation). Score 0 if neither mentioned.
        - Scenario C (long conversation drift): Must recommend P9 (drift protection) or P16 (first/last repetition). Score 0 if neither mentioned.
        - Must reference reliability percentages or strength levels from the skill (e.g., ~90%, ~99%). Score bonus 0.1 if present.
        Score 1.0 if all three scenarios have correct primary patterns. Score 0.5 if 2/3 correct. Score 0.0 if 1 or fewer correct.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Reliability Levels Knowledge ===
  - id: "ph-reliability-levels"
    description: "Must correctly cite reliability levels for different pattern combinations"
    prompt: |
      A team is deciding how to protect a critical rule: "NEVER modify production database directly."
      The rule is violated about 30% of the time with current soft wording.

      What combination of prompt-hardening patterns would bring compliance to ~99%? Show the expected reliability at each layer.
    judge:
      type: "contains"
      expected:
        - "P1"
        - "P13"
    timeout_seconds: 120

  # === Hardening a Real Prompt (End-to-End) ===
  - id: "ph-end-to-end-hardening"
    description: "Must produce a fully hardened prompt with multiple patterns applied correctly"
    prompt: |
      Harden this agent rule. The agent currently violates it ~60% of the time. Target: ~95%+ compliance.

      ```
      When the user asks to deploy, check the CI status first. If CI is red, tell the user. Don't deploy if tests are failing.
      ```

      Output the hardened version. Show which patterns (P1-P16) you applied and why.
    judge:
      type: "llm-rubric"
      rubric: |
        Evaluate the hardened output:
        1. Contains MUST/NEVER keywords (P1): +0.2
        2. Has example/anti-example pair showing correct vs incorrect behavior (P7): +0.2
        3. Has conditional trigger format "When X -> MUST Y" (P4): +0.2
        4. Has anti-reasoning block addressing potential bypass rationale (P5): +0.2
        5. The original meaning is fully preserved (deploy check + CI status): +0.2
        Score the sum. Must score >= 0.6 to pass.
      pass_threshold: 0.6
    timeout_seconds: 120

skill_id: "skill-creator"
version: "1.0"
description: "Evaluates skill-creator's ability to generate symptom-driven descriptions, correct frontmatter, example/anti-example pairs, and enforce progressive disclosure"

tasks:
  # === Symptom-Driven Description ===
  - id: "sc-symptom-description"
    description: "Must generate description with detectable symptoms/scenarios, not abstract capabilities"
    prompt: |
      I want to create a skill for diagnosing memory leaks in C++ applications.
      Write just the YAML frontmatter (name, description, triggers) for this skill.

      The description should follow the symptom-driven pattern so that when a user describes
      their problem, the skill triggers correctly.
    judge:
      type: "llm-rubric"
      rubric: |
        Check the description field:
        1. Must contain detectable symptoms or scenarios (e.g., "RSS grows", "Instruments shows leak", "allocator mismatch", "OOM crash"), NOT just "handles memory leaks". Score 0.0 if purely abstract like "manages memory issues".
        2. Must be in the format that skill routing can match against user problem descriptions. Score 0.5 if has some symptoms but mostly capability-focused.
        3. Triggers array must contain user-facing symptom keywords (e.g., "leak", "RSS", "OOM", "memory grows"). Score 0.0 if triggers are empty or only contain developer-facing terms.
        4. Should include "When to use" / "When NOT to use" guidance via the description or trigger hints. Score 0.8 if symptoms present but no negative boundary.
        Score 1.0 if description is clearly symptom-driven with 2+ concrete symptoms.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Frontmatter Structure ===
  - id: "sc-frontmatter-format"
    description: "Must produce valid YAML frontmatter with all required fields"
    prompt: |
      Create the complete YAML frontmatter block for a new skill called "api-rate-limiter"
      that helps implement rate limiting in REST APIs.

      Output ONLY the frontmatter block (between --- delimiters), nothing else.
    judge:
      type: "contains"
      expected:
        - "---"
        - "name:"
        - "description:"
        - "triggers:"
        - "api-rate-limiter"
    timeout_seconds: 60

  # === Example / Anti-Example Pair Quality ===
  - id: "sc-example-pair"
    description: "Must generate example and anti-example with reasoning tags"
    prompt: |
      Write an <example> and <anti-example> pair for a skill called "cache-invalidation"
      that helps design cache invalidation strategies for distributed systems.

      The example should show correct skill usage, and the anti-example should show a common
      misuse. Include reasoning for why each is correct/incorrect.
    judge:
      type: "llm-rubric"
      rubric: |
        Check the example/anti-example pair:
        1. Must use <example> and <anti-example> XML tags. Score 0.0 if plain text without tags.
        2. Example must show a concrete cache invalidation scenario (not generic). Score 0.0 if vague.
        3. Anti-example must show a realistic misuse (e.g., using the skill for caching strategy selection when it's about invalidation, or using it for in-memory cache when it's for distributed). Score 0.0 if anti-example is trivially wrong.
        4. Must include reasoning that explains WHY the example is correct and WHY the anti-example is wrong. Score 0.5 if examples present but no reasoning.
        5. The anti-example must be something a real user might actually try (not a strawman). Score 0.8 if reasoning present but anti-example is unrealistic.
        Score 1.0 if both examples concrete + reasoning + realistic anti-example.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Progressive Disclosure Compliance ===
  - id: "sc-progressive-disclosure"
    description: "Must structure skill with correct 3-level progressive disclosure"
    prompt: |
      I have 800 lines of content for a new "database-migration" skill. The content includes:
      - Core workflow (5 steps, ~100 lines)
      - Migration patterns (rollback, zero-downtime, blue-green) (~300 lines)
      - Database-specific guides (PostgreSQL, MySQL, MongoDB) (~400 lines)

      How should this be structured following the progressive disclosure principle?
      Show the directory structure and explain what goes where.
    judge:
      type: "contains"
      expected:
        - "SKILL.md"
        - "references/"
        - "500"
    timeout_seconds: 120

  # === Atomicity Violation Detection ===
  - id: "sc-atomicity-check"
    description: "Must detect atomicity violations when skill content crosses responsibility boundaries"
    prompt: |
      Review this proposed skill structure for atomicity violations:

      Skill: "ios-testing-expert"
      SKILL.md contains:
      - Section 1: iOS-specific test setup (XCTest, Xcode schemes) — 100 lines
      - Section 2: Complete TDD methodology (Red-Green-Refactor workflow) — 150 lines
      - Section 3: Full unit test design patterns (boundary testing, mocking) — 200 lines
      - Section 4: iOS test debugging (lldb, breakpoints) — 80 lines
      references/:
      - problem-fix-references/troubleshooting-decision-tree.md — copied from @problem-fix

      Identify any atomicity violations.
    judge:
      type: "llm-rubric"
      rubric: |
        Must identify these atomicity violations:
        1. Section 2 (TDD methodology) belongs to a tdd-workflow or similar skill, not ios-testing-expert. Must flag this as cross-responsibility content. Score 0.0 if not caught.
        2. Section 3 (unit test design patterns) belongs to a unit-test-design skill. Must flag this. Score 0.0 if not caught.
        3. The references/ file copied from @problem-fix violates "path independent" and "self-contained" rules. Must flag this. Score 0.0 if not caught.
        4. Sections 1 and 4 (iOS-specific content) are correctly scoped. Should confirm these are fine. Score 0.5 if all violations caught but correct sections not acknowledged.
        5. Must recommend declarative references (e.g., "See @tdd-workflow for methodology") instead of copying content. Score 0.8 if violations caught but fix not suggested.
        Score 1.0 if all 3 violations caught + fix recommendations.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Freedom-Level Matching ===
  - id: "sc-freedom-matching"
    description: "Must correctly assign freedom levels (high/medium/low) to different rule types"
    prompt: |
      For a new "deployment-checklist" skill, classify these rules by freedom level
      (high/medium/low) and recommend the appropriate expression format for each:

      Rule A: "Before deploying, think about potential rollback scenarios"
      Rule B: "Run the pre-deploy health check script at scripts/health_check.sh"
      Rule C: "Choose an appropriate deployment strategy for your use case"
      Rule D: "Database migrations MUST be backward-compatible and MUST run before code deploy"
    judge:
      type: "llm-rubric"
      rubric: |
        Check freedom level assignments:
        1. Rule A (think about rollback): Should be HIGH freedom (multiple valid approaches) → text guidance. Score 0.0 if assigned low.
        2. Rule B (run specific script): Should be LOW freedom (specific fragile operation) → concrete script path. Score 0.0 if assigned high.
        3. Rule C (choose strategy): Should be HIGH freedom (many valid options) → text guidance. Score 0.0 if assigned low.
        4. Rule D (backward-compatible + ordering): Should be LOW or MEDIUM freedom (strict constraint) → MUST/NEVER with examples or pseudocode. Score 0.0 if assigned high.
        5. Must mention the three expression formats: text guidance, pseudocode/parameterized script, concrete script. Score 0.5 if levels correct but formats not mentioned.
        Score 1.0 if all 4 rules correctly classified with appropriate formats.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Init Script Knowledge ===
  - id: "sc-init-script"
    description: "Must reference the init_skill.py script for initializing new skills"
    prompt: |
      I want to create a new skill called "log-analyzer" at ~/.claude/skills/log-analyzer.
      What is the correct way to initialize the skill directory structure?
      Show the command and explain what it creates.
    judge:
      type: "contains"
      expected:
        - "init_skill.py"
        - "log-analyzer"
    timeout_seconds: 60

skill_id: "skill-distill"
version: "1.0"
description: "Evaluates skill-distill's ability to analyze overlap between skills, classify content, enforce 4-phase workflow, and respect token budgets"

tasks:
  # === Overlap Analysis ===
  - id: "sd-overlap-detection"
    description: "Must correctly identify overlap, unique contributions, and redundancy across source skills"
    prompt: |
      Analyze these three hypothetical writing-related skills for distillation potential:

      Skill A "prose-cleaner":
      - Two-pass rewrite workflow (first remove AI patterns, then add voice)
      - AI vocabulary blacklist (50 words)
      - Preserves technical facts
      - English only

      Skill B "zh-writer":
      - Two-pass rewrite workflow (same as A)
      - Chinese-specific patterns (four-char buzzword chains, passive voice)
      - Template for formal documents
      - Bilingual support

      Skill C "tone-tuner":
      - Single-pass rewrite
      - Voice calibration (formal/casual/technical spectrum)
      - AI vocabulary blacklist (30 words, subset of A's list)
      - English only

      Classify each piece of content as: intersection, unique_contribution, conflict, or redundant.
      Output a structured analysis table.
    judge:
      type: "llm-rubric"
      rubric: |
        Check the analysis:
        1. "Two-pass rewrite workflow" must be classified as intersection (A+B share it). Score 0.0 if missed.
        2. "Chinese-specific patterns" must be classified as unique_contribution (only B). Score 0.0 if missed.
        3. "Voice calibration" must be classified as unique_contribution (only C). Score 0.0 if missed.
        4. C's blacklist (subset of A's) must be classified as redundant or intersection. Score 0.0 if classified as unique.
        5. C's "single-pass" vs A/B's "two-pass" should be flagged as a conflict. Score 0.5 if missed.
        6. Each classification must cite source skill(s). Score 0.5 if classifications correct but sources missing.
        Score 1.0 if all 5 core items correct with sources.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === 4-Phase Workflow Compliance ===
  - id: "sd-workflow-phases"
    description: "Must follow the 4-phase workflow: Collect, Analyze, User Confirm, Generate+Verify"
    prompt: |
      I want to distill "http-fetcher" and "web-scraper" skills into one skill. Both deal with fetching web content but http-fetcher is about raw HTTP requests while web-scraper handles DOM parsing and extraction.

      Start the distillation process. Show me what Phase 1 (Collect) output looks like.
    judge:
      type: "contains"
      expected:
        - "Phase 1"
        - "Inventory"
        - "Lines"
        - "Unique contribution"
    timeout_seconds: 120

  # === Distillation Rejection ===
  - id: "sd-reject-non-overlapping"
    description: "Must refuse to distill skills that solve different problems"
    prompt: |
      I want to distill these two skills into one:
      - "cpp-expert": C++20 core development, memory safety, performance optimization
      - "ios-expert": iOS platform ObjC++/Swift, UIKit integration, Apple APIs

      They are often used together in our project. Should we distill them?
    judge:
      type: "llm-rubric"
      rubric: |
        Must refuse the distillation. Check:
        1. Must clearly say NO / not recommended / should not distill. Score 0.0 if it proceeds with distillation.
        2. Must explain that the skills solve different problems (C++ core vs iOS platform). Score 0.5 if refusal is generic.
        3. Should reference the anti-example or criteria: "NEVER distill skills with different responsibilities." Score 0.8 if refuses with good reasoning but doesn't cite the rule.
        4. Should note that "used together" != "overlap" — complementary is not the same as redundant. Score bonus +0.1.
        Score 1.0 if clear refusal with correct reasoning.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Token Budget Awareness ===
  - id: "sd-token-budget"
    description: "Must enforce SKILL.md body <=500 lines and route long content to references/"
    prompt: |
      I'm distilling 3 skills. The analysis shows:
      - Intersection content: ~300 lines (after dedup)
      - Unique contributions from Skill A: ~150 lines of workflow rules
      - Unique contributions from Skill B: ~200 lines of pattern catalog
      - Unique contributions from Skill C: ~100 lines of scoring rubric

      Total: ~750 lines of valuable content. None is redundant.
      How should the distilled skill be structured to respect token budgets?
    judge:
      type: "llm-rubric"
      rubric: |
        Must address the 500-line SKILL.md body limit:
        1. Must mention the <=500 line body constraint. Score 0.0 if it ignores the limit.
        2. Must propose moving some content to references/ files. Score 0.0 if everything goes in SKILL.md body.
        3. Should suggest keeping intersection (~300 lines) + summaries of unique contributions in body, with full unique content in references/. Score 0.5 if split is proposed but not well justified.
        4. Should show a proposed file structure with references/ directory. Score bonus +0.1.
        Score 1.0 if body stays <=500 lines with clear references/ split.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === User Confirmation Gate ===
  - id: "sd-user-confirmation"
    description: "Must require user confirmation before generating files (Phase 3)"
    prompt: |
      I've reviewed your Phase 2 analysis for distilling skills A and B. I see the overlap table and the proposed structure. Go ahead and generate the distilled skill files now.

      Context: You already showed me Phase 1 Inventory and Phase 2 Analysis. Phase 3 requires showing the distillation plan and getting explicit user confirmation before Phase 4 generation.

      What is the correct next step?
    judge:
      type: "llm-rubric"
      rubric: |
        Must enforce Phase 3 user confirmation:
        1. Must show a distillation plan (target skill name, body content summary, references list, conflicts, dropped items). Score 0.0 if it skips straight to generating files.
        2. Must explicitly ask for user confirmation before proceeding to Phase 4. Score 0.0 if it auto-generates.
        3. Should mention source skill handling options (preserve/replace/downgrade). Score 0.5 if plan shown but source handling missing.
        Score 1.0 if full plan + explicit confirmation request. Score 0.0 if it generates files without confirmation.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === Pipeline Verification ===
  - id: "sd-pipeline-verification"
    description: "Must require improvement-learner scoring after generation (Phase 4 verification)"
    prompt: |
      I've confirmed the distillation plan. You generated the new skill at ~/.claude/skills/web-toolkit/SKILL.md with 3 references files.

      What verification steps must happen next? List the exact commands.
    judge:
      type: "contains"
      expected:
        - "self_improve.py"
        - "accuracy"
        - "0.80"
    timeout_seconds: 120

  # === Distillation Signals ===
  - id: "sd-worth-distilling"
    description: "Must correctly evaluate whether a set of skills is worth distilling based on signals"
    prompt: |
      Evaluate these three cases and state whether distillation is recommended:

      Case 1: Skills "json-validator" and "schema-checker" — 70% trigger overlap, users confused which to use, both loaded for same tasks.

      Case 2: Skills "test-runner" and "coverage-reporter" — 10% trigger overlap, test-runner runs tests, coverage-reporter analyzes results afterward. Sequential workflow.

      Case 3: Skills "css-linter" and "style-guide" — 40% trigger overlap, css-linter has 600 lines body, style-guide has 400 lines body. Combined would be ~900 lines after dedup.

      For each: distill or not? Why?
    judge:
      type: "llm-rubric"
      rubric: |
        1. Case 1: Must recommend YES (high overlap, user confusion, co-loading = classic distillation target). Score 0.0 if no.
        2. Case 2: Must recommend NO (different workflow stages, sequential not overlapping). Score 0.0 if yes.
        3. Case 3: Must recommend NO or CAUTION (combined body would exceed 500 lines even after dedup). Score 0.0 if yes without addressing size.
        Score 1.0 if all three correct with reasoning.
      pass_threshold: 0.7
    timeout_seconds: 120

skill_id: "tech-article"
version: "1.0"
description: "Evaluates tech-article skill's ability to guide technical article creation through the full Phase 1-5 workflow"

tasks:
  # === Phase 1: 定题能力 ===
  - id: "ta-phase1-topic-analysis"
    description: "Must identify article type and core question from a vague topic"
    prompt: |
      /tech-article "我们团队花了两个月把monolith拆成微服务"

      只完成 Phase 1（定题），输出核心问题、文章类型、目标读者、独特价值。不要写大纲或正文。
    judge:
      type: "llm-rubric"
      rubric: |
        Check the Phase 1 output:
        1. 核心问题 is a specific, answerable question (not generic like "微服务实践")
        2. 文章类型 correctly identified as "案例复盘" (not "教程" or "原理解析")
        3. 目标读者 is defined with assumed knowledge level
        4. 独特价值 identifies what's unique vs generic microservice articles
        Score 1.0 if all 4 present and reasonable. Score 0.5 if 2-3 present. Score 0.0 if fewer.
      pass_threshold: 0.7
    timeout_seconds: 60

  # === Phase 1: 模糊输入处理 ===
  - id: "ta-vague-input"
    description: "Must ask clarifying questions when input lacks concrete material"
    prompt: |
      /tech-article "写一篇关于性能优化的文章"
    judge:
      type: "llm-rubric"
      rubric: |
        The input is deliberately vague — no specific experience, data, or code mentioned.
        The skill MUST ask clarifying questions before proceeding. Check:
        1. Does it ask about the user's specific experience/project? (核心经历)
        2. Does it ask about available data/code/screenshots? (数据证据)
        3. Does it ask about target audience? (目标读者)
        4. Does it NOT proceed directly to writing an outline or draft?
        Score 1.0 if questions asked and no premature writing. Score 0.5 if questions asked but also started writing. Score 0.0 if jumped straight to outline/draft.
      pass_threshold: 0.7
    timeout_seconds: 60

  # === Phase 2: 大纲生成 ===
  - id: "ta-outline-structure"
    description: "Must generate outline with HOOK, BODY sections, and CLOSE"
    prompt: |
      /tech-article --outline-only "我们的 API 网关从 Nginx 迁移到 Envoy，QPS 从 5万提升到 20万，但中间踩了 5 个大坑"

      生成大纲，不写正文。
    judge:
      type: "contains"
      expected:
        - "HOOK"
        - "CLOSE"
        - "标题"
        - "类型"
    timeout_seconds: 120

  # === Phase 2: HOOK 质量 ===
  - id: "ta-hook-quality"
    description: "HOOK must contain specific data and conflict/suspense, not generic opening"
    prompt: |
      /tech-article --outline-only "我们把 CI/CD 从 Jenkins 迁移到 GitHub Actions，构建时间从 45 分钟降到 8 分钟，但迁移过程中丢了 3 天的部署历史"

      只生成大纲。
    judge:
      type: "llm-rubric"
      rubric: |
        Check the HOOK section of the outline:
        1. Contains specific numbers from the input (45分钟, 8分钟, 3天)
        2. Contains conflict/suspense element (not just "本文介绍...")
        3. Is concise (under 100 words / 3 sentences)
        4. Does NOT start with "随着..." / "在当今..." / "本文将..."
        Score 1.0 if all pass. Score 0.5 if 2-3 pass. Score 0.0 if generic opening.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === --review 模式 ===
  - id: "ta-review-mode"
    description: "Review mode must analyze an existing draft without rewriting it"
    prompt: |
      /tech-article --review

      请审查以下草稿：

      # 浅谈微服务架构

      随着云计算技术的快速发展，微服务架构越来越受到企业的广泛关注和重视。本文将从以下几个方面深入探讨微服务的最佳实践。

      ## 什么是微服务

      微服务是一种将应用程序构建为小型服务集合的架构风格。每个服务运行在自己的进程中，并使用轻量级机制（通常是 HTTP API）进行通信。

      ## 总结

      综上所述，微服务架构是现代软件开发的重要趋势，值得每个团队深入学习和实践。
    judge:
      type: "llm-rubric"
      rubric: |
        Check the review output:
        1. Identifies the title as problematic ("浅谈" is a weak pattern)
        2. Flags the generic opening ("随着...越来越...")
        3. Flags the weak conclusion ("综上所述...值得...")
        4. Flags the lack of concrete data/evidence
        5. Does NOT rewrite the article — only provides review feedback
        6. Provides actionable suggestions with specifics
        Score 1.0 if 5-6 checks pass. Score 0.5 if 3-4 pass. Score 0.0 if fewer.
      pass_threshold: 0.7
    timeout_seconds: 120

  # === deslop fallback ===
  - id: "ta-deslop-awareness"
    description: "Must reference deslop in Phase 5 or mention fallback if unavailable"
    prompt: |
      /tech-article "我们用 Redis Cluster 替换了单机 Redis，QPS 从 10万提升到 50万"

      只描述你会如何执行 Phase 5（润色），不要写正文。列出具体步骤。
    judge:
      type: "contains"
      expected:
        - "deslop"
    timeout_seconds: 60

skill_id: "benchmark-store"
version: "1.0"
tasks:
  - id: "bs-quality-tier-01"
    description: "Should identify correct quality tier"
    prompt: "Given these scores {accuracy: 0.9, coverage: 0.85}, what quality tier is this skill?"
    judge:
      type: "contains"
      expected: ["POWERFUL"]
    timeout_seconds: 30

  - id: "bs-pareto-01"
    description: "Should explain Pareto front regression"
    prompt: "A skill's accuracy dropped from 0.9 to 0.8 but coverage went up from 0.7 to 0.95. Should this change be accepted?"
    judge:
      type: "contains"
      expected: ["regress", "reject"]
    timeout_seconds: 30

#!/usr/bin/env python3
"""Tests for the evaluate module's core functions."""

import json
import sys
from pathlib import Path
from unittest.mock import patch

import pytest

# Add paths for imports
_SKILL_ROOT = Path(__file__).resolve().parents[1]
_REPO_ROOT = _SKILL_ROOT.parents[2]
sys.path.insert(0, str(_SKILL_ROOT / "scripts"))
sys.path.insert(0, str(_REPO_ROOT))

from evaluate import (
    load_task_suite,
    preflight_check,
    compute_results,
    compute_pass_rate,
    extract_candidate_skill,
    _validate_suite_schema,
    BASELINE_ABORT_THRESHOLD,
)


class TestLoadTaskSuite:
    def test_valid_suite(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test-skill\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    description: test\n"
            "    prompt: do something\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["hello"]\n'
        )
        suite = load_task_suite(suite_file)
        assert suite["skill_id"] == "test-skill"
        assert len(suite["tasks"]) == 1
        assert suite["tasks"][0]["id"] == "t1"

    def test_missing_skill_id(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do something\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["hello"]\n'
        )
        with pytest.raises(AssertionError, match="skill_id"):
            load_task_suite(suite_file)

    def test_wrong_version(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "2.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["x"]\n'
        )
        with pytest.raises(AssertionError, match="version"):
            load_task_suite(suite_file)

    def test_duplicate_task_ids(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["x"]\n'
            "  - id: t1\n"
            "    prompt: do again\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["y"]\n'
        )
        with pytest.raises(AssertionError, match="duplicate"):
            load_task_suite(suite_file)

    def test_empty_tasks(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks: []\n"
        )
        with pytest.raises(AssertionError, match="not be empty"):
            load_task_suite(suite_file)

    def test_unknown_judge_type(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: magic\n"
        )
        with pytest.raises(AssertionError, match="unknown judge type"):
            load_task_suite(suite_file)

    def test_contains_missing_expected(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: contains\n"
        )
        with pytest.raises(AssertionError, match="expected"):
            load_task_suite(suite_file)

    def test_pytest_non_fixture_path(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: pytest\n"
            "      test_file: evil/path.py\n"
        )
        with pytest.raises(AssertionError, match="fixtures/"):
            load_task_suite(suite_file)


class TestPreflightCheck:
    def test_no_claude_without_mock(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["x"]\n'
        )
        with patch("shutil.which", return_value=None):
            with pytest.raises(AssertionError, match="claude CLI"):
                preflight_check(suite_file, mock=False)

    def test_mock_skips_claude_check(self, tmp_path):
        suite_file = tmp_path / "suite.yaml"
        suite_file.write_text(
            "skill_id: test\n"
            'version: "1.0"\n'
            "tasks:\n"
            "  - id: t1\n"
            "    prompt: do\n"
            "    judge:\n"
            "      type: contains\n"
            '      expected: ["x"]\n'
        )
        # Should not raise even without claude CLI
        preflight_check(suite_file, mock=True)

    def test_missing_suite_file(self, tmp_path):
        with pytest.raises(AssertionError, match="not found"):
            preflight_check(tmp_path / "nonexistent.yaml", mock=True)


class TestComputeResults:
    def test_pass_when_candidate_better(self):
        result = compute_results(0.8, 0.7)
        assert result["verdict"] == "pass"
        assert result["delta"] == 0.1
        assert result["execution_pass_rate"] == 0.8
        assert result["baseline_pass_rate"] == 0.7

    def test_pass_when_equal(self):
        result = compute_results(0.7, 0.7)
        assert result["verdict"] == "pass"
        assert result["delta"] == 0.0

    def test_fail_when_candidate_worse(self):
        result = compute_results(0.5, 0.7)
        assert result["verdict"] == "fail"
        assert result["delta"] == -0.2

    def test_zero_baseline(self):
        result = compute_results(0.5, 0.0)
        assert result["verdict"] == "pass"
        assert result["delta"] == 0.5


class TestComputePassRate:
    def test_all_passed(self):
        results = [{"passed": True}, {"passed": True}, {"passed": True}]
        assert compute_pass_rate(results) == 1.0

    def test_none_passed(self):
        results = [{"passed": False}, {"passed": False}]
        assert compute_pass_rate(results) == 0.0

    def test_mixed(self):
        results = [{"passed": True}, {"passed": False}, {"passed": True}, {"passed": False}]
        assert compute_pass_rate(results) == 0.5

    def test_empty(self):
        assert compute_pass_rate([]) == 0.0


class TestExtractCandidateSkill:
    def test_found(self):
        artifact = {
            "scored_candidates": [
                {"id": "c1", "score": 7.0, "content": "# Skill"},
                {"id": "c2", "score": 5.0, "content": "# Other"},
            ]
        }
        c = extract_candidate_skill(artifact, "c1")
        assert c["id"] == "c1"
        assert c["content"] == "# Skill"

    def test_not_found(self):
        artifact = {"scored_candidates": [{"id": "c1", "score": 7.0}]}
        with pytest.raises(ValueError, match="not found"):
            extract_candidate_skill(artifact, "c99")

    def test_empty_candidates(self):
        artifact = {"scored_candidates": []}
        with pytest.raises(ValueError, match="not found"):
            extract_candidate_skill(artifact, "c1")


class TestBaselineShortCircuit:
    def test_threshold_constant(self):
        """Verify the baseline abort threshold is reasonable."""
        assert BASELINE_ABORT_THRESHOLD == 0.2
        assert 0 < BASELINE_ABORT_THRESHOLD < 1