Improvement Orchestrator

[
  {
    "type": "coverage",
    "succeeded": true,
    "context": {
      "dimension": "coverage",
      "scores": {
        "coverage": 1.0,
        "accuracy": 0.8,
        "efficiency": 1.0,
        "reliability": 1.0,
        "security": 0.8
      }
    },
    "timestamp": "2026-04-02T12:14:16Z",
    "hit_count": 1
  },
  {
    "type": "instruction",
    "succeeded": true,
    "context": {
      "dimension": "accuracy",
      "scores": {
        "coverage": 1.0,
        "accuracy": 0.8,
        "efficiency": 1.0,
        "reliability": 1.0,
        "security": 0.8
      }
    },
    "timestamp": "2026-04-02T12:14:17Z",
    "hit_count": 1
  }
]

#!/usr/bin/env python3
"""Orchestrator dispatch for the auto-improvement pipeline.

Coordinates the full PROPOSE → DISCRIMINATE → EVALUATE → EXECUTE → GATE loop
with Ralph Wiggum-style retry: on revert, capture a structured failure trace
and feed it back into the next proposal round.
"""

from __future__ import annotations

import argparse
import json
import subprocess
import sys
from pathlib import Path
from typing import Any

_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))
REPO_ROOT = _REPO_ROOT

from lib.common import read_json, utc_now_iso, write_json

# ---------------------------------------------------------------------------
# Script paths (relative to repo root)
# ---------------------------------------------------------------------------

GENERATOR_SCRIPT = REPO_ROOT / "skills" / "improvement-generator" / "scripts" / "propose.py"
DISCRIMINATOR_SCRIPT = REPO_ROOT / "skills" / "improvement-discriminator" / "scripts" / "score.py"
EVALUATOR_SCRIPT = REPO_ROOT / "skills" / "improvement-evaluator" / "scripts" / "evaluate.py"
EXECUTOR_SCRIPT = REPO_ROOT / "skills" / "improvement-executor" / "scripts" / "execute.py"
GATE_SCRIPT = REPO_ROOT / "skills" / "improvement-gate" / "scripts" / "gate.py"

# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------


def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Orchestrate the full auto-improvement pipeline",
    )
    parser.add_argument("--target", required=True, help="Target skill/file path")
    parser.add_argument(
        "--source",
        action="append",
        default=[],
        help="Feedback/memory source (repeatable)",
    )
    parser.add_argument("--state-root", required=True, help="State directory root")
    parser.add_argument(
        "--max-retries",
        type=int,
        default=3,
        help="Max retry attempts on revert (default: 3)",
    )
    parser.add_argument(
        "--auto",
        action="store_true",
        help="Run full pipeline without pausing",
    )
    parser.add_argument(
        "--task-suite",
        help="Path to task_suite.yaml for evaluator (enables real LLM evaluation)",
    )
    parser.add_argument(
        "--eval-mock",
        action="store_true",
        help="Run evaluator in mock mode (no claude -p calls)",
    )
    return parser.parse_args(argv)


# ---------------------------------------------------------------------------
# Subprocess helpers
# ---------------------------------------------------------------------------


def _run_script(cmd: list[str], label: str) -> str:
    """Run a subprocess and return its stdout (stripped).

    Raises RuntimeError on non-zero exit.
    """
    result = subprocess.run(cmd, capture_output=True, text=True, timeout=1200)
    if result.returncode != 0:
        raise RuntimeError(
            f"{label} failed (exit {result.returncode}):\n"
            f"  stdout: {result.stdout.strip()}\n"
            f"  stderr: {result.stderr.strip()}"
        )
    return result.stdout.strip()


def run_script(script: Path, args: list[str], label: str) -> Path:
    """Run a script with args and return the artifact path as a Path.

    Higher-level wrapper around subprocess that:
    - Raises RuntimeError (with stderr) on non-zero exit.
    - Raises RuntimeError when stdout is blank.
    - Returns the last non-blank line of stdout as a resolved Path.
    """
    cmd = [sys.executable, str(script)] + args
    result = subprocess.run(cmd, capture_output=True, text=True, timeout=1200)
    if result.returncode != 0:
        raise RuntimeError(
            f"{label} failed (exit {result.returncode}):\n"
            f"  stdout: {result.stdout.strip()}\n"
            f"  stderr: {result.stderr.strip()}"
        )
    stdout = result.stdout.strip()
    if not stdout:
        raise RuntimeError(f"{label} produced no output")
    path = Path(stdout)
    if not path.is_absolute():
        path = Path.cwd() / path
    return path.resolve()


def run_proposer(
    target: str,
    sources: list[str],
    state_root: str,
    trace: str | None = None,
) -> dict[str, Any]:
    """Call propose.py and return the candidate artifact."""
    cmd = [
        sys.executable,
        str(GENERATOR_SCRIPT),
        "--target",
        str(target),
        "--state-root",
        str(state_root),
    ]
    for s in sources:
        cmd.extend(["--source", str(s)])
    if trace:
        cmd.extend(["--trace", str(trace)])
    artifact_path = _run_script(cmd, "proposer")
    return read_json(Path(artifact_path))


def run_discriminator(
    candidate_artifact_path: str,
    state_root: str,
) -> dict[str, Any]:
    """Call score.py and return the ranking artifact."""
    cmd = [
        sys.executable,
        str(DISCRIMINATOR_SCRIPT),
        "--input",
        str(candidate_artifact_path),
        "--state-root",
        str(state_root),
    ]
    artifact_path = _run_script(cmd, "discriminator")
    return read_json(Path(artifact_path))


def run_executor(
    ranking_artifact_path: str,
    candidate_id: str,
    state_root: str,
) -> dict[str, Any]:
    """Call execute.py and return the execution artifact."""
    cmd = [
        sys.executable,
        str(EXECUTOR_SCRIPT),
        "--input",
        str(ranking_artifact_path),
        "--candidate-id",
        candidate_id,
        "--state-root",
        str(state_root),
    ]
    artifact_path = _run_script(cmd, "executor")
    return read_json(Path(artifact_path))


def run_evaluator(
    ranking_artifact_path: str,
    candidate_id: str,
    state_root: str,
    task_suite: str | None = None,
    eval_threshold: float = 6.0,
    mock: bool = False,
) -> dict[str, Any] | None:
    """Call evaluate.py if a task suite exists for the target skill.

    Returns evaluation artifact dict, or None if skipped (no task suite
    or evaluator script not found).
    """
    if not EVALUATOR_SCRIPT.exists():
        print("  Evaluator: skipped (script not found)")
        return None
    if not task_suite:
        print("  Evaluator: skipped (no --task-suite provided)")
        return None
    cmd = [
        sys.executable,
        str(EVALUATOR_SCRIPT),
        "--input", str(ranking_artifact_path),
        "--candidate-id", candidate_id,
        "--task-suite", str(task_suite),
        "--state-root", str(state_root),
        "--eval-threshold", str(eval_threshold),
    ]
    if mock:
        cmd.append("--mock")
    try:
        artifact_path = _run_script(cmd, "evaluator")
        result = read_json(Path(artifact_path))
        evaluation = result.get("evaluation", {})
        verdict = evaluation.get("verdict", result.get("verdict", "skipped"))
        if verdict == "skipped":
            print("  Evaluator: skipped (no task suite)")
            return None
        print(f"  Evaluator: {verdict} (pass_rate={evaluation.get('execution_pass_rate', 'N/A')})")
        return result
    except RuntimeError as exc:
        print(f"  Evaluator: error ({exc}), continuing without evaluation")
        return None


def run_gate(
    ranking_artifact_path: str,
    execution_artifact_path: str,
    state_root: str,
) -> dict[str, Any]:
    """Call gate.py and return the gate receipt."""
    cmd = [
        sys.executable,
        str(GATE_SCRIPT),
        "--ranking",
        str(ranking_artifact_path),
        "--execution",
        str(execution_artifact_path),
        "--state-root",
        str(state_root),
    ]
    artifact_path = _run_script(cmd, "gate")
    return read_json(Path(artifact_path))


# ---------------------------------------------------------------------------
# Candidate selection
# ---------------------------------------------------------------------------


def find_best_accepted(ranking_artifact: dict[str, Any]) -> dict[str, Any] | None:
    """Return the highest-scored candidate with recommendation=accept_for_execution.

    Candidates in the ranking artifact are already sorted by score (descending),
    so the first match is the best.
    """
    for candidate in ranking_artifact.get("scored_candidates", []):
        if candidate.get("recommendation") == "accept_for_execution":
            return candidate
    return None


# ---------------------------------------------------------------------------
# Failure trace (Ralph Wiggum feedback loop)
# ---------------------------------------------------------------------------


def extract_failure_trace(
    receipt: dict[str, Any],
    execution_artifact: dict[str, Any],
    state_root: str,
) -> str:
    """Extract structured failure trace and write it to a file.

    The returned path can be passed as --trace to the next proposer round,
    enabling the Ralph Wiggum retry pattern: failures feed forward as
    context for the next attempt.
    """
    trace = {
        "type": "failure_trace",
        "candidate_id": receipt.get("candidate_id"),
        "decision": receipt.get("decision"),
        "reason": receipt.get("reason"),
        "execution_status": execution_artifact.get("result", {}).get("status"),
        "diff": execution_artifact.get("result", {}).get("diff", ""),
        "gate_blockers": receipt.get("blockers", []),
        "timestamp": utc_now_iso(),
    }
    run_id = receipt.get("run_id", "unknown")
    trace_dir = Path(state_root) / "traces"
    trace_path = trace_dir / f"trace-{run_id}.json"
    write_json(trace_path, trace)
    return str(trace_path)


# ---------------------------------------------------------------------------
# Pipeline
# ---------------------------------------------------------------------------


def run_baseline_evaluation(
    target: str,
    task_suite: str,
    state_root: str,
    mock: bool = False,
) -> str | None:
    """Run evaluator in standalone mode on current SKILL.md to get baseline failures.

    Returns path to a feedback source file containing per-task failure details,
    or None if no failures found or evaluator unavailable.
    """
    if not EVALUATOR_SCRIPT.exists():
        return None

    cmd = [
        sys.executable,
        str(EVALUATOR_SCRIPT),
        "--standalone",
        "--task-suite", str(task_suite),
        "--state-root", str(state_root),
        "--skill-path", str(target),
    ]
    if mock:
        cmd.append("--mock")

    try:
        stdout = _run_script(cmd, "baseline-evaluator")
        result = read_json(Path(stdout))
    except RuntimeError as exc:
        print(f"  Baseline evaluation failed ({exc}), continuing without feedback")
        return None

    # Extract failed tasks as feedback for generator
    task_results = result.get("task_results", [])
    failed_tasks = [r for r in task_results if not r.get("passed")]
    if not failed_tasks:
        print(f"  Baseline: all tasks passed, no failures to feed back")
        return None

    pass_rate = result.get("evaluation", {}).get("pass_rate", 0)
    print(f"  Baseline: {len(failed_tasks)} task(s) failed (pass_rate={pass_rate})")

    # Write failure details as a source file for generator
    feedback = {
        "type": "evaluator_baseline_failures",
        "pass_rate": pass_rate,
        "failed_tasks": [],
        "timestamp": utc_now_iso(),
    }
    for t in failed_tasks:
        feedback["failed_tasks"].append({
            "task_id": t.get("task_id"),
            "score": t.get("score", 0),
            "details": t.get("details", ""),
            "error": t.get("error", ""),
        })

    feedback_dir = Path(state_root) / "traces"
    feedback_dir.mkdir(parents=True, exist_ok=True)
    feedback_path = feedback_dir / "baseline-failures.json"
    write_json(feedback_path, feedback)
    return str(feedback_path)


def run_pipeline(
    target: str,
    sources: list[str],
    state_root: str,
    max_retries: int = 3,
    task_suite: str | None = None,
    eval_mock: bool = False,
) -> dict[str, Any]:
    """Run the full PROPOSE → DISCRIMINATE → EVALUATE → EXECUTE → GATE loop.

    Returns a summary dict with the final outcome.
    """
    # Mutable copy so we can append failure traces across retries
    active_sources = list(sources)
    active_trace: str | None = None
    final_decision = "no_candidates"
    final_candidate_id: str | None = None
    final_artifact_path: str | None = None
    attempts_used = 0

    # 0. BASELINE EVALUATION — run evaluator on current SKILL.md first
    #    to discover which tasks fail, then inject as feedback for generator
    if task_suite:
        baseline_feedback = run_baseline_evaluation(
            target, task_suite, state_root, mock=eval_mock,
        )
        if baseline_feedback:
            active_sources.append(baseline_feedback)

    for attempt in range(1, max_retries + 1):
        attempts_used = attempt
        # 1. PROPOSE
        candidate_artifact = run_proposer(target, active_sources, state_root, trace=active_trace)
        candidate_artifact_path = candidate_artifact.get("truth_anchor", "")

        # 2. DISCRIMINATE (score + rank)
        ranking_artifact = run_discriminator(candidate_artifact_path, state_root)
        ranking_artifact_path = ranking_artifact.get("truth_anchor", "")

        # 3. Find best accepted candidate
        best = find_best_accepted(ranking_artifact)
        if not best:
            if attempt >= max_retries:
                final_decision = "no_accepted_candidates"
                print(f"  No accepted candidates after {attempt} attempts")
                break
            # Inject rejection trace for next round
            trace_dir = Path(state_root) / "traces"
            trace_dir.mkdir(parents=True, exist_ok=True)
            rejection_trace = {
                "type": "all_rejected_trace",
                "scored_candidates": [
                    {"id": c.get("id"), "score": c.get("score"), "recommendation": c.get("recommendation")}
                    for c in ranking_artifact.get("scored_candidates", [])
                ],
                "timestamp": utc_now_iso(),
            }
            trace_path = trace_dir / f"rejection-trace-attempt-{attempt}.json"
            write_json(trace_path, rejection_trace)
            active_trace = str(trace_path)
            print(f"  No accepted candidates, retrying ({attempt}/{max_retries})")
            continue

        candidate_id = best["id"]

        # 3.5 EVALUATE (optional — skipped if no --task-suite provided)
        eval_result = run_evaluator(
            ranking_artifact_path,
            candidate_id,
            state_root,
            task_suite=task_suite,
            mock=eval_mock,
        )
        eval_verdict = eval_result.get("evaluation", {}).get("verdict") if eval_result else None
        if eval_verdict == "fail":
            # Evaluation failed — treat as revert and retry
            trace = {
                "type": "evaluation_failure_trace",
                "candidate_id": candidate_id,
                "evaluation_results": eval_result.get("evaluation", {}),
                "timestamp": utc_now_iso(),
            }
            trace_dir = Path(state_root) / "traces"
            trace_dir.mkdir(parents=True, exist_ok=True)
            trace_path = trace_dir / f"eval-trace-{candidate_id}.json"
            write_json(trace_path, trace)
            active_trace = str(trace_path)
            print(f"  Evaluation failed, retrying ({attempt}/{max_retries})")
            continue

        # 4. EXECUTE
        execution_artifact = run_executor(
            ranking_artifact_path,
            candidate_id,
            state_root,
        )
        execution_artifact_path = execution_artifact.get("truth_anchor", "")

        # 5. GATE (verify after execution)
        receipt = run_gate(
            ranking_artifact_path,
            execution_artifact_path,
            state_root,
        )

        # 6. DECIDE
        decision = receipt.get("decision", "reject")
        final_decision = decision
        final_candidate_id = candidate_id
        final_artifact_path = receipt.get("truth_anchor")

        if decision == "keep":
            print(f"  Kept: {candidate_id}")
            break
        elif decision == "revert":
            # Ralph Wiggum: capture trace, inject into next round
            trace_path = extract_failure_trace(
                receipt, execution_artifact, state_root,
            )
            active_trace = trace_path
            print(
                f"  Reverted, retrying ({attempt}/{max_retries})"
            )
            continue
        elif decision == "pending_promote":
            print(f"  Pending human review: {candidate_id}")
            break
        else:  # reject
            print(f"  Rejected: {candidate_id}")
            break

    return {
        "target": target,
        "attempts": attempts_used,
        "max_retries": max_retries,
        "final_decision": final_decision,
        "final_candidate_id": final_candidate_id,
        "final_artifact_path": final_artifact_path,
    }


# ---------------------------------------------------------------------------
# Summary report
# ---------------------------------------------------------------------------


def print_summary(summary: dict[str, Any]) -> None:
    """Print a human-readable pipeline summary."""
    print("\nPipeline Summary:")
    print(f"  Target: {summary['target']}")
    print(f"  Attempts: {summary['attempts']}/{summary['max_retries']}")
    print(f"  Final Decision: {summary['final_decision']}")
    print(f"  Candidate: {summary.get('final_candidate_id', 'N/A')}")
    print(f"  Artifact: {summary.get('final_artifact_path', 'N/A')}")


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------


def main(argv: list[str] | None = None) -> int:
    args = parse_args(argv)
    target = str(Path(args.target).expanduser().resolve())
    state_root = str(Path(args.state_root).expanduser().resolve())
    sources = [str(Path(s).expanduser().resolve()) for s in args.source if s]

    task_suite = str(Path(args.task_suite).expanduser().resolve()) if args.task_suite else None

    try:
        summary = run_pipeline(
            target=target,
            sources=sources,
            state_root=state_root,
            max_retries=args.max_retries,
            task_suite=task_suite,
            eval_mock=args.eval_mock,
        )
    except RuntimeError as exc:
        print(f"Pipeline error: {exc}", file=sys.stderr)
        return 1

    print_summary(summary)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Tests for the orchestrator dispatch pipeline."""

from __future__ import annotations

import json
import sys
import textwrap
from pathlib import Path
from unittest.mock import MagicMock, patch

import pytest

_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))
REPO_ROOT = _REPO_ROOT

# ---------------------------------------------------------------------------
# Import the module directly via importlib (scripts dir is not a package).
# ---------------------------------------------------------------------------

import importlib.util

_spec = importlib.util.spec_from_file_location(
    "orchestrate",
    REPO_ROOT / "skills" / "improvement-orchestrator" / "scripts" / "orchestrate.py",
)
orchestrate = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(orchestrate)


# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------


@pytest.fixture
def tmp_state(tmp_path):
    """Create a minimal state tree for testing."""
    for sub in ("candidate_versions", "rankings", "executions", "state", "receipts", "traces"):
        (tmp_path / sub).mkdir()
    # Seed state files so ensure_tree doesn't fail
    (tmp_path / "state" / "current_state.json").write_text("{}")
    (tmp_path / "state" / "pending_promote.json").write_text('{"pending":[], "last_updated":null}')
    (tmp_path / "state" / "veto.json").write_text('{"vetoes":[], "last_updated":null}')
    (tmp_path / "state" / "last_run.json").write_text("{}")
    return tmp_path


def _make_ranking_artifact(candidates: list[dict]) -> dict:
    """Build a minimal ranking artifact with scored_candidates."""
    return {
        "run_id": "test-run-001",
        "target": {"path": "/fake/target"},
        "scored_candidates": candidates,
        "truth_anchor": "/fake/ranking.json",
    }


def _make_execution_artifact(
    candidate_id: str = "cand-01-docs",
    status: str = "success",
    modified: bool = True,
    diff: str = "--- a\n+++ b\n@@ -1 +1,2 @@\n+new line",
) -> dict:
    return {
        "run_id": "test-run-001",
        "candidate_id": candidate_id,
        "result": {
            "status": status,
            "modified": modified,
            "diff": diff,
        },
        "truth_anchor": "/fake/execution.json",
    }


def _make_receipt(
    candidate_id: str = "cand-01-docs",
    decision: str = "keep",
    reason: str = "low-risk docs candidate",
    run_id: str = "test-run-001",
    blockers: list | None = None,
) -> dict:
    return {
        "run_id": run_id,
        "candidate_id": candidate_id,
        "decision": decision,
        "reason": reason,
        "blockers": blockers or [],
        "truth_anchor": "/fake/receipt.json",
    }


# ---------------------------------------------------------------------------
# Tests: find_best_accepted
# ---------------------------------------------------------------------------


class TestFindBestAccepted:
    def test_returns_first_accepted(self):
        ranking = _make_ranking_artifact([
            {"id": "cand-01", "score": 8.0, "recommendation": "accept_for_execution"},
            {"id": "cand-02", "score": 7.0, "recommendation": "accept_for_execution"},
            {"id": "cand-03", "score": 6.0, "recommendation": "hold"},
        ])
        best = orchestrate.find_best_accepted(ranking)
        assert best is not None
        assert best["id"] == "cand-01"

    def test_returns_none_when_no_accepted(self):
        ranking = _make_ranking_artifact([
            {"id": "cand-01", "score": 5.0, "recommendation": "hold"},
            {"id": "cand-02", "score": 3.0, "recommendation": "reject"},
        ])
        best = orchestrate.find_best_accepted(ranking)
        assert best is None

    def test_returns_none_for_empty_candidates(self):
        ranking = _make_ranking_artifact([])
        best = orchestrate.find_best_accepted(ranking)
        assert best is None

    def test_skips_non_accepted(self):
        ranking = _make_ranking_artifact([
            {"id": "cand-01", "score": 9.0, "recommendation": "hold"},
            {"id": "cand-02", "score": 7.5, "recommendation": "accept_for_execution"},
        ])
        best = orchestrate.find_best_accepted(ranking)
        assert best["id"] == "cand-02"


# ---------------------------------------------------------------------------
# Tests: extract_failure_trace
# ---------------------------------------------------------------------------


class TestExtractFailureTrace:
    def test_writes_trace_file(self, tmp_state):
        receipt = _make_receipt(
            decision="revert",
            reason="execution failed validation",
            blockers=["risk_medium"],
        )
        execution = _make_execution_artifact(status="error", diff="some diff")

        trace_path = orchestrate.extract_failure_trace(
            receipt, execution, str(tmp_state),
        )

        path = Path(trace_path)
        assert path.exists()
        assert path.suffix == ".json"

        trace = json.loads(path.read_text())
        assert trace["type"] == "failure_trace"
        assert trace["candidate_id"] == "cand-01-docs"
        assert trace["decision"] == "revert"
        assert trace["reason"] == "execution failed validation"
        assert trace["execution_status"] == "error"
        assert trace["diff"] == "some diff"
        assert trace["gate_blockers"] == ["risk_medium"]
        assert "timestamp" in trace

    def test_trace_path_uses_run_id(self, tmp_state):
        receipt = _make_receipt(run_id="my-special-run-42", decision="revert")
        execution = _make_execution_artifact()

        trace_path = orchestrate.extract_failure_trace(
            receipt, execution, str(tmp_state),
        )
        assert "my-special-run-42" in trace_path

    def test_handles_missing_fields_gracefully(self, tmp_state):
        receipt = {"run_id": "run-x"}
        execution = {}

        trace_path = orchestrate.extract_failure_trace(
            receipt, execution, str(tmp_state),
        )

        trace = json.loads(Path(trace_path).read_text())
        assert trace["type"] == "failure_trace"
        assert trace["candidate_id"] is None
        assert trace["decision"] is None
        assert trace["execution_status"] is None


# ---------------------------------------------------------------------------
# Tests: run_pipeline (with mocked subprocesses)
# ---------------------------------------------------------------------------


def _mock_subprocess_side_effects(
    tmp_state: Path,
    decisions: list[str],
    *,
    has_accepted: bool = True,
):
    """Build a subprocess.run side-effect function that simulates
    the full pipeline across multiple attempts.

    Each call to subprocess.run returns a CompletedProcess with stdout
    pointing to a temporary artifact file.

    ``decisions`` controls what the gate returns on each attempt.
    """
    call_counter = {"n": 0, "attempt": 0}

    def side_effect(cmd, **kwargs):
        script_name = cmd[1] if len(cmd) > 1 else ""

        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stderr = ""

        if "propose.py" in script_name:
            attempt = call_counter["attempt"]
            artifact_path = tmp_state / "candidate_versions" / f"run-attempt-{attempt}.json"
            artifact = {
                "run_id": f"run-attempt-{attempt}",
                "stage": "proposed",
                "status": "success",
                "target": {"path": "/fake/target"},
                "candidates": [
                    {
                        "id": "cand-01-docs",
                        "category": "docs",
                        "risk_level": "low",
                        "executor_support": True,
                        "target_path": "/fake/target/README.md",
                    },
                ],
                "truth_anchor": str(artifact_path),
            }
            artifact_path.write_text(json.dumps(artifact))
            mock_result.stdout = str(artifact_path)

        elif "score.py" in script_name:
            attempt = call_counter["attempt"]
            artifact_path = tmp_state / "rankings" / f"run-attempt-{attempt}.json"
            rec = "accept_for_execution" if has_accepted else "hold"
            artifact = {
                "run_id": f"run-attempt-{attempt}",
                "stage": "ranked",
                "status": "success",
                "target": {"path": "/fake/target"},
                "scored_candidates": [
                    {
                        "id": "cand-01-docs",
                        "score": 8.5,
                        "recommendation": rec,
                        "category": "docs",
                        "risk_level": "low",
                        "target_path": "/fake/target/README.md",
                    },
                ],
                "truth_anchor": str(artifact_path),
            }
            artifact_path.write_text(json.dumps(artifact))
            mock_result.stdout = str(artifact_path)

        elif "execute.py" in script_name:
            attempt = call_counter["attempt"]
            artifact_path = tmp_state / "executions" / f"run-attempt-{attempt}.json"
            artifact = {
                "run_id": f"run-attempt-{attempt}",
                "candidate_id": "cand-01-docs",
                "stage": "executed",
                "status": "success",
                "result": {
                    "status": "success",
                    "modified": True,
                    "diff": "+new line",
                },
                "truth_anchor": str(artifact_path),
            }
            artifact_path.write_text(json.dumps(artifact))
            mock_result.stdout = str(artifact_path)

        elif "gate.py" in script_name:
            attempt = call_counter["attempt"]
            decision = decisions[attempt] if attempt < len(decisions) else "reject"
            artifact_path = tmp_state / "receipts" / f"gate-run-attempt-{attempt}.json"
            receipt = {
                "run_id": f"run-attempt-{attempt}",
                "candidate_id": "cand-01-docs",
                "decision": decision,
                "reason": f"test decision: {decision}",
                "blockers": [],
                "truth_anchor": str(artifact_path),
            }
            artifact_path.write_text(json.dumps(receipt))
            mock_result.stdout = str(artifact_path)
            call_counter["attempt"] += 1

        return mock_result

    return side_effect


class TestRunPipeline:
    # Patch EVALUATOR_SCRIPT to a non-existent path so run_evaluator() skips
    # (the evaluator is optional and returns None when script doesn't exist).
    _no_evaluator = patch.object(
        orchestrate, "EVALUATOR_SCRIPT", Path("/nonexistent/evaluate.py"),
    )

    def test_keep_on_first_attempt(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(tmp_state, ["keep"])

        with self._no_evaluator, patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        assert summary["final_decision"] == "keep"
        assert summary["attempts"] == 1
        assert summary["final_candidate_id"] == "cand-01-docs"

    def test_revert_then_keep(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(
            tmp_state, ["revert", "keep"],
        )

        with self._no_evaluator, patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        assert summary["final_decision"] == "keep"
        assert summary["attempts"] == 2

        # Verify trace file was created for the revert
        traces = list((tmp_state / "traces").glob("trace-*.json"))
        assert len(traces) == 1

    def test_pending_promote_stops_loop(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(
            tmp_state, ["pending_promote"],
        )

        with self._no_evaluator, patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        assert summary["final_decision"] == "pending_promote"
        assert summary["attempts"] == 1

    def test_reject_stops_loop(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(
            tmp_state, ["reject"],
        )

        with self._no_evaluator, patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        assert summary["final_decision"] == "reject"
        assert summary["attempts"] == 1

    def test_max_retries_exhausted(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(
            tmp_state, ["revert", "revert", "revert"],
        )

        with self._no_evaluator, patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        # After 3 reverts the loop ends; last decision is "revert"
        assert summary["final_decision"] == "revert"
        assert summary["attempts"] == 3

        # All 3 traces should exist
        traces = list((tmp_state / "traces").glob("trace-*.json"))
        assert len(traces) == 3

    def test_no_accepted_candidates_exits_early(self, tmp_state):
        side_effect = _mock_subprocess_side_effects(
            tmp_state, ["keep"], has_accepted=False,
        )

        with patch("subprocess.run", side_effect=side_effect):
            summary = orchestrate.run_pipeline(
                target="/fake/target",
                sources=[],
                state_root=str(tmp_state),
                max_retries=3,
            )

        assert summary["final_decision"] == "no_accepted_candidates"
        # Pipeline retries up to max_retries before giving up
        assert summary["attempts"] == 3

    def test_failure_trace_fed_back_as_source(self, tmp_state):
        """Verify that after a revert the trace path is passed via --trace."""
        call_log = {"proposer_calls": []}

        original_side_effect = _mock_subprocess_side_effects(
            tmp_state, ["revert", "keep"],
        )

        def tracking_side_effect(cmd, **kwargs):
            script_name = cmd[1] if len(cmd) > 1 else ""
            if "propose.py" in script_name:
                # Capture --source and --trace args
                sources_in_cmd = []
                trace_in_cmd = None
                for i, arg in enumerate(cmd):
                    if arg == "--source" and i + 1 < len(cmd):
                        sources_in_cmd.append(cmd[i + 1])
                    if arg == "--trace" and i + 1 < len(cmd):
                        trace_in_cmd = cmd[i + 1]
                call_log["proposer_calls"].append({
                    "sources": sources_in_cmd,
                    "trace": trace_in_cmd,
                })
            return original_side_effect(cmd, **kwargs)

        with self._no_evaluator, patch("subprocess.run", side_effect=tracking_side_effect):
            orchestrate.run_pipeline(
                target="/fake/target",
                sources=["/original/source.md"],
                state_root=str(tmp_state),
                max_retries=3,
            )

        # First call: only original source, no trace
        assert call_log["proposer_calls"][0]["sources"] == ["/original/source.md"]
        assert call_log["proposer_calls"][0]["trace"] is None

        # Second call: same source, plus a --trace pointing to trace file
        second_call = call_log["proposer_calls"][1]
        assert second_call["sources"] == ["/original/source.md"]
        assert second_call["trace"] is not None
        assert "trace-" in second_call["trace"]


# ---------------------------------------------------------------------------
# Tests: _run_script error handling
# ---------------------------------------------------------------------------


class TestRunScript:
    def test_raises_on_nonzero_exit(self):
        mock_result = MagicMock()
        mock_result.returncode = 1
        mock_result.stdout = "some output"
        mock_result.stderr = "some error"

        with patch("subprocess.run", return_value=mock_result):
            with pytest.raises(RuntimeError, match="proposer failed"):
                orchestrate._run_script(["python3", "fake.py"], "proposer")



class TestRunScriptNew:
    """Tests for the new run_script function."""

    def test_raises_on_nonzero_exit(self):
        mock_result = MagicMock()
        mock_result.returncode = 1
        mock_result.stdout = ""
        mock_result.stderr = "traceback: something broke"
        with patch("subprocess.run", return_value=mock_result):
            with pytest.raises(RuntimeError, match="executor failed"):
                orchestrate.run_script(Path("fake.py"), [], "executor")

    def test_raises_on_empty_stdout(self):
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = "   \n  \n  "
        mock_result.stderr = ""
        with patch("subprocess.run", return_value=mock_result):
            with pytest.raises(RuntimeError, match="produced no output"):
                orchestrate.run_script(Path("fake.py"), [], "scorer")

    def test_returns_absolute_path_when_exists(self, tmp_path):
        artifact = tmp_path / "artifact.json"
        artifact.write_text("{}")
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = f"  \n{artifact}\n"
        mock_result.stderr = ""
        with patch("subprocess.run", return_value=mock_result):
            result = orchestrate.run_script(Path("fake.py"), ["--flag"], "proposer")
        assert result == artifact

    def test_resolves_relative_path_via_cwd(self, tmp_path, monkeypatch):
        monkeypatch.chdir(tmp_path)
        artifact = tmp_path / "out" / "result.json"
        artifact.parent.mkdir()
        artifact.write_text("{}")
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = "out/result.json\n"
        mock_result.stderr = ""
        with patch("subprocess.run", return_value=mock_result):
            result = orchestrate.run_script(Path("fake.py"), [], "gate")
        assert result.exists()
        assert result.name == "result.json"

    def test_stderr_included_in_error_message(self):
        mock_result = MagicMock()
        mock_result.returncode = 2
        mock_result.stdout = ""
        mock_result.stderr = "ImportError: no module named foo"
        with patch("subprocess.run", return_value=mock_result):
            with pytest.raises(RuntimeError) as exc_info:
                orchestrate.run_script(Path("fake.py"), [], "proposer")
        assert "ImportError: no module named foo" in str(exc_info.value)

    def test_returns_path_even_when_nonexistent(self):
        mock_result = MagicMock()
        mock_result.returncode = 0
        mock_result.stdout = "/nonexistent/artifact.json\n"
        mock_result.stderr = ""
        with patch("subprocess.run", return_value=mock_result):
            result = orchestrate.run_script(Path("fake.py"), [], "executor")
        assert result == Path("/nonexistent/artifact.json")

# ---------------------------------------------------------------------------
# Tests: CLI parse_args
# ---------------------------------------------------------------------------


class TestParseArgs:
    def test_required_args(self):
        args = orchestrate.parse_args([
            "--target", "/some/skill",
            "--state-root", "/some/state",
        ])
        assert args.target == "/some/skill"
        assert args.state_root == "/some/state"
        assert args.max_retries == 3
        assert args.auto is False
        assert args.source == []

    def test_all_args(self):
        args = orchestrate.parse_args([
            "--target", "/some/skill",
            "--source", "/a.md",
            "--source", "/b.md",
            "--state-root", "/some/state",
            "--max-retries", "5",
            "--auto",
        ])
        assert args.source == ["/a.md", "/b.md"]
        assert args.max_retries == 5
        assert args.auto is True

"""Tests for the orchestrator state machine and common helpers."""

from __future__ import annotations

import json
import sys
from pathlib import Path

import pytest

# Add repo root to path so we can import lib.*
_REPO_ROOT = Path(__file__).resolve().parents[3]
if str(_REPO_ROOT) not in sys.path:
    sys.path.insert(0, str(_REPO_ROOT))

from lib.common import (
    slugify,
    utc_now_iso,
    compact_timestamp,
    protected_target,
    classify_feedback,
    infer_source_kind,
    compute_target_profile,
    read_json,
    write_json,
    KEEP_CATEGORIES,
    EXECUTOR_SUPPORTED_CATEGORIES,
)
from lib.state_machine import (
    ensure_tree,
    next_step_for_stage,
    update_state,
    append_pending_promote,
    append_veto,
)


# --- slugify ---


class TestSlugify:
    def test_simple(self):
        assert slugify("Hello World") == "hello-world"

    def test_special_chars(self):
        assert slugify("my_skill/v2.0") == "my-skill-v2-0"

    def test_empty(self):
        assert slugify("") == "item"

    def test_trailing_special(self):
        assert slugify("---test---") == "test"


# --- timestamp helpers ---


class TestTimestamps:
    def test_utc_now_iso_format(self):
        result = utc_now_iso()
        assert result.endswith("Z")
        assert "T" in result

    def test_compact_timestamp_format(self):
        result = compact_timestamp()
        assert result.endswith("Z")
        assert "T" in result
        assert len(result) == 16  # YYYYMMDDTHHMMSSz


# --- state machine transitions ---


class TestStateMachine:
    def test_proposed_goes_to_critic(self):
        step, owner = next_step_for_stage("proposed")
        assert step == "rank_candidates"
        assert owner == "critic"

    def test_ranked_goes_to_evaluator(self):
        step, owner = next_step_for_stage("ranked")
        assert step == "evaluate_candidate"
        assert owner == "evaluator"

    def test_evaluated_goes_to_executor(self):
        step, owner = next_step_for_stage("evaluated")
        assert step == "execute_candidate"
        assert owner == "executor"

    def test_executed_goes_to_gate(self):
        step, owner = next_step_for_stage("executed")
        assert step == "apply_gate"
        assert owner == "gate"

    def test_gated_keep_cycles_back(self):
        step, owner = next_step_for_stage("gated_keep")
        assert step == "propose_candidates"
        assert owner == "proposer"

    def test_gated_pending_needs_human(self):
        step, owner = next_step_for_stage("gated_pending")
        assert owner == "human"

    def test_gated_revert_goes_to_proposer(self):
        step, owner = next_step_for_stage("gated_revert")
        assert owner == "proposer"

    def test_gated_reject_goes_to_proposer(self):
        step, owner = next_step_for_stage("gated_reject")
        assert owner == "proposer"

    def test_unknown_stage_falls_back(self):
        step, owner = next_step_for_stage("nonexistent")
        assert step == "inspect_state"
        assert owner == "human"


# --- ensure_tree ---


class TestEnsureTree:
    def test_creates_directory_structure(self, tmp_path):
        root = tmp_path / "test_state"
        paths = ensure_tree(root)
        assert paths["root"].exists()
        assert paths["candidate_versions"].is_dir()
        assert paths["rankings"].is_dir()
        assert paths["executions"].is_dir()
        assert paths["state"].is_dir()
        assert paths["receipts"].is_dir()

    def test_creates_state_files(self, tmp_path):
        root = tmp_path / "test_state"
        paths = ensure_tree(root)
        state_dir = paths["state"]
        assert (state_dir / "current_state.json").exists()
        assert (state_dir / "pending_promote.json").exists()
        assert (state_dir / "veto.json").exists()
        assert (state_dir / "last_run.json").exists()

    def test_initial_state_is_idle(self, tmp_path):
        root = tmp_path / "test_state"
        paths = ensure_tree(root)
        state = read_json(paths["state"] / "current_state.json")
        assert state["status"] == "idle"
        assert state["stage"] == "idle"
        assert state["next_owner"] == "proposer"

    def test_idempotent(self, tmp_path):
        root = tmp_path / "test_state"
        ensure_tree(root)
        ensure_tree(root)  # should not raise


# --- update_state ---


class TestUpdateState:
    def test_updates_current_and_last_run(self, tmp_path):
        root = tmp_path / "state"
        ensure_tree(root)
        update_state(
            root,
            run_id="run-001",
            stage="proposed",
            status="completed",
            target_path="/some/skill",
            truth_anchor="/some/artifact.json",
        )
        current = read_json(root / "state" / "current_state.json")
        assert current["current_run_id"] == "run-001"
        assert current["stage"] == "proposed"
        assert current["next_owner"] == "critic"

        last_run = read_json(root / "state" / "last_run.json")
        assert last_run["last_run_id"] == "run-001"
        assert last_run["last_stage"] == "proposed"

    def test_full_cycle(self, tmp_path):
        """Simulate a full Proposer→Critic→Executor→Gate cycle."""
        root = tmp_path / "state"
        ensure_tree(root)

        stages = [
            ("proposed", "completed"),
            ("ranked", "completed"),
            ("evaluated", "completed"),
            ("executed", "completed"),
            ("gated_keep", "completed"),
        ]
        expected_owners = ["critic", "evaluator", "executor", "gate", "proposer"]

        for (stage, status), expected_owner in zip(stages, expected_owners):
            update_state(
                root,
                run_id="run-cycle",
                stage=stage,
                status=status,
                target_path="/skill",
                truth_anchor=f"/artifact-{stage}.json",
            )
            current = read_json(root / "state" / "current_state.json")
            assert current["next_owner"] == expected_owner, f"After {stage}"


# --- pending_promote / veto ---


class TestPendingAndVeto:
    def test_append_pending_promote(self, tmp_path):
        root = tmp_path / "state"
        ensure_tree(root)
        append_pending_promote(root, {"id": "c1", "reason": "hold"})
        append_pending_promote(root, {"id": "c2", "reason": "hold"})
        data = read_json(root / "state" / "pending_promote.json")
        assert len(data["pending"]) == 2
        assert data["last_updated"] is not None

    def test_append_veto(self, tmp_path):
        root = tmp_path / "state"
        ensure_tree(root)
        append_veto(root, {"id": "c3", "reason": "high risk"})
        data = read_json(root / "state" / "veto.json")
        assert len(data["vetoes"]) == 1


# --- protected_target ---


class TestProtectedTarget:
    def test_trading_is_protected(self):
        assert protected_target("/skills/trading-bot") is True

    def test_gateway_is_protected(self):
        assert protected_target("/infra/gateway-config") is True

    def test_normal_skill_not_protected(self):
        assert protected_target("/skills/summarize") is False


# --- classify_feedback ---


class TestClassifyFeedback:
    def test_basic_classification(self):
        entries = [
            {"snippet": "This test case fails on edge cases"},
            {"snippet": "The workflow step is unclear"},
            {"snippet": "Add a usage example for beginners"},
        ]
        result = classify_feedback(entries)
        assert len(result["tests"]) >= 1
        assert len(result["workflow"]) >= 1
        assert len(result["examples"]) >= 1


# --- infer_source_kind ---


class TestInferSourceKind:
    def test_feedback(self):
        assert infer_source_kind(Path(".feedback/note.md")) == "feedback"

    def test_learnings(self):
        assert infer_source_kind(Path("learnings/2024.md")) == "learnings"

    def test_memory(self):
        assert infer_source_kind(Path("memory/state.json")) == "memory"

    def test_generic(self):
        assert infer_source_kind(Path("README.md")) == "source"


# --- compute_target_profile ---


class TestComputeTargetProfile:
    def test_directory_profile(self, tmp_path):
        (tmp_path / "README.md").write_text("# Test")
        (tmp_path / "references").mkdir()
        profile = compute_target_profile(tmp_path)
        assert profile["exists"] is True
        assert profile["kind"] == "directory"
        assert profile["has_references"] is True
        assert len(profile["markdown_files"]) >= 1

    def test_missing_path(self, tmp_path):
        profile = compute_target_profile(tmp_path / "nonexistent")
        assert profile["exists"] is False
        assert profile["kind"] == "missing"


# --- constants consistency ---


class TestConstants:
    def test_keep_is_subset_of_executor_supported(self):
        assert KEEP_CATEGORIES.issubset(EXECUTOR_SUPPORTED_CATEGORIES)