Benchmark Sandbox

/**
 * Structured logger for the sandbox benchmark runner.
 *
 * Supports two output formats:
 * - "human" (default): timestamped human-readable lines → stderr
 * - "json": NDJSON RunnerEvent objects → stderr
 *
 * All log output goes to stderr so that --json mode can keep stdout clean
 * for the final RunSummary.
 *
 * API keys are automatically redacted from all output.
 */

import type { RunnerEvent, RunnerEventName } from "./types.js";

// ---------------------------------------------------------------------------
// Secret redaction
// ---------------------------------------------------------------------------

/** Patterns that look like API keys (sk-ant-*, anthropic key prefixes, long hex strings preceded by key-like env var names). */
const SECRET_PATTERNS = [
  // Anthropic keys: sk-ant-api03-... or sk-...
  /\bsk-ant-[a-zA-Z0-9_-]{20,}\b/g,
  /\bsk-[a-zA-Z0-9_-]{40,}\b/g,
  // Generic long bearer/api tokens following common env var assignments
  /(?<=(?:API_KEY|AUTH_TOKEN|SECRET|ANTHROPIC_API_KEY|AI_GATEWAY_API_KEY)\s*[=:]\s*)[^\s"']{20,}/gi,
];

export function redact(input: string): string {
  let result = input;
  for (const pattern of SECRET_PATTERNS) {
    // Reset lastIndex for global regexes
    pattern.lastIndex = 0;
    result = result.replace(pattern, "[REDACTED]");
  }
  return result;
}

// ---------------------------------------------------------------------------
// Logger
// ---------------------------------------------------------------------------

export type LogFormat = "human" | "json";

export interface LoggerOptions {
  format: LogFormat;
  runId: string;
}

export class Logger {
  private readonly format: LogFormat;
  private readonly runId: string;

  constructor(opts: LoggerOptions) {
    this.format = opts.format;
    this.runId = opts.runId;
  }

  /** Informational message (human) or structured event (json). */
  info(message: string): void {
    this.write("info", message);
  }

  /** Warning message. */
  warn(message: string): void {
    this.write("warn", message);
  }

  /** Error message. */
  error(message: string): void {
    this.write("error", message);
  }

  /**
   * Emit a structured RunnerEvent. In human mode this prints a readable line;
   * in json mode it writes a single NDJSON line.
   */
  event(name: RunnerEventName, payload: Record<string, unknown> = {}): void {
    if (this.format === "json") {
      const ev: RunnerEvent = {
        schema_version: 1,
        run_id: this.runId,
        event: name,
        timestamp: new Date().toISOString(),
        payload: this.redactPayload(payload),
      };
      process.stderr.write(JSON.stringify(ev) + "\n");
    } else {
      const payloadStr = Object.keys(payload).length > 0
        ? " " + Object.entries(payload).map(([k, v]) => `${k}=${JSON.stringify(v)}`).join(" ")
        : "";
      this.write("event", `[${name}]${payloadStr}`);
    }
  }

  // -------------------------------------------------------------------------
  // Internal
  // -------------------------------------------------------------------------

  private write(level: string, message: string): void {
    const safe = redact(message);
    if (this.format === "json") {
      // In json mode, non-event messages are still human-readable on stderr
      // but prefixed with level for easy filtering
      process.stderr.write(`${this.timestamp()} [${level}] ${safe}\n`);
    } else {
      process.stderr.write(`${this.timestamp()} ${safe}\n`);
    }
  }

  private timestamp(): string {
    const now = new Date();
    return now.toISOString().slice(11, 23); // HH:mm:ss.SSS
  }

  private redactPayload(payload: Record<string, unknown>): Record<string, unknown> {
    const result: Record<string, unknown> = {};
    for (const [key, value] of Object.entries(payload)) {
      if (typeof value === "string") {
        result[key] = redact(value);
      } else {
        result[key] = value;
      }
    }
    return result;
  }
}

#!/usr/bin/env bun
/**
 * Sandbox benchmark analyzer: reads artifacts from sandbox-runner output,
 * extracts debug logs and claim dirs, and produces a coverage report.
 *
 * Usage:
 *   bun run .claude/skills/benchmark-sandbox/sandbox-analyze.ts [options]
 *
 *   --run-dir <path>   Path to a specific run directory (default: latest in sandbox-results/)
 *   --json             Output JSON report instead of human-readable
 *   --help             Print usage and exit
 */

import { readFile, readdir, stat } from "node:fs/promises";
import { join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { homedir } from "node:os";
import type { RunSummary, ScenarioSummary } from "./types.js";

// ---------------------------------------------------------------------------
// CLI
// ---------------------------------------------------------------------------

const { values: flags } = parseArgs({
  options: {
    "run-dir": { type: "string" },
    json: { type: "boolean", default: false },
    help: { type: "boolean", default: false },
  },
  strict: true,
});

if (flags.help) {
  console.log(`Usage: bun run .claude/skills/benchmark-sandbox/sandbox-analyze.ts [options]
  --run-dir <path>   Path to a specific run directory
  --json             Output JSON report
  --help             Print usage`);
  process.exit(0);
}

const DEFAULT_RESULTS = join(homedir(), "dev", "vercel-plugin-testing", "sandbox-results");

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface HookCoverage {
  sessionStart: boolean;
  preToolUse: boolean;
  postToolUse: boolean;
  userPromptSubmit: boolean;
}

interface ScenarioAnalysis {
  slug: string;
  sandboxId: string;
  success: boolean;
  timedOut: boolean;
  durationSec: number;
  expectedSkills: string[];
  claimedSkills: string[];
  missingSkills: string[];
  unexpectedSkills: string[];
  coveragePercent: number;
  hookCoverage: HookCoverage;
  debugLogLines: number;
  stderrTraceLines: number;
  error?: string;
}

interface CoverageReport {
  runId: string;
  timestamp: string;
  snapshotId: string;
  totalDurationSec: number;
  scenarioCount: number;
  passCount: number;
  failCount: number;
  timeoutCount: number;
  hookFiredCount: number;
  scenarios: ScenarioAnalysis[];
  skillInjectionMatrix: Record<string, string[]>; // skill → slugs where injected
  allExpectedSkills: string[];
  allClaimedSkills: string[];
  coveragePercent: number; // expected skills that were actually claimed
}

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

async function findLatestRunDir(): Promise<string> {
  const entries = await readdir(DEFAULT_RESULTS);
  const runDirs = entries.filter((e) => e.startsWith("run-")).sort().reverse();
  if (runDirs.length === 0) {
    console.error(`No run directories found in ${DEFAULT_RESULTS}`);
    process.exit(1);
  }
  return join(DEFAULT_RESULTS, runDirs[0]);
}

async function readJsonFile<T>(path: string): Promise<T> {
  const content = await readFile(path, "utf-8");
  return JSON.parse(content) as T;
}

async function countLines(path: string): Promise<number> {
  try {
    const content = await readFile(path, "utf-8");
    return content.split("\n").length;
  } catch {
    return 0;
  }
}

async function dirExists(path: string): Promise<boolean> {
  try {
    const s = await stat(path);
    return s.isDirectory();
  } catch {
    return false;
  }
}

function parseHookCoverageFromDebugLogs(logContent: string): HookCoverage {
  return {
    sessionStart:
      logContent.includes("SessionStart") || logContent.includes("session-start"),
    preToolUse:
      logContent.includes("PreToolUse") ||
      logContent.includes("pretooluse") ||
      logContent.includes("skill-inject"),
    postToolUse:
      logContent.includes("PostToolUse") ||
      logContent.includes("posttooluse") ||
      logContent.includes("validate"),
    userPromptSubmit:
      logContent.includes("UserPromptSubmit") ||
      logContent.includes("user-prompt-submit") ||
      logContent.includes("prompt-signal"),
  };
}

// ---------------------------------------------------------------------------
// Analysis
// ---------------------------------------------------------------------------

async function analyzeRun(runDir: string): Promise<CoverageReport> {
  const manifest = await readJsonFile<RunSummary>(join(runDir, "run-manifest.json"));

  const scenarios: ScenarioAnalysis[] = [];
  const skillInjectionMatrix: Record<string, string[]> = {};

  for (const s of manifest.scenarios) {
    const slugDir = join(runDir, s.slug);

    // Read debug logs if available
    let debugLogContent = "";
    const debugLogsDir = join(slugDir, "debug-logs");
    if (await dirExists(debugLogsDir)) {
      const logFiles = await readdir(debugLogsDir).catch(() => []);
      for (const f of logFiles) {
        debugLogContent += await readFile(join(debugLogsDir, f), "utf-8").catch(() => "");
      }
    }

    // Read stderr trace
    const stderrLines = await countLines(join(slugDir, "stderr-trace.txt"));

    // Parse hook coverage from debug logs + stderr
    let stderrContent = "";
    try {
      stderrContent = await readFile(join(slugDir, "stderr-trace.txt"), "utf-8");
    } catch { /* no stderr file */ }

    const combinedLogs = debugLogContent + "\n" + stderrContent;
    const hookCoverage = parseHookCoverageFromDebugLogs(combinedLogs);

    // If manifest already has hook evidence, merge it in
    if (s.hook_evidence.pre_tool_use_in_stderr) hookCoverage.preToolUse = true;
    if (s.hook_evidence.user_prompt_in_stderr) hookCoverage.userPromptSubmit = true;

    // Derive boolean flags from status
    const success = s.status === "pass";
    const timedOut = s.status === "timeout";

    // Compute skill coverage per scenario
    const expectedSet = new Set(s.expected_skills);
    const claimedSet = new Set(s.claimed_skills);
    const missingSkills = s.expected_skills.filter((sk) => !claimedSet.has(sk));
    const unexpectedSkills = s.claimed_skills.filter((sk) => !expectedSet.has(sk));
    const coveredCount = s.expected_skills.filter((sk) => claimedSet.has(sk)).length;
    const scenarioCoverage =
      s.expected_skills.length > 0 ? Math.round((coveredCount / s.expected_skills.length) * 100) : 100;

    // Track skill injection matrix
    for (const skill of s.claimed_skills) {
      if (!skillInjectionMatrix[skill]) skillInjectionMatrix[skill] = [];
      skillInjectionMatrix[skill].push(s.slug);
    }

    scenarios.push({
      slug: s.slug,
      sandboxId: s.sandbox_id,
      success,
      timedOut,
      durationSec: Math.round(s.duration_ms / 1000),
      expectedSkills: s.expected_skills,
      claimedSkills: s.claimed_skills,
      missingSkills,
      unexpectedSkills,
      coveragePercent: scenarioCoverage,
      hookCoverage,
      debugLogLines: debugLogContent.split("\n").length,
      stderrTraceLines: stderrLines,
      error: s.error,
    });
  }

  // Aggregate stats — average per-scenario coverage, not global unique skills
  const allExpected = [...new Set(manifest.scenarios.flatMap((s) => s.expected_skills))].sort();
  const allClaimed = [...new Set(manifest.scenarios.flatMap((s) => s.claimed_skills))].sort();
  const scenariosWithExpectations = scenarios.filter((s) => s.expectedSkills.length > 0);
  const coveragePercent =
    scenariosWithExpectations.length > 0
      ? Math.round(
          scenariosWithExpectations.reduce((sum, s) => sum + s.coveragePercent, 0) /
            scenariosWithExpectations.length,
        )
      : 0;

  return {
    runId: manifest.run_id,
    timestamp: manifest.timestamp,
    snapshotId: manifest.snapshot.snapshot_id,
    totalDurationSec: Math.round(manifest.timing.total_duration_ms / 1000),
    scenarioCount: scenarios.length,
    passCount: scenarios.filter((s) => s.success).length,
    failCount: scenarios.filter((s) => !s.success && !s.timedOut).length,
    timeoutCount: scenarios.filter((s) => s.timedOut).length,
    hookFiredCount: scenarios.filter(
      (s) => s.hookCoverage.preToolUse || s.hookCoverage.userPromptSubmit,
    ).length,
    scenarios,
    skillInjectionMatrix,
    allExpectedSkills: allExpected,
    allClaimedSkills: allClaimed,
    coveragePercent,
  };
}

// ---------------------------------------------------------------------------
// Output
// ---------------------------------------------------------------------------

function printReport(report: CoverageReport): void {
  console.log("=".repeat(70));
  console.log("SANDBOX BENCHMARK COVERAGE REPORT");
  console.log("=".repeat(70));
  console.log(`Run ID:     ${report.runId}`);
  console.log(`Timestamp:  ${report.timestamp}`);
  console.log(`Snapshot:   ${report.snapshotId}`);
  console.log(`Duration:   ${report.totalDurationSec}s`);
  console.log();

  // Summary
  console.log("--- Summary ---");
  console.log(`Scenarios:  ${report.scenarioCount}`);
  console.log(`Passed:     ${report.passCount}`);
  console.log(`Failed:     ${report.failCount}`);
  console.log(`Timed out:  ${report.timeoutCount}`);
  console.log(`Hooks fired: ${report.hookFiredCount}/${report.scenarioCount}`);
  console.log(`Skill coverage: ${report.coveragePercent}% (${report.allClaimedSkills.length}/${report.allExpectedSkills.length} expected skills seen)`);
  console.log();

  // Scenario table
  console.log("--- Scenarios ---");
  console.log(
    `${"Slug".padEnd(28)} ${"Status".padEnd(10)} ${"Cov%".padEnd(6)} ${"Hooks".padEnd(8)} ${"Expected".padEnd(20)} ${"Claimed".padEnd(20)} Missing`,
  );
  console.log("-".repeat(116));
  for (const s of report.scenarios) {
    const status = s.timedOut ? "TIMEOUT" : s.success ? "PASS" : "FAIL";
    const covPct = `${s.coveragePercent}%`;
    const hooks = [
      s.hookCoverage.preToolUse ? "P" : "-",
      s.hookCoverage.postToolUse ? "V" : "-",
      s.hookCoverage.userPromptSubmit ? "U" : "-",
      s.hookCoverage.sessionStart ? "S" : "-",
    ].join("");
    const expected = s.expectedSkills.join(",");
    const claimed = s.claimedSkills.join(",") || "(none)";
    const missing = s.missingSkills.join(",") || "-";
    console.log(
      `${s.slug.padEnd(28)} ${status.padEnd(10)} ${covPct.padEnd(6)} ${hooks.padEnd(8)} ${expected.padEnd(20)} ${claimed.padEnd(20)} ${missing}`,
    );
    if (s.error) {
      console.log(`  ERROR: ${s.error.slice(0, 100)}`);
    }
  }
  console.log();
  console.log("Hook legend: P=PreToolUse V=PostToolUse(validate) U=UserPromptSubmit S=SessionStart");
  console.log();

  // Skill injection matrix
  console.log("--- Skill Injection Matrix ---");
  const sortedSkills = Object.entries(report.skillInjectionMatrix).sort(
    (a, b) => b[1].length - a[1].length,
  );
  for (const [skill, slugs] of sortedSkills) {
    console.log(`  ${skill.padEnd(25)} → ${slugs.join(", ")}`);
  }
  if (sortedSkills.length === 0) {
    console.log("  (no skills claimed)");
  }
  console.log();

  // Missing coverage
  const uncoveredSkills = report.allExpectedSkills.filter(
    (sk) => !report.allClaimedSkills.includes(sk),
  );
  if (uncoveredSkills.length > 0) {
    console.log("--- Uncovered Expected Skills ---");
    for (const sk of uncoveredSkills) {
      const scenarios = report.scenarios
        .filter((s) => s.expectedSkills.includes(sk))
        .map((s) => s.slug);
      console.log(`  ${sk.padEnd(25)} expected in: ${scenarios.join(", ")}`);
    }
    console.log();
  }

  // Failures
  const failures = report.scenarios.filter((s) => !s.success);
  if (failures.length > 0) {
    console.log("--- Failures ---");
    for (const f of failures) {
      console.log(`  ${f.slug}: ${f.timedOut ? "TIMEOUT" : f.error ?? "unknown error"}`);
      console.log(`    debug logs: ${f.debugLogLines} lines, stderr: ${f.stderrTraceLines} lines`);
    }
    console.log();
  }
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

async function main() {
  const runDir = flags["run-dir"] ? resolve(flags["run-dir"]) : await findLatestRunDir();
  console.log(`Analyzing: ${runDir}\n`);

  const report = await analyzeRun(runDir);

  if (flags.json) {
    console.log(JSON.stringify(report, null, 2));
  } else {
    printReport(report);
  }

  const MIN_COVERAGE_THRESHOLD = 50;
  const allPassed = report.passCount === report.scenarioCount;
  const coverageMet = report.coveragePercent >= MIN_COVERAGE_THRESHOLD;
  if (!coverageMet) {
    console.log(
      `\nFAILED: Coverage ${report.coveragePercent}% is below minimum threshold of ${MIN_COVERAGE_THRESHOLD}%`,
    );
  }
  process.exit(allPassed && coverageMet ? 0 : 1);
}

main().catch((err) => {
  console.error("Fatal:", err);
  process.exit(2);
});

#!/usr/bin/env bun
/**
 * Sandbox benchmark runner: provisions Vercel Sandboxes from a snapshot,
 * runs benchmark scenarios with Claude Code + plugin, and extracts artifacts.
 *
 * Scenarios are imported from scripts/benchmark-runner.ts (no duplication).
 *
 * Usage:
 *   bun run .claude/skills/benchmark-sandbox/sandbox-runner.ts [options]
 *
 *   --snapshot-only     Create/update the base snapshot and exit
 *   --quick             Run Tier 1 only (scenarios 01, 04, 09)
 *   --scenario <slug>   Run a single scenario by slug
 *   --concurrency <n>   Max parallel sandboxes (default: 3)
 *   --timeout <ms>      Per-scenario timeout (default: 300000 = 5 min)
 *   --results-dir <p>   Override results directory
 *   --help              Print usage and exit
 */

import { Sandbox } from "@vercel/sandbox";
import { mkdir, writeFile, readFile, readdir, stat, unlink } from "node:fs/promises";
import { join, resolve } from "node:path";
import { parseArgs } from "node:util";
import { homedir } from "node:os";
import { execSync } from "node:child_process";
import type { RunSummary, ScenarioSummary, SnapshotMeta } from "./types.js";
import {
  RunnerError,
  scenarioStatus,
  overallStatus,
  EXIT_ALL_PASS,
  EXIT_SOME_FAIL,
  EXIT_FATAL,
} from "./types.js";
import { Logger, type LogFormat } from "./logger.js";

// ---------------------------------------------------------------------------
// Import scenarios from the canonical source
// ---------------------------------------------------------------------------

import { PROJECTS, type BenchmarkProject } from "../../../scripts/benchmark-scenarios.js";

const SCENARIOS: BenchmarkProject[] = PROJECTS;

// Tier 1 quick-run slugs
const TIER1_SLUGS = new Set(["01-doc-qa-agent", "04-multi-model-router", "09-code-sandbox-tutor"]);

// ---------------------------------------------------------------------------
// CLI
// ---------------------------------------------------------------------------

const { values: flags } = parseArgs({
  options: {
    "snapshot-only": { type: "boolean", default: false },
    "force-snapshot": { type: "boolean", default: false },
    quick: { type: "boolean", default: false },
    scenario: { type: "string" },
    concurrency: { type: "string", default: "3" },
    timeout: { type: "string", default: "300000" },
    "results-dir": { type: "string" },
    json: { type: "boolean", default: false },
    "log-format": { type: "string", default: "human" },
    help: { type: "boolean", default: false },
  },
  strict: true,
});

if (flags.help) {
  console.log(`Usage: bun run .claude/skills/benchmark-sandbox/sandbox-runner.ts [options]
  --snapshot-only     Create/update the base snapshot and exit
  --force-snapshot    Force fresh snapshot creation (ignore cache)
  --quick             Run Tier 1 only (scenarios 01, 04, 09)
  --scenario <slug>   Run a single scenario by slug
  --concurrency <n>   Max parallel sandboxes (default: 3, max: 10)
  --timeout <ms>      Per-scenario timeout (default: 300000 = 5 min)
  --results-dir <p>   Override results directory
  --json              Write only RunSummary JSON to stdout (all other output → stderr)
  --log-format <fmt>  Log format: human (default) or json (NDJSON events to stderr)
  --help              Print usage

Exit codes:
  0  All scenarios passed
  1  One or more scenarios failed or timed out
  2  Fatal runner error (snapshot failure, scenario load failure, etc.)`);
  process.exit(0);
}

const JSON_MODE = flags.json!;
const LOG_FORMAT = (flags["log-format"] === "json" ? "json" : "human") as LogFormat;

// runId declared early so Logger can use it; the timestamped form is set later in main()
let runId = `run-${new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19)}`;
const logger = new Logger({ format: LOG_FORMAT, runId });

/** Shorthand — all human output goes to stderr (stdout reserved for --json RunSummary). */
const log = (msg: string) => logger.info(msg);
const warn = (msg: string) => logger.warn(msg);

const MAX_CONCURRENCY = 10;
const WARN_CONCURRENCY = 6;
const CONCURRENCY_RAW = parseInt(flags.concurrency!, 10);
if (CONCURRENCY_RAW > MAX_CONCURRENCY) {
  logger.error(`Concurrency ${CONCURRENCY_RAW} exceeds maximum of ${MAX_CONCURRENCY}. Clamping to ${MAX_CONCURRENCY}.`);
}
if (CONCURRENCY_RAW > WARN_CONCURRENCY && CONCURRENCY_RAW <= MAX_CONCURRENCY) {
  warn(`Warning: concurrency ${CONCURRENCY_RAW} exceeds ${WARN_CONCURRENCY} — may hit sandbox rate limits.`);
}
const CONCURRENCY = Math.min(Math.max(CONCURRENCY_RAW, 1), MAX_CONCURRENCY);
const TIMEOUT_MS = parseInt(flags.timeout!, 10);
const LOCAL_PLUGIN_DIR = join(homedir(), "dev", "vercel-plugin");
const SANDBOX_HOME = "/home/vercel-sandbox";
const SANDBOX_PLUGIN_DIR = `${SANDBOX_HOME}/vercel-plugin`;
const DEFAULT_RESULTS = join(homedir(), "dev", "vercel-plugin-testing", "sandbox-results");
const RESULTS_DIR = resolve(flags["results-dir"] ?? DEFAULT_RESULTS);
const SNAPSHOT_CACHE_PATH = join(DEFAULT_RESULTS, ".snapshot-cache.json");
const SNAPSHOT_MAX_AGE_MS = 24 * 60 * 60 * 1000; // 24 hours

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

function elapsed(start: number): string {
  return `${((performance.now() - start) / 1000).toFixed(1)}s`;
}

function resolveApiKey(): string {
  // VERCEL_API_KEY is for sandbox provisioning, not for Claude Code
  const key =
    process.env.ANTHROPIC_API_KEY ??
    process.env.AI_GATEWAY_API_KEY;
  if (key) return key;

  // Try macOS Keychain via apiKeyHelper
  try {
    const keychainKey = execSync(
      'security find-generic-password -a "$USER" -s "ANTHROPIC_AUTH_TOKEN" -w',
      { encoding: "utf-8", timeout: 5000 },
    ).trim();
    if (keychainKey) return keychainKey;
  } catch { /* keychain not available or key not found */ }

  logger.error("Missing API key. Set ANTHROPIC_API_KEY or AI_GATEWAY_API_KEY (or store in macOS Keychain as ANTHROPIC_AUTH_TOKEN)");
  process.exit(1);
}

function resolveBaseUrl(): string | undefined {
  return process.env.ANTHROPIC_BASE_URL ?? "https://ai-gateway.vercel.sh";
}

/** Essential plugin directories/files to upload into the sandbox. */
const PLUGIN_UPLOAD_DIRS = ["hooks", "skills", "generated"];
const PLUGIN_UPLOAD_FILES = ["hooks/hooks.json", "package.json"];

/**
 * Recursively collect files from a local directory, returning
 * { relativePath: fileContent } pairs suitable for sandbox.writeFiles().
 */
async function collectPluginFiles(): Promise<Record<string, string>> {
  const files: Record<string, string> = {};

  async function walkDir(dir: string, prefix: string): Promise<void> {
    const entries = await readdir(join(LOCAL_PLUGIN_DIR, dir), { withFileTypes: true });
    for (const entry of entries) {
      const relPath = join(dir, entry.name);
      const fullPath = join(LOCAL_PLUGIN_DIR, relPath);
      if (entry.isDirectory()) {
        // Skip node_modules, .git, src (only need compiled hooks)
        if (["node_modules", ".git", "src", ".claude", "tests", "scripts", ".playground"].includes(entry.name)) continue;
        await walkDir(relPath, prefix);
      } else if (entry.isFile()) {
        // Skip .mts source files (only need compiled .mjs), test files, and large files
        if (entry.name.endsWith(".mts") || entry.name.endsWith(".test.ts")) continue;
        const s = await stat(fullPath);
        if (s.size > 200_000) continue; // skip files > 200KB
        const content = await readFile(fullPath, "utf-8");
        files[join(prefix, relPath)] = content;
      }
    }
  }

  for (const dir of PLUGIN_UPLOAD_DIRS) {
    await walkDir(dir, "");
  }

  // Also upload root-level files
  for (const f of PLUGIN_UPLOAD_FILES) {
    try {
      const content = await readFile(join(LOCAL_PLUGIN_DIR, f), "utf-8");
      files[f] = content;
    } catch { /* optional file */ }
  }

  return files;
}

/**
 * Upload the local plugin into the sandbox at SANDBOX_PLUGIN_DIR,
 * then run `npx add-plugin <sandbox-path>` to install it.
 */
async function uploadAndInstallPlugin(
  sandbox: SandboxInstance,
  projectDir: string,
): Promise<{ out: string; err: string; exit: number }> {
  const pluginFiles = await collectPluginFiles();
  const entries = Object.entries(pluginFiles);
  log(`  [plugin] Uploading ${entries.length} files to sandbox via writeFiles()...`);

  // Use native sandbox.writeFiles() API — much more reliable than heredocs
  const fileBatch = entries.map(([relPath, content]) => ({
    path: join(SANDBOX_PLUGIN_DIR, relPath),
    content: Buffer.from(content, "utf-8"),
  }));

  // Upload in chunks of 10 to avoid payload size limits
  const CHUNK_SIZE = 10;
  const totalChunks = Math.ceil(fileBatch.length / CHUNK_SIZE);
  logger.event("plugin.upload.started", { file_count: fileBatch.length, chunk_size: CHUNK_SIZE, total_chunks: totalChunks });

  for (let i = 0; i < fileBatch.length; i += CHUNK_SIZE) {
    const chunk = fileBatch.slice(i, i + CHUNK_SIZE);
    const chunkIndex = Math.floor(i / CHUNK_SIZE);
    const chunkBytes = chunk.reduce((a, f) => a + f.content.length, 0);
    try {
      await sandbox.writeFiles(chunk);
      logger.event("plugin.upload.chunk", { index: chunkIndex, total: totalChunks, bytes: chunkBytes });
    } catch (err: any) {
      logger.error(`  [plugin] writeFiles chunk ${i}-${i + chunk.length} failed: ${err.message}`);
      logger.error(`  [plugin] Paths: ${chunk.map(f => f.path).join(", ")}`);
      logger.error(`  [plugin] Total bytes: ${chunkBytes}`);
      throw err;
    }
  }
  logger.event("plugin.upload.completed", { file_count: fileBatch.length });

  // Manually write Claude Code plugin config instead of using add-plugin
  // (add-plugin calls `claude mcp add-json` which requires claude in /bin/sh PATH)
  const globalSettings = {
    extraKnownMarketplaces: {
      "vercel-plugin": {
        source: { source: "directory", path: SANDBOX_PLUGIN_DIR },
      },
    },
    enabledPlugins: { "vercel-plugin@vercel-plugin": true },
  };
  const projectSettings = {
    enabledPlugins: { "vercel-plugin@vercel-plugin": true },
  };

  log(`  [plugin] Writing plugin config manually...`);
  await sandbox.writeFiles([
    {
      path: `${SANDBOX_HOME}/.claude/settings.json`,
      content: Buffer.from(JSON.stringify(globalSettings, null, 2)),
    },
    {
      path: `${projectDir}/.claude/settings.json`,
      content: Buffer.from(JSON.stringify(projectSettings, null, 2)),
    },
  ]);

  return { out: "Plugin config written manually", err: "", exit: 0 };
}

type SandboxInstance = InstanceType<typeof Sandbox>;

async function run(
  sandbox: SandboxInstance,
  cmd: string,
  args: string[],
  opts?: { timeout?: number },
): Promise<{ out: string; err: string; exit: number }> {
  // @vercel/sandbox runCommand does not accept `timeout` — use AbortSignal instead
  const runOpts: { signal?: AbortSignal } = {};
  if (opts?.timeout) {
    runOpts.signal = AbortSignal.timeout(opts.timeout);
  }
  try {
    const result = await sandbox.runCommand(cmd, args, runOpts);
    return {
      out: (await result.stdout()).trim(),
      err: (await result.stderr()).trim(),
      exit: (result as any).exitCode ?? 0,
    };
  } catch (err: any) {
    // runCommand throws on 400 (e.g. executable_not_found) instead of returning exit code
    const msg = err.text ?? err.message ?? String(err);
    logger.error(`  [run] Command failed: ${cmd} ${args.slice(0, 2).join(" ")}`);
    logger.error(`  [run] Error: ${msg.slice(0, 300)}`);
    return { out: "", err: msg, exit: 127 };
  }
}

// ---------------------------------------------------------------------------
// Snapshot cache
// ---------------------------------------------------------------------------

interface SnapshotCache {
  snapshotId: string;
  createdAt: string; // ISO timestamp
}

async function loadCachedSnapshot(): Promise<string | null> {
  try {
    const raw = await readFile(SNAPSHOT_CACHE_PATH, "utf-8");
    const cache: SnapshotCache = JSON.parse(raw);
    const age = Date.now() - new Date(cache.createdAt).getTime();
    if (age < SNAPSHOT_MAX_AGE_MS) {
      log(`[snapshot] Using cached snapshot ${cache.snapshotId} (age: ${(age / 3600_000).toFixed(1)}h)`);
      return cache.snapshotId;
    }
    log(`[snapshot] Cached snapshot expired (age: ${(age / 3600_000).toFixed(1)}h > 24h)`);
    return null;
  } catch {
    return null; // no cache file or invalid JSON
  }
}

async function saveCachedSnapshot(snapshotId: string): Promise<void> {
  await mkdir(join(SNAPSHOT_CACHE_PATH, ".."), { recursive: true });
  const cache: SnapshotCache = { snapshotId, createdAt: new Date().toISOString() };
  await writeFile(SNAPSHOT_CACHE_PATH, JSON.stringify(cache, null, 2));
}

async function removeCachedSnapshot(): Promise<void> {
  try {
    await unlink(SNAPSHOT_CACHE_PATH);
  } catch { /* no cache to remove */ }
}

// ---------------------------------------------------------------------------
// Snapshot verification — boot from snapshot, run `claude --version`
// ---------------------------------------------------------------------------

async function verifySnapshot(snapshotId: string): Promise<boolean> {
  logger.event("snapshot.verify.started", { snapshot_id: snapshotId });
  let sandbox: SandboxInstance | undefined;
  try {
    sandbox = await Sandbox.create({ fromSnapshot: snapshotId, timeout: 120_000 });
    const ver = await run(sandbox, "sh", ["-c", "claude --version"], { timeout: 30_000 });
    if (ver.exit === 0 && ver.out.length > 0) {
      logger.event("snapshot.verify.succeeded", { snapshot_id: snapshotId, claude_version: ver.out.slice(0, 80) });
      return true;
    }
    logger.event("snapshot.verify.failed", { snapshot_id: snapshotId, exit: ver.exit, stderr: ver.err.slice(0, 200) });
    return false;
  } catch (err: any) {
    logger.event("snapshot.verify.failed", { snapshot_id: snapshotId, error: err.message?.slice(0, 200) ?? String(err) });
    return false;
  } finally {
    if (sandbox) {
      try { await sandbox.stop(); } catch { /* already stopped */ }
    }
  }
}

// ---------------------------------------------------------------------------
// Snapshot creation
// ---------------------------------------------------------------------------

async function createSnapshot(): Promise<string> {
  const t0 = performance.now();
  const apiKey = resolveApiKey();
  const baseUrl = resolveBaseUrl();

  log("[snapshot] Creating base sandbox (node24)...");
  const env: Record<string, string> = {
    ANTHROPIC_API_KEY: apiKey,
    VERCEL_PLUGIN_LOG_LEVEL: "trace",
  };
  if (baseUrl) env.ANTHROPIC_BASE_URL = baseUrl;

  const sandbox = await Sandbox.create({ runtime: "node24", env, timeout: 900_000 });
  log(`[snapshot] Sandbox ${sandbox.sandboxId} created (${elapsed(t0)})`);

  // Install Claude Code
  log("[snapshot] Installing Claude Code...");
  const t1 = performance.now();
  const install = await run(sandbox, "sh", ["-c", "npm install -g @anthropic-ai/claude-code"]);
  if (install.exit !== 0) {
    await sandbox.stop();
    throw new Error(`Claude Code install failed: ${install.err.slice(0, 300)}`);
  }
  log(`[snapshot] Claude Code installed (${elapsed(t1)})`);

  // Symlink claude + node binaries to /usr/local/bin so /bin/sh subprocesses can find them
  await run(sandbox, "sh", [
    "-c",
    "ln -sf $(which claude) /usr/local/bin/claude && ln -sf $(which node) /usr/local/bin/node && ln -sf $(which npm) /usr/local/bin/npm && ln -sf $(which npx) /usr/local/bin/npx",
  ]);

  // Verify claude --version (use sh -c to ensure PATH resolution)
  const ver = await run(sandbox, "sh", ["-c", "claude --version"]);
  log(`[snapshot] claude version: ${ver.out}`);

  // Take snapshot
  log("[snapshot] Taking snapshot...");
  const t2 = performance.now();
  const snapshot = await sandbox.snapshot();
  const snapshotId = (snapshot as any).snapshotId ?? (snapshot as any).id ?? String(snapshot);
  log(`[snapshot] Snapshot created: ${snapshotId} (${elapsed(t2)})`);

  await sandbox.stop();
  log(`[snapshot] Total: ${elapsed(t0)}`);

  return snapshotId;
}

// ---------------------------------------------------------------------------
// Per-scenario runner
// ---------------------------------------------------------------------------

interface ScenarioResult {
  slug: string;
  sandboxId: string;
  success: boolean;
  timedOut: boolean;
  durationMs: number;
  sessionMethod: string;
  expectedSkills: string[];
  claimedSkills: string[];
  hookEvidence: {
    claimDirs: boolean;
    seenFile: boolean;
    debugLogCount: number;
    preToolUseInStderr: boolean;
    userPromptInStderr: boolean;
  };
  error?: string;
}

async function runScenario(
  project: BenchmarkProject,
  snapshotId: string,
  apiKey: string,
  baseUrl: string | undefined,
): Promise<ScenarioResult> {
  const t0 = performance.now();
  const projectDir = `${SANDBOX_HOME}/${project.slug}`;

  const env: Record<string, string> = {
    ANTHROPIC_API_KEY: apiKey,
    VERCEL_PLUGIN_LOG_LEVEL: "trace",
  };
  if (baseUrl) env.ANTHROPIC_BASE_URL = baseUrl;

  let sandbox: SandboxInstance | undefined;
  try {
    // Create sandbox from snapshot (snapshot has node runtime but global npm packages don't persist)
    sandbox = await Sandbox.create({ fromSnapshot: snapshotId, env, timeout: 900_000 });
    log(`  [${project.slug}] Sandbox ${sandbox.sandboxId} ready (${elapsed(t0)})`);

    // Install Claude Code (global npm packages don't survive snapshot restore)
    const claudeInstall = await run(sandbox, "sh", ["-c", "npm install -g @anthropic-ai/claude-code"]);
    const claudeBin = (await run(sandbox, "sh", ["-c", "which claude"])).out;
    log(`  [${project.slug}] Claude installed at ${claudeBin} (${elapsed(t0)})`);

    // Set up project directory and install plugin from local dev copy
    await run(sandbox, "mkdir", ["-p", projectDir]);
    await run(sandbox, "sh", ["-c", `cd ${projectDir} && npm init -y`]);

    const pluginInstall = await uploadAndInstallPlugin(sandbox, projectDir);
    log(`  [${project.slug}] add-plugin exit=${pluginInstall.exit}`);
    if (pluginInstall.out) log(`  [${project.slug}] add-plugin out: ${pluginInstall.out.slice(0, 200)}`);
    if (pluginInstall.err) log(`  [${project.slug}] add-plugin err: ${pluginInstall.err.slice(0, 200)}`);
    if (pluginInstall.exit !== 0 && !pluginInstall.out.includes("successfully") && !pluginInstall.out.includes("Installed")) {
      throw new Error(`Plugin install failed (exit ${pluginInstall.exit}): ${(pluginInstall.out + " " + pluginInstall.err).slice(0, 300)}`);
    }

    // Write prompt to a temp file to avoid shell injection from prompt content
    const promptTmpPath = "/tmp/claude-prompt.txt";
    await sandbox.writeFiles([{ path: promptTmpPath, content: Buffer.from(project.prompt, "utf-8") }]);

    // Check for PTY wrapper
    const scriptCheck = await run(sandbox, "sh", ["-c", "which script 2>/dev/null && echo FOUND || echo MISSING"]);
    const hasScript = scriptCheck.out.includes("FOUND");

    // Run Claude Code session — prompt is read from temp file via cat, never interpolated
    const settingsPath = `${projectDir}/.claude/settings.json`;
    let sessionResult: { out: string; err: string; exit: number };
    let sessionMethod: string;

    if (hasScript) {
      sessionMethod = "script-pty";
      sessionResult = await run(
        sandbox,
        "sh",
        [
          "-c",
          `cd ${projectDir} && VERCEL_PLUGIN_LOG_LEVEL=trace CLAUDE_PLUGIN_ROOT=${SANDBOX_PLUGIN_DIR} script -qec "${SANDBOX_HOME}/.global/npm/bin/claude --dangerously-skip-permissions --settings ${settingsPath} \\"\\$(cat ${promptTmpPath})\\"" /dev/null`,
        ],
        { timeout: TIMEOUT_MS },
      );
    } else {
      sessionMethod = "direct";
      sessionResult = await run(
        sandbox,
        "sh",
        [
          "-c",
          `cd ${projectDir} && VERCEL_PLUGIN_LOG_LEVEL=trace CLAUDE_PLUGIN_ROOT=${SANDBOX_PLUGIN_DIR} ${SANDBOX_HOME}/.global/npm/bin/claude --dangerously-skip-permissions --settings ${settingsPath} "$(cat ${promptTmpPath})"`,
        ],
        { timeout: TIMEOUT_MS },
      );
    }

    const timedOut = sessionResult.exit === 124; // timeout exit code

    // Extract hook evidence
    const claimCheck = await run(sandbox, "sh", [
      "-c",
      "find /tmp -maxdepth 1 -name 'vercel-plugin-*-seen-skills.d' -type d 2>/dev/null | head -5",
    ]);
    const claimDirs = claimCheck.out.split("\n").filter(Boolean);
    let claimedSkills: string[] = [];
    if (claimDirs.length > 0) {
      const contents = await run(sandbox, "sh", ["-c", `ls ${claimDirs[0]} 2>/dev/null`]);
      claimedSkills = contents.out.split("\n").filter(Boolean);
    }

    const seenFileCheck = await run(sandbox, "sh", [
      "-c",
      "find /tmp -maxdepth 1 -name 'vercel-plugin-*-seen-skills.txt' 2>/dev/null | head -1",
    ]);
    const debugLogCheck = await run(sandbox, "sh", [
      "-c",
      `find /root/.claude/debug -name '*.txt' -o -name '*.log' 2>/dev/null; find ${SANDBOX_HOME}/.claude/debug -name '*.txt' -o -name '*.log' 2>/dev/null`,
    ]);
    const debugLogFiles = debugLogCheck.out.split("\n").filter(Boolean);

    const hookEvidence = {
      claimDirs: claimDirs.length > 0,
      seenFile: seenFileCheck.out.length > 0,
      debugLogCount: debugLogFiles.length,
      preToolUseInStderr:
        sessionResult.err.includes("PreToolUse") ||
        sessionResult.err.includes("pretooluse") ||
        sessionResult.err.includes("skill-inject"),
      userPromptInStderr:
        sessionResult.err.includes("UserPromptSubmit") ||
        sessionResult.err.includes("user-prompt-submit"),
    };

    // Save artifacts to results dir
    const slugDir = join(RESULTS_DIR, runId, project.slug);
    await mkdir(slugDir, { recursive: true });
    await mkdir(join(slugDir, "claim-dir"), { recursive: true });
    await mkdir(join(slugDir, "debug-logs"), { recursive: true });

    await writeFile(join(slugDir, "claude-output.txt"), sessionResult.out);
    await writeFile(join(slugDir, "stderr-trace.txt"), sessionResult.err);

    // Copy claim dir contents
    for (const skill of claimedSkills) {
      await writeFile(join(slugDir, "claim-dir", skill), "");
    }

    // Extract seen-skills.txt
    if (seenFileCheck.out.length > 0) {
      const seenContents = await run(sandbox, "cat", [seenFileCheck.out.split("\n")[0]]);
      await writeFile(join(slugDir, "seen-skills.txt"), seenContents.out);
    }

    // Extract debug logs
    for (const logFile of debugLogFiles.slice(0, 5)) {
      try {
        const content = await run(sandbox, "sh", ["-c", `head -500 '${logFile}'`]);
        const basename = logFile.split("/").pop() ?? "unknown.txt";
        await writeFile(join(slugDir, "debug-logs", basename), content.out);
      } catch { /* skip unreadable logs */ }
    }

    // Project tree
    const tree = await run(sandbox, "sh", [
      "-c",
      `find ${projectDir} -maxdepth 3 -not -path '*/node_modules/*' -not -path '*/.git/*' | head -80`,
    ]);
    await writeFile(join(slugDir, "project-tree.txt"), tree.out);

    // Run meta
    const result: ScenarioResult = {
      slug: project.slug,
      sandboxId: sandbox.sandboxId,
      success: sessionResult.exit === 0 && !timedOut,
      timedOut,
      durationMs: performance.now() - t0,
      sessionMethod,
      expectedSkills: project.expectedSkills,
      claimedSkills,
      hookEvidence,
    };
    await writeFile(join(slugDir, "run-meta.json"), JSON.stringify(result, null, 2));

    return result;
  } catch (err: any) {
    return {
      slug: project.slug,
      sandboxId: sandbox?.sandboxId ?? "unknown",
      success: false,
      timedOut: false,
      durationMs: performance.now() - t0,
      sessionMethod: "none",
      expectedSkills: project.expectedSkills,
      claimedSkills: [],
      hookEvidence: {
        claimDirs: false,
        seenFile: false,
        debugLogCount: 0,
        preToolUseInStderr: false,
        userPromptInStderr: false,
      },
      error: err.message?.slice(0, 400) ?? String(err),
    };
  } finally {
    if (sandbox) {
      try {
        await sandbox.stop();
      } catch { /* already stopped */ }
    }
  }
}

// ---------------------------------------------------------------------------
// Parallel orchestrator
// ---------------------------------------------------------------------------

async function runParallel(
  scenarios: BenchmarkProject[],
  snapshotId: string,
  concurrency: number,
): Promise<ScenarioResult[]> {
  const apiKey = resolveApiKey();
  const baseUrl = resolveBaseUrl();
  const results: ScenarioResult[] = [];
  const queue = [...scenarios];

  async function worker(): Promise<void> {
    while (queue.length > 0) {
      const project = queue.shift()!;
      log(`\n--- ${project.slug} (${scenarios.length - queue.length}/${scenarios.length}) ---`);
      const result = await runScenario(project, snapshotId, apiKey, baseUrl);
      results.push(result);

      const status = result.timedOut ? "TIMEOUT" : result.success ? "OK" : "FAIL";
      log(
        `  [${project.slug}] ${status} | skills: ${result.claimedSkills.join(", ") || "none"} | ${(result.durationMs / 1000).toFixed(1)}s`,
      );
    }
  }

  // Launch N workers
  const workers = Array.from({ length: Math.min(concurrency, scenarios.length) }, () => worker());
  await Promise.all(workers);

  return results;
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

// runId is declared at the top alongside the Logger; no re-declaration needed here.

async function main() {
  const t0 = performance.now();

  log("=== Benchmark Sandbox Runner ===\n");
  logger.event("runner.start", { mode: flags.scenario ? "single" : flags.quick ? "quick" : "full" });

  // Step 1: Create or reuse snapshot (cached for 24h, verified before use)
  let snapshotId: string | null = null;
  let snapshotCached = false;
  let snapshotAgeHours: number | undefined;
  let snapshotCreationMs: number | undefined;

  if (!flags["force-snapshot"]) {
    snapshotId = await loadCachedSnapshot();
    if (snapshotId) {
      logger.event("snapshot.cache.hit", { snapshot_id: snapshotId });
      try {
        const raw = await readFile(SNAPSHOT_CACHE_PATH, "utf-8");
        const cache = JSON.parse(raw) as SnapshotCache;
        snapshotAgeHours = (Date.now() - new Date(cache.createdAt).getTime()) / 3600_000;
      } catch { /* ignore */ }

      // Verify cached snapshot is still usable
      const verified = await verifySnapshot(snapshotId);
      if (verified) {
        snapshotCached = true;
      } else {
        // Stale — invalidate and recreate once
        const staleErr = new RunnerError(
          "SNAPSHOT_CACHE_STALE",
          `Cached snapshot ${snapshotId} failed verification`,
        );
        warn(`[snapshot] ${staleErr.message} — ${staleErr.hint}`);
        logger.event("snapshot.verify.failed", { snapshot_id: snapshotId, error_code: staleErr.code });
        await removeCachedSnapshot();
        snapshotId = null; // fall through to creation
      }
    }
  }

  if (!snapshotId) {
    // Remove stale cache before attempting creation so a failure doesn't leave one
    await removeCachedSnapshot();
    logger.event("snapshot.cache.miss", {});
    logger.event("snapshot.create.started", {});
    const snapT0 = performance.now();
    snapshotId = await createSnapshot();
    snapshotCreationMs = performance.now() - snapT0;
    logger.event("snapshot.create.succeeded", { snapshot_id: snapshotId, duration_ms: Math.round(snapshotCreationMs) });

    // Verify newly created snapshot before caching
    const newVerified = await verifySnapshot(snapshotId);
    if (!newVerified) {
      const verifyErr = new RunnerError(
        "SNAPSHOT_VERIFY_FAILED",
        `Newly created snapshot ${snapshotId} failed verification`,
      );
      logger.error(`Fatal: ${verifyErr.message}`);
      logger.error(`Hint: ${verifyErr.hint}`);
      logger.event("runner.failed", { error: verifyErr.message, error_code: verifyErr.code });
      process.exit(EXIT_FATAL);
    }

    await saveCachedSnapshot(snapshotId);
  }
  log(`\nSnapshot: ${snapshotId}\n`);

  if (flags["snapshot-only"]) {
    log("--snapshot-only: done.");
    process.exit(0);
  }

  // Step 2: Select scenarios
  let scenarios = SCENARIOS;
  if (flags.scenario) {
    const match = SCENARIOS.find((s) => s.slug === flags.scenario);
    if (!match) {
      logger.error(`Unknown scenario: ${flags.scenario}`);
      logger.error(`Available: ${SCENARIOS.map((s) => s.slug).join(", ")}`);
      process.exit(1);
    }
    scenarios = [match];
  } else if (flags.quick) {
    scenarios = SCENARIOS.filter((s) => TIER1_SLUGS.has(s.slug));
  }

  log(`Scenarios: ${scenarios.length} (${flags.quick ? "quick" : flags.scenario ?? "full"})`);
  log(`Concurrency: ${CONCURRENCY}`);
  log(`Timeout: ${TIMEOUT_MS / 1000}s per scenario`);
  log(`Results: ${RESULTS_DIR}/${runId}\n`);

  // Create results directory
  await mkdir(join(RESULTS_DIR, runId), { recursive: true });

  // Step 3: Run scenarios
  const results = await runParallel(scenarios, snapshotId, CONCURRENCY);

  // Step 4: Build RunSummary (single source of truth for both human + JSON output)
  const scenarioSummaries: ScenarioSummary[] = results.map((r) => ({
    slug: r.slug,
    sandbox_id: r.sandboxId,
    status: scenarioStatus(r),
    duration_ms: r.durationMs,
    session_method: r.sessionMethod,
    expected_skills: r.expectedSkills,
    claimed_skills: r.claimedSkills,
    hook_evidence: {
      claim_dirs: r.hookEvidence.claimDirs,
      seen_file: r.hookEvidence.seenFile,
      debug_log_count: r.hookEvidence.debugLogCount,
      pre_tool_use_in_stderr: r.hookEvidence.preToolUseInStderr,
      user_prompt_in_stderr: r.hookEvidence.userPromptInStderr,
    },
    error: r.error,
  }));

  const snapshotMeta: SnapshotMeta = {
    snapshot_id: snapshotId,
    cached: snapshotCached,
    ...(snapshotAgeHours !== undefined && { age_hours: Math.round(snapshotAgeHours * 10) / 10 }),
    ...(snapshotCreationMs !== undefined && { creation_duration_ms: Math.round(snapshotCreationMs) }),
  };

  const mode = flags.scenario ? "single" as const : flags.quick ? "quick" as const : "full" as const;

  const summary: RunSummary = {
    schema_version: 1,
    run_id: runId,
    status: overallStatus(scenarioSummaries),
    timestamp: new Date().toISOString(),
    snapshot: snapshotMeta,
    scenarios: scenarioSummaries,
    timing: {
      total_duration_ms: Math.round(performance.now() - t0),
      scenario_durations_ms: Object.fromEntries(
        scenarioSummaries.map((s) => [s.slug, Math.round(s.duration_ms)]),
      ),
    },
    config: {
      concurrency: CONCURRENCY,
      timeout_ms: TIMEOUT_MS,
      results_dir: join(RESULTS_DIR, runId),
      mode,
    },
  };

  // Write manifest (always, regardless of --json)
  await writeFile(join(RESULTS_DIR, runId, "run-manifest.json"), JSON.stringify(summary, null, 2));

  // Step 5: Print human summary table (derived from RunSummary)
  log("\n=== Summary ===");
  log(
    `${"Slug".padEnd(28)} ${"Status".padEnd(10)} ${"Method".padEnd(12)} ${"Skills".padEnd(30)} Duration`,
  );
  log("-".repeat(100));
  for (const s of summary.scenarios) {
    const dur = `${(s.duration_ms / 1000).toFixed(1)}s`;
    const skills = s.claimed_skills.join(", ") || "(none)";
    log(`${s.slug.padEnd(28)} ${s.status.toUpperCase().padEnd(10)} ${s.session_method.padEnd(12)} ${skills.padEnd(30)} ${dur}`);
  }

  const passed = summary.scenarios.filter((s) => s.status === "pass").length;
  const hookFired = summary.scenarios.filter(
    (s) => s.hook_evidence.claim_dirs || s.hook_evidence.pre_tool_use_in_stderr,
  ).length;
  log(`\n${passed}/${summary.scenarios.length} scenarios succeeded`);
  log(`${hookFired}/${summary.scenarios.length} scenarios had hook evidence`);
  log(`Results: ${summary.config.results_dir}`);
  log(`Total time: ${elapsed(t0)}\n`);

  // --json: write only RunSummary to stdout (everything else already went to stderr)
  if (JSON_MODE) {
    process.stdout.write(JSON.stringify(summary, null, 2) + "\n");
  }

  // Exit codes: 0=all pass, 1=some fail, 2=fatal
  logger.event("runner.completed", { status: summary.status, scenarios: summary.scenarios.length });
  const exitCode = summary.status === "pass" ? EXIT_ALL_PASS : EXIT_SOME_FAIL;
  process.exit(exitCode);
}

main().catch((err) => {
  logger.error(`Fatal: ${err.message ?? err}`);
  logger.event("runner.failed", { error: err.message ?? String(err) });
  process.exit(EXIT_FATAL);
});

#!/usr/bin/env bun
/**
 * Create a pre-baked Vercel Sandbox snapshot with Claude Code + vercel-plugin
 * pre-installed. New sandboxes from this snapshot start in seconds instead of
 * waiting for fresh installs each time.
 *
 * Acceptance criteria:
 *  1. Snapshot includes Claude Code globally installed
 *  2. Snapshot includes vercel-plugin (hooks.json + settings) in a template dir
 *  3. New sandbox from snapshot starts in under 10 seconds
 *  4. `claude --version` passes on snapshot-based sandbox
 *  5. Hook verification (hooks.json present) passes on snapshot-based sandbox
 *  6. Snapshot ID is exported (stdout JSON + optional file) for sandbox-runner.ts
 *
 * Usage:
 *   bun run .claude/skills/benchmark-sandbox/spike/create-snapshot.ts [options]
 *
 *   --output <path>   Write snapshot metadata JSON to file (default: stdout only)
 *   --verify          Create a second sandbox from the snapshot and verify it
 *   --help            Print usage and exit
 */

import { Sandbox } from "@vercel/sandbox";
import { writeFile, readFile, readdir, stat } from "node:fs/promises";
import { join } from "node:path";
import { parseArgs } from "node:util";
import { homedir } from "node:os";
import { execSync } from "node:child_process";

// ── CLI ──────────────────────────────────────────────────────────────

const { values: flags } = parseArgs({
  options: {
    output: { type: "string" },
    verify: { type: "boolean", default: false },
    help: { type: "boolean", default: false },
  },
  strict: true,
});

if (flags.help) {
  console.log(`Usage: bun run .claude/skills/benchmark-sandbox/spike/create-snapshot.ts [options]
  --output <path>   Write snapshot metadata JSON to file
  --verify          Verify the snapshot by booting a sandbox from it
  --help            Print usage`);
  process.exit(0);
}

// ── Helpers ──────────────────────────────────────────────────────────

function resolveApiKey(): string {
  const key =
    process.env.ANTHROPIC_API_KEY ??
    process.env.AI_GATEWAY_API_KEY ??
    process.env.VERCEL_API_KEY;
  if (key) return key;

  // Try macOS Keychain via apiKeyHelper
  try {
    const keychainKey = execSync(
      'security find-generic-password -a "$USER" -s "ANTHROPIC_AUTH_TOKEN" -w',
      { encoding: "utf-8", timeout: 5000 },
    ).trim();
    if (keychainKey) return keychainKey;
  } catch { /* keychain not available or key not found */ }

  console.error(
    "Missing API key. Set one of: ANTHROPIC_API_KEY, AI_GATEWAY_API_KEY, VERCEL_API_KEY (or store in macOS Keychain as ANTHROPIC_AUTH_TOKEN)",
  );
  process.exit(1);
}

function resolveBaseUrl(): string | undefined {
  return process.env.ANTHROPIC_BASE_URL ?? "https://ai-gateway.vercel.sh";
}

function elapsed(start: number): string {
  return `${((performance.now() - start) / 1000).toFixed(1)}s`;
}

type SandboxInstance = InstanceType<typeof Sandbox>;

async function run(
  sandbox: SandboxInstance,
  cmd: string,
  args: string[],
  opts?: { timeout?: number },
): Promise<{ out: string; err: string; exit: number }> {
  const result = await sandbox.runCommand(cmd, args, opts);
  return {
    out: (await result.stdout()).trim(),
    err: (await result.stderr()).trim(),
    exit: (result as any).exitCode ?? 0,
  };
}

const LOCAL_PLUGIN_DIR = join(homedir(), "dev", "vercel-plugin");
const SANDBOX_PLUGIN_DIR = "/home/vercel-sandbox/vercel-plugin";
// Template directory inside the snapshot where plugin config lives.
// sandbox-runner copies from here into each project dir for fast setup.
const TEMPLATE_DIR = "/home/vercel-sandbox/.vercel-plugin-template";

/** Essential plugin directories to upload into the sandbox. */
const PLUGIN_UPLOAD_DIRS = ["hooks", "skills", "generated"];

async function collectPluginFiles(): Promise<Record<string, string>> {
  const files: Record<string, string> = {};

  async function walkDir(dir: string): Promise<void> {
    const entries = await readdir(join(LOCAL_PLUGIN_DIR, dir), { withFileTypes: true });
    for (const entry of entries) {
      const relPath = join(dir, entry.name);
      const fullPath = join(LOCAL_PLUGIN_DIR, relPath);
      if (entry.isDirectory()) {
        if (["node_modules", ".git", "src", ".claude", "tests", "scripts", ".playground"].includes(entry.name)) continue;
        await walkDir(relPath);
      } else if (entry.isFile()) {
        if (entry.name.endsWith(".mts") || entry.name.endsWith(".test.ts")) continue;
        const s = await stat(fullPath);
        if (s.size > 200_000) continue;
        const content = await readFile(fullPath, "utf-8");
        files[relPath] = content;
      }
    }
  }

  for (const dir of PLUGIN_UPLOAD_DIRS) {
    await walkDir(dir);
  }

  // Root-level files
  for (const f of ["hooks/hooks.json", "package.json"]) {
    try {
      const content = await readFile(join(LOCAL_PLUGIN_DIR, f), "utf-8");
      files[f] = content;
    } catch { /* optional */ }
  }

  return files;
}

async function uploadPluginToSandbox(sandbox: InstanceType<typeof Sandbox>): Promise<void> {
  const pluginFiles = await collectPluginFiles();
  console.log(`[snapshot] Uploading ${Object.keys(pluginFiles).length} plugin files to sandbox...`);

  await run(sandbox, "mkdir", ["-p", SANDBOX_PLUGIN_DIR]);
  for (const [relPath, content] of Object.entries(pluginFiles)) {
    const sandboxPath = join(SANDBOX_PLUGIN_DIR, relPath);
    const dir = sandboxPath.split("/").slice(0, -1).join("/");
    await run(sandbox, "mkdir", ["-p", dir]);
    await run(sandbox, "sh", ["-c", `cat > '${sandboxPath}' << 'PLUGIN_EOF'\n${content}\nPLUGIN_EOF`]);
  }
}

// ── Snapshot creation ────────────────────────────────────────────────

interface SnapshotMetadata {
  snapshotId: string;
  claudeVersion: string;
  pluginInstalled: boolean;
  templateDir: string;
  createdAt: string;
  createDurationMs: number;
}

async function createSnapshot(): Promise<SnapshotMetadata> {
  const t0 = performance.now();
  const apiKey = resolveApiKey();
  const baseUrl = resolveBaseUrl();

  const env: Record<string, string> = {
    ANTHROPIC_API_KEY: apiKey,
    VERCEL_PLUGIN_LOG_LEVEL: "trace",
  };
  if (baseUrl) env.ANTHROPIC_BASE_URL = baseUrl;

  // Step 1: Create base sandbox
  console.log("[snapshot] Creating base sandbox (node24)...");
  const t1 = performance.now();
  const sandbox = await Sandbox.create({ runtime: "node24", env });
  console.log(`[snapshot] Sandbox ${sandbox.sandboxId} created (${elapsed(t1)})`);

  try {
    // Step 2: Install Claude Code globally
    console.log("[snapshot] Installing Claude Code...");
    const t2 = performance.now();
    const install = await run(sandbox, "npm", [
      "install",
      "-g",
      "@anthropic-ai/claude-code",
    ]);
    if (install.exit !== 0) {
      throw new Error(`Claude Code install failed (exit=${install.exit}): ${install.err.slice(0, 300)}`);
    }
    console.log(`[snapshot] Claude Code installed (${elapsed(t2)})`);

    // Step 3: Verify claude --version
    const ver = await run(sandbox, "claude", ["--version"]);
    if (!/\d+\.\d+/.test(ver.out)) {
      throw new Error(`claude --version returned unexpected output: ${ver.out}`);
    }
    console.log(`[snapshot] claude version: ${ver.out}`);

    // Step 4: Upload local plugin and install into a template directory.
    // This uses the local development version instead of fetching from GitHub.
    console.log("[snapshot] Uploading local plugin and installing into template dir...");
    const t4 = performance.now();
    await uploadPluginToSandbox(sandbox);
    await run(sandbox, "mkdir", ["-p", TEMPLATE_DIR]);
    await run(sandbox, "sh", ["-c", `cd ${TEMPLATE_DIR} && npm init -y`]);
    const pluginInstall = await run(sandbox, "sh", [
      "-c",
      `cd ${TEMPLATE_DIR} && npx -y add-plugin ${SANDBOX_PLUGIN_DIR} -s project -y`,
    ]);
    const pluginOk =
      pluginInstall.exit === 0 ||
      pluginInstall.out.includes("successfully") ||
      pluginInstall.out.includes("added");
    if (!pluginOk) {
      throw new Error(`Plugin install failed: ${pluginInstall.out.slice(0, 300)}`);
    }
    console.log(`[snapshot] Plugin installed into template (${elapsed(t4)})`);

    // Verify hooks.json exists in the template
    const hooksCheck = await run(sandbox, "sh", [
      "-c",
      `test -f ${TEMPLATE_DIR}/.claude/hooks.json && echo EXISTS || echo MISSING`,
    ]);
    if (!hooksCheck.out.includes("EXISTS")) {
      throw new Error(`hooks.json not found in template dir after plugin install`);
    }
    console.log("[snapshot] hooks.json verified in template");

    // Verify settings.json exists
    const settingsCheck = await run(sandbox, "sh", [
      "-c",
      `test -f ${TEMPLATE_DIR}/.claude/settings.json && echo EXISTS || echo MISSING`,
    ]);
    console.log(`[snapshot] settings.json: ${settingsCheck.out}`);

    // Step 5: Take snapshot
    console.log("[snapshot] Taking snapshot...");
    const t5 = performance.now();
    const snapshot = await sandbox.snapshot();
    const snapshotId =
      (snapshot as any).snapshotId ?? (snapshot as any).id ?? String(snapshot);
    console.log(`[snapshot] Snapshot created: ${snapshotId} (${elapsed(t5)})`);

    const metadata: SnapshotMetadata = {
      snapshotId,
      claudeVersion: ver.out,
      pluginInstalled: true,
      templateDir: TEMPLATE_DIR,
      createdAt: new Date().toISOString(),
      createDurationMs: performance.now() - t0,
    };

    return metadata;
  } finally {
    try {
      await sandbox.stop();
      console.log("[snapshot] Base sandbox stopped");
    } catch {
      /* already stopped */
    }
  }
}

// ── Verification ─────────────────────────────────────────────────────

interface VerifyResult {
  bootDurationMs: number;
  claudeVersionOk: boolean;
  claudeVersion: string;
  hooksJsonPresent: boolean;
  settingsJsonPresent: boolean;
  templateDirPresent: boolean;
}

async function verifySnapshot(snapshotId: string): Promise<VerifyResult> {
  const apiKey = resolveApiKey();
  const baseUrl = resolveBaseUrl();
  const env: Record<string, string> = { ANTHROPIC_API_KEY: apiKey };
  if (baseUrl) env.ANTHROPIC_BASE_URL = baseUrl;

  console.log("\n[verify] Booting sandbox from snapshot...");
  const tBoot = performance.now();
  const sandbox = await Sandbox.create({ fromSnapshot: snapshotId, env });
  const bootMs = performance.now() - tBoot;
  console.log(`[verify] Sandbox ${sandbox.sandboxId} booted in ${(bootMs / 1000).toFixed(1)}s`);

  try {
    // Check claude --version
    const ver = await run(sandbox, "claude", ["--version"]);
    const claudeVersionOk = /\d+\.\d+/.test(ver.out);
    console.log(`[verify] claude --version: ${ver.out} (${claudeVersionOk ? "OK" : "FAIL"})`);

    // Check hooks.json in template
    const hooksCheck = await run(sandbox, "sh", [
      "-c",
      `test -f ${TEMPLATE_DIR}/.claude/hooks.json && echo EXISTS || echo MISSING`,
    ]);
    const hooksJsonPresent = hooksCheck.out.includes("EXISTS");
    console.log(`[verify] hooks.json: ${hooksCheck.out}`);

    // Check settings.json in template
    const settingsCheck = await run(sandbox, "sh", [
      "-c",
      `test -f ${TEMPLATE_DIR}/.claude/settings.json && echo EXISTS || echo MISSING`,
    ]);
    const settingsJsonPresent = settingsCheck.out.includes("EXISTS");
    console.log(`[verify] settings.json: ${settingsCheck.out}`);

    // Check template dir exists with content
    const templateCheck = await run(sandbox, "sh", [
      "-c",
      `test -d ${TEMPLATE_DIR}/.claude && echo EXISTS || echo MISSING`,
    ]);
    const templateDirPresent = templateCheck.out.includes("EXISTS");
    console.log(`[verify] template dir (.claude): ${templateCheck.out}`);

    // Boot time check
    const bootUnder10s = bootMs < 10_000;
    console.log(
      `[verify] Boot time: ${(bootMs / 1000).toFixed(1)}s (${bootUnder10s ? "UNDER 10s" : "OVER 10s"})`,
    );

    return {
      bootDurationMs: bootMs,
      claudeVersionOk,
      claudeVersion: ver.out,
      hooksJsonPresent,
      settingsJsonPresent,
      templateDirPresent,
    };
  } finally {
    try {
      await sandbox.stop();
      console.log("[verify] Verification sandbox stopped");
    } catch {
      /* already stopped */
    }
  }
}

// ── Main ─────────────────────────────────────────────────────────────

async function main() {
  const t0 = performance.now();
  console.log("=== Snapshot Creator ===\n");

  // Create snapshot
  const metadata = await createSnapshot();
  console.log(`\n[result] Snapshot ID: ${metadata.snapshotId}`);
  console.log(`[result] Claude version: ${metadata.claudeVersion}`);
  console.log(`[result] Plugin pre-installed: ${metadata.pluginInstalled}`);
  console.log(`[result] Template dir: ${metadata.templateDir}`);
  console.log(`[result] Create time: ${(metadata.createDurationMs / 1000).toFixed(1)}s`);

  // Optionally verify
  let verification: VerifyResult | undefined;
  if (flags.verify) {
    verification = await verifySnapshot(metadata.snapshotId);
  }

  // Build output JSON
  const output = {
    ...metadata,
    ...(verification
      ? {
          verification: {
            bootDurationMs: verification.bootDurationMs,
            bootUnder10s: verification.bootDurationMs < 10_000,
            claudeVersionOk: verification.claudeVersionOk,
            hooksJsonPresent: verification.hooksJsonPresent,
            settingsJsonPresent: verification.settingsJsonPresent,
            templateDirPresent: verification.templateDirPresent,
          },
        }
      : {}),
  };

  // Write to file if requested
  if (flags.output) {
    await writeFile(flags.output, JSON.stringify(output, null, 2));
    console.log(`\n[output] Metadata written to ${flags.output}`);
  }

  // Always print JSON to stdout for piping
  console.log("\n" + JSON.stringify(output, null, 2));

  // Summary
  console.log("\n" + "=".repeat(60));
  const allOk =
    metadata.pluginInstalled &&
    /\d+\.\d+/.test(metadata.claudeVersion) &&
    (!verification ||
      (verification.claudeVersionOk &&
        verification.hooksJsonPresent &&
        verification.bootDurationMs < 10_000));

  if (allOk) {
    console.log("ALL CHECKS PASSED");
  } else {
    console.log("SOME CHECKS FAILED");
  }
  console.log("=".repeat(60));
  console.log(`Total time: ${elapsed(t0)}`);

  // Export-friendly: print just the snapshot ID on the last line for easy capture
  // e.g. SNAPSHOT_ID=$(bun run create-snapshot.ts 2>/dev/null | tail -1)
  console.log(`\nSNAPSHOT_ID=${metadata.snapshotId}`);

  process.exit(allOk ? 0 : 1);
}

main().catch((err) => {
  console.error("Fatal:", err);
  process.exit(2);
});