Evals

# Domain-Specific Grader Patterns
# Based on Anthropic's "Demystifying Evals for AI Agents"

domains:

  coding:
    description: "Coding agent evaluation"
    primary_graders:
      - type: binary_tests
        weight: 0.30
        description: "Unit tests must pass"
      - type: static_analysis
        weight: 0.15
        params:
          commands: [biome, tsc]
      - type: tool_calls
        weight: 0.20
        params:
          sequence: [read_file, edit_file, run_tests]
      - type: llm_rubric
        weight: 0.25
        params:
          rubric: |
            Evaluate the code change for:
            - Correctness: Does it solve the problem?
            - Style: Does it follow project conventions?
            - Safety: Does it avoid introducing bugs?
      - type: state_check
        weight: 0.10
    tracked_metrics:
      - type: transcript
        metrics: [n_turns, n_toolcalls, n_total_tokens]

  conversational:
    description: "Conversational agent evaluation"
    primary_graders:
      - type: llm_rubric
        weight: 0.40
        params:
          rubric: |
            Evaluate the conversation for:
            - Task Resolution: Was the user's issue resolved?
            - Empathy: Did the agent acknowledge frustration?
            - Clarity: Were explanations clear?
            - Groundedness: Were claims backed by tool results?
      - type: natural_language_assert
        weight: 0.25
        params:
          assertions:
            - "Agent showed empathy for customer's situation"
            - "Resolution was clearly explained"
            - "Agent's response was grounded in tool results"
      - type: state_check
        weight: 0.15
        params:
          expect:
            status: resolved
      - type: tool_calls
        weight: 0.20
        params:
          required:
            - tool: verify_identity
            - tool: send_confirmation
    tracked_metrics:
      - type: transcript
        metrics: [n_turns]
    constraints:
      max_turns: 10

  research:
    description: "Research agent evaluation"
    primary_graders:
      - type: llm_rubric
        weight: 0.50
        params:
          rubric: |
            Evaluate the research output for:
            - Groundedness: Are claims backed by sources?
            - Coverage: Are key topics addressed?
            - Source Quality: Are sources reputable?
            - Synthesis: Is information well-organized?
      - type: natural_language_assert
        weight: 0.30
        params:
          assertions:
            - "Claims are backed by cited sources"
            - "Multiple perspectives are considered"
            - "Sources are from reputable domains"
      - type: tool_calls
        weight: 0.20
        params:
          required:
            - tool: web_search
            - tool: read_source
    tracked_metrics:
      - type: transcript
        metrics: [n_turns, n_toolcalls]

  computer_use:
    description: "Computer/GUI agent evaluation"
    primary_graders:
      - type: state_check
        weight: 0.40
        params:
          verify_gui: true
      - type: tool_calls
        weight: 0.30
        params:
          required:
            - tool: screenshot
            - tool: click
      - type: llm_rubric
        weight: 0.30
        params:
          rubric: |
            Evaluate the GUI interaction for:
            - Efficiency: Were actions minimal and direct?
            - Correctness: Was the target element correct?
            - Error Handling: Were errors handled gracefully?
    tracked_metrics:
      - type: latency
        metrics: [time_to_last_token]

  general:
    description: "General-purpose evaluation"
    primary_graders:
      - type: llm_rubric
        weight: 0.50
        params:
          rubric: |
            Evaluate the output for:
            - Accuracy: Is the information correct?
            - Completeness: Is the task fully addressed?
            - Quality: Is the output well-structured?
      - type: natural_language_assert
        weight: 0.30
        params:
          assertions:
            - "The output addresses the user's request"
            - "The information appears accurate"
      - type: tool_calls
        weight: 0.20
    tracked_metrics:
      - type: transcript
        metrics: [n_turns, n_toolcalls]

# Default thresholds by eval type
thresholds:
  capability:
    pass_threshold: 0.70
    saturation_threshold: 0.95
    description: "Stretch goals - expecting <100% pass rate"

  regression:
    pass_threshold: 0.95
    alert_threshold: 0.90
    description: "Quality gates - must maintain near 100%"

/**
 * Base Grader Interface
 * All graders implement this interface for consistent execution
 */

import type { GraderConfig, GraderResult, Transcript, GraderType } from '../Types/index.ts';

export interface GraderContext {
  task_id: string;
  trial_id: string;
  transcript: Transcript;
  output: string;  // Final output text
  working_dir?: string;
  reference?: string;  // Reference/golden output if available
}

export abstract class BaseGrader {
  abstract type: GraderType;
  abstract category: 'code_based' | 'model_based' | 'human';

  protected config: GraderConfig;

  constructor(config: GraderConfig) {
    this.config = config;
  }

  /**
   * Execute the grader and return results
   */
  abstract grade(context: GraderContext): Promise<GraderResult>;

  /**
   * Get the weight for this grader
   */
  getWeight(): number {
    return this.config.weight ?? 1.0;
  }

  /**
   * Check if this grader is required (task fails if grader fails)
   */
  isRequired(): boolean {
    return this.config.required ?? false;
  }

  /**
   * Create a result object
   */
  protected createResult(
    score: number,
    passed: boolean,
    duration_ms: number,
    options?: {
      reasoning?: string;
      details?: Record<string, unknown>;
    }
  ): GraderResult {
    return {
      grader_type: this.type,
      weight: this.getWeight(),
      score,
      passed,
      duration_ms,
      reasoning: options?.reasoning,
      details: options?.details,
    };
  }
}

/**
 * Grader registry for dynamic instantiation
 */
const graderRegistry = new Map<GraderType, new (config: GraderConfig) => BaseGrader>();

export function registerGrader(type: GraderType, graderClass: new (config: GraderConfig) => BaseGrader): void {
  graderRegistry.set(type, graderClass);
}

export function createGrader(config: GraderConfig): BaseGrader {
  const GraderClass = graderRegistry.get(config.type);
  if (!GraderClass) {
    throw new Error(`Unknown grader type: ${config.type}`);
  }
  return new GraderClass(config);
}

export function listGraders(): GraderType[] {
  return Array.from(graderRegistry.keys());
}

/**
 * Run multiple graders and aggregate results
 */
export async function runGraders(
  graders: BaseGrader[],
  context: GraderContext
): Promise<{ results: GraderResult[]; aggregate_score: number; passed: boolean }> {
  const results: GraderResult[] = [];
  let totalWeight = 0;
  let weightedSum = 0;
  let allRequiredPassed = true;

  for (const grader of graders) {
    const result = await grader.grade(context);
    results.push(result);

    // Aggregate
    const weight = grader.getWeight();
    totalWeight += weight;
    weightedSum += result.score * weight;

    // Check required
    if (grader.isRequired() && !result.passed) {
      allRequiredPassed = false;
    }
  }

  const aggregate_score = totalWeight > 0 ? weightedSum / totalWeight : 0;
  const passed = allRequiredPassed && aggregate_score >= 0.5;  // Default threshold

  return { results, aggregate_score, passed };
}

/**
 * Binary Tests Grader
 * Run actual test files and check pass/fail
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, BinaryTestsParams } from '../../Types/index.ts';
import { $ } from 'bun';

export class BinaryTestsGrader extends BaseGrader {
  type = 'binary_tests' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as BinaryTestsParams;

    if (!params?.test_files?.length) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No test files configured',
      });
    }

    const workingDir = context.working_dir ?? process.cwd();
    const timeout = params.timeout_ms ?? 60000;
    const results: { file: string; passed: boolean; output: string; error?: string }[] = [];

    for (const testFile of params.test_files) {
      try {
        // Detect test command based on file extension
        const command = params.test_command ?? this.detectTestCommand(testFile);

        const result = await $`cd ${workingDir} && timeout ${Math.ceil(timeout/1000)} ${command} ${testFile}`
          .quiet()
          .nothrow();

        const passed = result.exitCode === 0;
        results.push({
          file: testFile,
          passed,
          output: result.stdout.toString().slice(-500),  // Last 500 chars
          error: passed ? undefined : result.stderr.toString().slice(-500),
        });
      } catch (e) {
        results.push({
          file: testFile,
          passed: false,
          output: '',
          error: String(e),
        });
      }
    }

    const passCount = results.filter(r => r.passed).length;
    const score = passCount / params.test_files.length;
    const passed = passCount === params.test_files.length;

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `${passCount}/${params.test_files.length} tests passed`,
      details: {
        results,
        working_dir: workingDir,
      },
    });
  }

  private detectTestCommand(file: string): string {
    if (file.endsWith('.py')) return 'python -m pytest';
    if (file.endsWith('.ts')) return 'bun test';
    if (file.endsWith('.js')) return 'node --test';
    if (file.endsWith('.go')) return 'go test';
    if (file.endsWith('.rs')) return 'cargo test --';
    return 'bun test';  // Default
  }
}

registerGrader('binary_tests', BinaryTestsGrader);

/**
 * Code-Based Graders Index
 * Fast, deterministic graders
 */

// Import to register graders
import './StringMatch.ts';
import './RegexMatch.ts';
import './BinaryTests.ts';
import './StaticAnalysis.ts';
import './StateCheck.ts';
import './ToolCallVerification.ts';

export { StringMatchGrader } from './StringMatch.ts';
export { RegexMatchGrader } from './RegexMatch.ts';
export { BinaryTestsGrader } from './BinaryTests.ts';
export { StaticAnalysisGrader } from './StaticAnalysis.ts';
export { StateCheckGrader } from './StateCheck.ts';
export { ToolCallVerificationGrader } from './ToolCallVerification.ts';

/**
 * Regex Match Grader
 * Pattern matching with regular expressions
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, RegexMatchParams } from '../../Types/index.ts';

export class RegexMatchGrader extends BaseGrader {
  type = 'regex_match' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as RegexMatchParams;

    if (!params?.patterns?.length) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No patterns configured',
      });
    }

    const flags = params.flags ?? 'gm';
    const results = params.patterns.map(pattern => {
      try {
        const regex = new RegExp(pattern, flags);
        const matched = regex.test(context.output);
        return { pattern, matched, error: null };
      } catch (e) {
        return { pattern, matched: false, error: String(e) };
      }
    });

    const matchCount = results.filter(r => r.matched).length;
    const errorCount = results.filter(r => r.error).length;

    let passed: boolean;
    let score: number;

    if (params.mode === 'all') {
      passed = matchCount === params.patterns.length;
      score = matchCount / params.patterns.length;
    } else {
      passed = matchCount > 0;
      score = passed ? 1 : 0;
    }

    // Penalize for errors
    if (errorCount > 0) {
      score *= (params.patterns.length - errorCount) / params.patterns.length;
    }

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `Matched ${matchCount}/${params.patterns.length} patterns${errorCount > 0 ? ` (${errorCount} errors)` : ''} (mode: ${params.mode})`,
      details: {
        results,
        mode: params.mode,
        flags,
      },
    });
  }
}

registerGrader('regex_match', RegexMatchGrader);

/**
 * State Check Grader
 * Verify system state after agent execution
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, StateCheckParams } from '../../Types/index.ts';
import { existsSync, readFileSync } from 'fs';
import { join } from 'path';

export class StateCheckGrader extends BaseGrader {
  type = 'state_check' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as StateCheckParams;

    const checks: { check: string; passed: boolean; expected?: unknown; actual?: unknown }[] = [];
    const workingDir = context.working_dir ?? process.cwd();

    // Check expected state object (e.g., security_logs: {event_type: "auth_blocked"})
    if (params.expect) {
      for (const [key, expected] of Object.entries(params.expect)) {
        const checkResult = await this.checkState(key, expected, context);
        checks.push({
          check: `state.${key}`,
          passed: checkResult.passed,
          expected,
          actual: checkResult.actual,
        });
      }
    }

    // Check file contents
    if (params.check_files) {
      for (const fileCheck of params.check_files) {
        const filePath = join(workingDir, fileCheck.path);

        if (!existsSync(filePath)) {
          checks.push({
            check: `file.${fileCheck.path}`,
            passed: false,
            expected: 'file exists',
            actual: 'file not found',
          });
          continue;
        }

        const content = readFileSync(filePath, 'utf-8');

        // Check contains
        if (fileCheck.contains) {
          for (const pattern of fileCheck.contains) {
            const found = content.includes(pattern);
            checks.push({
              check: `file.${fileCheck.path}.contains`,
              passed: found,
              expected: pattern,
              actual: found ? 'found' : 'not found',
            });
          }
        }

        // Check not_contains
        if (fileCheck.not_contains) {
          for (const pattern of fileCheck.not_contains) {
            const found = content.includes(pattern);
            checks.push({
              check: `file.${fileCheck.path}.not_contains`,
              passed: !found,
              expected: `NOT: ${pattern}`,
              actual: found ? 'found (should not exist)' : 'not found (correct)',
            });
          }
        }
      }
    }

    // Check environment variables
    if (params.check_env) {
      for (const [key, expected] of Object.entries(params.check_env)) {
        const actual = process.env[key];
        checks.push({
          check: `env.${key}`,
          passed: actual === expected,
          expected,
          actual,
        });
      }
    }

    const passCount = checks.filter(c => c.passed).length;
    const score = checks.length > 0 ? passCount / checks.length : 1;
    const passed = passCount === checks.length;

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `${passCount}/${checks.length} state checks passed`,
      details: { checks },
    });
  }

  private async checkState(
    key: string,
    expected: unknown,
    context: GraderContext
  ): Promise<{ passed: boolean; actual?: unknown }> {
    // Check if expected state exists in the transcript's final outcome
    if (context.transcript.final_outcome) {
      const outcome = context.transcript.final_outcome as Record<string, unknown>;
      if (key in outcome) {
        const actual = outcome[key];
        const passed = this.deepEqual(actual, expected);
        return { passed, actual };
      }
    }

    // Also check in the output text for JSON-like patterns
    try {
      const jsonMatch = context.output.match(/\{[\s\S]*\}/);
      if (jsonMatch) {
        const parsed = JSON.parse(jsonMatch[0]);
        if (key in parsed) {
          const actual = parsed[key];
          const passed = this.deepEqual(actual, expected);
          return { passed, actual };
        }
      }
    } catch {
      // Not valid JSON, continue
    }

    return { passed: false, actual: undefined };
  }

  private deepEqual(a: unknown, b: unknown): boolean {
    if (a === b) return true;
    if (typeof a !== typeof b) return false;
    if (typeof a !== 'object' || a === null || b === null) return false;

    const aObj = a as Record<string, unknown>;
    const bObj = b as Record<string, unknown>;

    // For expected, check if all expected keys match (subset matching)
    for (const key of Object.keys(bObj)) {
      if (!(key in aObj)) return false;
      if (!this.deepEqual(aObj[key], bObj[key])) return false;
    }

    return true;
  }
}

registerGrader('state_check', StateCheckGrader);

/**
 * Static Analysis Grader
 * Run linters, type checkers, security scanners
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, StaticAnalysisParams } from '../../Types/index.ts';
import { $ } from 'bun';

export class StaticAnalysisGrader extends BaseGrader {
  type = 'static_analysis' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as StaticAnalysisParams;

    if (!params?.commands?.length) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No analysis commands configured',
      });
    }

    const workingDir = context.working_dir ?? process.cwd();
    const results: { command: string; passed: boolean; output: string; warnings: number; errors: number }[] = [];

    for (const command of params.commands) {
      try {
        const result = await $`cd ${workingDir} && ${command}`.quiet().nothrow();

        const output = result.stdout.toString() + result.stderr.toString();
        const warnings = this.countIssues(output, 'warning');
        const errors = this.countIssues(output, 'error');

        // Pass if no errors (and no warnings if fail_on_warning is set)
        const passed = errors === 0 && (!params.fail_on_warning || warnings === 0);

        results.push({
          command,
          passed,
          output: output.slice(-1000),  // Last 1000 chars
          warnings,
          errors,
        });
      } catch (e) {
        results.push({
          command,
          passed: false,
          output: String(e),
          warnings: 0,
          errors: 1,
        });
      }
    }

    const passCount = results.filter(r => r.passed).length;
    const totalErrors = results.reduce((sum, r) => sum + r.errors, 0);
    const totalWarnings = results.reduce((sum, r) => sum + r.warnings, 0);

    const score = passCount / params.commands.length;
    const passed = passCount === params.commands.length;

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `${passCount}/${params.commands.length} checks passed (${totalErrors} errors, ${totalWarnings} warnings)`,
      details: {
        results,
        total_errors: totalErrors,
        total_warnings: totalWarnings,
        fail_on_warning: params.fail_on_warning ?? false,
      },
    });
  }

  private countIssues(output: string, type: 'warning' | 'error'): number {
    const patterns = type === 'error'
      ? [/error:/gi, /\berror\b/gi, /failed/gi, /\[E\d+\]/g]
      : [/warning:/gi, /\bwarn\b/gi, /\[W\d+\]/g];

    let count = 0;
    for (const pattern of patterns) {
      const matches = output.match(pattern);
      count += matches?.length ?? 0;
    }
    return Math.min(count, 100);  // Cap at 100
  }
}

registerGrader('static_analysis', StaticAnalysisGrader);

/**
 * String Match Grader
 * Fast deterministic check for exact or pattern matching
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, StringMatchParams } from '../../Types/index.ts';

export class StringMatchGrader extends BaseGrader {
  type = 'string_match' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as StringMatchParams;

    if (!params?.patterns?.length) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No patterns configured',
      });
    }

    const output = params.case_sensitive ? context.output : context.output.toLowerCase();
    const patterns = params.patterns.map(p =>
      params.case_sensitive ? p : p.toLowerCase()
    );

    const matches = patterns.map(pattern => output.includes(pattern));
    const matchCount = matches.filter(Boolean).length;

    let passed: boolean;
    let score: number;

    if (params.mode === 'all') {
      passed = matchCount === patterns.length;
      score = matchCount / patterns.length;
    } else {
      // 'any' mode
      passed = matchCount > 0;
      score = passed ? 1 : 0;
    }

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `Matched ${matchCount}/${patterns.length} patterns (mode: ${params.mode})`,
      details: {
        patterns,
        matches: patterns.map((p, i) => ({ pattern: p, matched: matches[i] })),
        mode: params.mode,
      },
    });
  }
}

registerGrader('string_match', StringMatchGrader);

/**
 * Tool Call Verification Grader
 * Verify that specific tools were called with expected parameters
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, ToolCallsParams } from '../../Types/index.ts';

export class ToolCallVerificationGrader extends BaseGrader {
  type = 'tool_calls' as const;
  category = 'code_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as ToolCallsParams;
    const toolCalls = context.transcript.tool_calls;

    const checks: { check: string; passed: boolean; details?: string }[] = [];

    // Check required tool calls
    if (params.required) {
      for (const req of params.required) {
        const matchingCall = toolCalls.find(tc => {
          if (tc.name !== req.tool) return false;

          // If params specified, check they match
          if (req.params) {
            for (const [key, expected] of Object.entries(req.params)) {
              const actual = tc.params[key];

              // Support glob patterns for paths
              if (typeof expected === 'string' && expected.includes('*')) {
                const pattern = new RegExp('^' + expected.replace(/\*/g, '.*') + '$');
                if (!pattern.test(String(actual))) return false;
              } else if (actual !== expected) {
                return false;
              }
            }
          }

          return true;
        });

        checks.push({
          check: `required.${req.tool}`,
          passed: !!matchingCall,
          details: matchingCall
            ? `Found: ${JSON.stringify(matchingCall.params).slice(0, 100)}`
            : `Not found in ${toolCalls.length} tool calls`,
        });
      }
    }

    // Check forbidden tool calls
    if (params.forbidden) {
      for (const forbidden of params.forbidden) {
        const found = toolCalls.some(tc => tc.name === forbidden);
        checks.push({
          check: `forbidden.${forbidden}`,
          passed: !found,
          details: found ? 'Found (should not exist)' : 'Not found (correct)',
        });
      }
    }

    // Check sequence (tools must be called in order)
    if (params.sequence) {
      const toolOrder = toolCalls.map(tc => tc.name);
      let seqIndex = 0;

      for (const tool of toolOrder) {
        if (seqIndex < params.sequence.length && tool === params.sequence[seqIndex]) {
          seqIndex++;
        }
      }

      const sequenceComplete = seqIndex === params.sequence.length;
      checks.push({
        check: 'sequence',
        passed: sequenceComplete,
        details: sequenceComplete
          ? `Sequence complete: ${params.sequence.join(' → ')}`
          : `Incomplete: found ${seqIndex}/${params.sequence.length} in order`,
      });
    }

    // Check max calls
    if (params.max_calls !== undefined) {
      const withinLimit = toolCalls.length <= params.max_calls;
      checks.push({
        check: 'max_calls',
        passed: withinLimit,
        details: `${toolCalls.length} calls (max: ${params.max_calls})`,
      });
    }

    const passCount = checks.filter(c => c.passed).length;
    const score = checks.length > 0 ? passCount / checks.length : 1;
    const passed = passCount === checks.length;

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `${passCount}/${checks.length} tool call checks passed`,
      details: {
        checks,
        total_tool_calls: toolCalls.length,
        tool_call_summary: toolCalls.map(tc => tc.name),
      },
    });
  }
}

registerGrader('tool_calls', ToolCallVerificationGrader);

/**
 * Graders Index
 * Central export for all grader types
 */

// Base
export * from './Base.ts';

// Code-based graders
export * from './CodeBased/index.ts';

// Model-based graders
export * from './ModelBased/index.ts';

// Note: Human graders require separate implementation
// See Graders/Human/ for review workflow

/**
 * Model-Based Graders Index
 * LLM-powered graders for nuanced evaluation
 */

// Import to register graders
import './LLMRubric.ts';
import './NaturalLanguageAssert.ts';
import './PairwiseComparison.ts';

export { LLMRubricGrader } from './LLMRubric.ts';
export { NaturalLanguageAssertGrader } from './NaturalLanguageAssert.ts';
export { PairwiseComparisonGrader } from './PairwiseComparison.ts';

/**
 * LLM Rubric Grader
 * Score output against a detailed rubric using an LLM judge
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, LLMRubricParams } from '../../Types/index.ts';
import { inference, type InferenceLevel } from '../../../../PAI/TOOLS/Inference.ts';
import { readFileSync, existsSync } from 'fs';

export class LLMRubricGrader extends BaseGrader {
  type = 'llm_rubric' as const;
  category = 'model_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as LLMRubricParams;

    // Load rubric
    let rubric = params.rubric;
    if (existsSync(params.rubric)) {
      rubric = readFileSync(params.rubric, 'utf-8');
    }

    const scale = params.scale ?? '1-5';
    // Map model preference to inference level (default to standard/Sonnet)
    const levelMap: Record<string, InferenceLevel> = {
      'claude-haiku-4-5-20251001': 'fast',
      'claude-sonnet-4-6': 'standard',
      'claude-opus-4-6': 'smart',
      'claude-sonnet-4-20250514': 'standard',
      'claude-opus-4-20250514': 'smart',
    };
    const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'standard';

    // Build prompt
    const systemPrompt = this.buildSystemPrompt(scale, params.reasoning_first ?? true);
    const userPrompt = this.buildUserPrompt(rubric, params.assertions, context);

    try {
      const result = await inference({
        systemPrompt,
        userPrompt,
        level,
        timeout: 30000,
      });

      if (!result.success) {
        throw new Error(result.error || 'Inference failed');
      }

      const text = result.output;
      const { score, reasoning, assertion_results } = this.parseResponse(text, scale, params.assertions);

      const passed = this.scoreToPassed(score, scale);

      return this.createResult(score, passed, performance.now() - start, {
        reasoning,
        details: {
          assertion_results,
          inference_level: level,
          scale,
          raw_response: text,
        },
      });
    } catch (e) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: `LLM judge error: ${e}`,
      });
    }
  }

  private buildSystemPrompt(scale: string, reasoningFirst: boolean): string {
    const scaleInstructions = {
      '1-5': 'Score from 1 (very poor) to 5 (excellent)',
      '1-10': 'Score from 1 (very poor) to 10 (excellent)',
      'pass-fail': 'Determine if the output PASSES or FAILS the criteria',
    }[scale];

    const format = reasoningFirst
      ? `First explain your reasoning, then provide your score. Format:
REASONING: <your detailed analysis>
SCORE: <your score>`
      : `Provide your score first, then explain. Format:
SCORE: <your score>
REASONING: <your explanation>`;

    return `You are an expert evaluator assessing AI-generated output against quality criteria.

${scaleInstructions}

${format}

Be objective and fair. Consider both strengths and weaknesses.`;
  }

  private buildUserPrompt(
    rubric: string,
    assertions: string[] | undefined,
    context: GraderContext
  ): string {
    let prompt = `## Evaluation Rubric

${rubric}

## Output to Evaluate

${context.output}
`;

    if (assertions?.length) {
      prompt += `
## Specific Assertions to Check

For each assertion, determine if it is TRUE or FALSE:

${assertions.map((a, i) => `${i + 1}. ${a}`).join('\n')}

After the main evaluation, provide assertion results in this format:
ASSERTIONS:
${assertions.map((_, i) => `${i + 1}. TRUE/FALSE`).join('\n')}
`;
    }

    if (context.reference) {
      prompt += `
## Reference Output (for comparison)

${context.reference}
`;
    }

    prompt += `
## Your Evaluation

Evaluate the output against the rubric and provide your assessment.`;

    return prompt;
  }

  private parseResponse(
    text: string,
    scale: string,
    assertions?: string[]
  ): { score: number; reasoning: string; assertion_results?: boolean[] } {
    // Extract score
    let score = 0;
    const scoreMatch = text.match(/SCORE:\s*(\d+(?:\.\d+)?|PASS|FAIL)/i);

    if (scoreMatch) {
      if (scale === 'pass-fail') {
        score = scoreMatch[1].toUpperCase() === 'PASS' ? 1 : 0;
      } else if (scale === '1-5') {
        score = (parseFloat(scoreMatch[1]) - 1) / 4;  // Normalize to 0-1
      } else if (scale === '1-10') {
        score = (parseFloat(scoreMatch[1]) - 1) / 9;  // Normalize to 0-1
      }
    }

    // Extract reasoning
    const reasoningMatch = text.match(/REASONING:\s*([\s\S]*?)(?=SCORE:|ASSERTIONS:|$)/i);
    const reasoning = reasoningMatch?.[1]?.trim() ?? text;

    // Extract assertion results
    let assertion_results: boolean[] | undefined;
    if (assertions?.length) {
      const assertionsMatch = text.match(/ASSERTIONS:\s*([\s\S]*?)$/i);
      if (assertionsMatch) {
        assertion_results = assertions.map((_, i) => {
          const lineMatch = assertionsMatch[1].match(new RegExp(`${i + 1}\\.\\s*(TRUE|FALSE)`, 'i'));
          return lineMatch?.[1]?.toUpperCase() === 'TRUE';
        });
      }
    }

    return { score: Math.max(0, Math.min(1, score)), reasoning, assertion_results };
  }

  private scoreToPassed(score: number, scale: string): boolean {
    if (scale === 'pass-fail') return score >= 0.5;
    // For 1-5 and 1-10, pass if score is above middle
    return score >= 0.5;
  }
}

registerGrader('llm_rubric', LLMRubricGrader);

/**
 * Natural Language Assertion Grader
 * Check if specific assertions are true about the output
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, NaturalLanguageAssertParams } from '../../Types/index.ts';
import { inference, type InferenceLevel } from '../../../../PAI/TOOLS/Inference.ts';

export class NaturalLanguageAssertGrader extends BaseGrader {
  type = 'natural_language_assert' as const;
  category = 'model_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as NaturalLanguageAssertParams;

    if (!params?.assertions?.length) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No assertions configured',
      });
    }

    // Map model preference to inference level (default to standard/Sonnet)
    const levelMap: Record<string, InferenceLevel> = {
      'claude-haiku-4-5-20251001': 'fast',
      'claude-sonnet-4-6': 'standard',
      'claude-opus-4-6': 'smart',
      'claude-sonnet-4-20250514': 'standard',
      'claude-opus-4-20250514': 'smart',
    };
    const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'standard';
    const requireAll = params.require_all ?? true;

    const systemPrompt = `You are an assertion checker. For each assertion, determine if it is TRUE or FALSE based on the given output.

Be strict and literal. If you cannot clearly verify an assertion, mark it FALSE.

Respond in this exact format for each assertion:
1. TRUE/FALSE: <brief explanation>
2. TRUE/FALSE: <brief explanation>
...`;

    const userPrompt = `## Output to Check

${context.output}

## Tool Calls Made (for context)

${context.transcript.tool_calls.map(tc => `- ${tc.name}(${JSON.stringify(tc.params)})`).join('\n') || 'None'}

## Assertions to Verify

${params.assertions.map((a, i) => `${i + 1}. ${a}`).join('\n')}

Check each assertion against the output and tool calls.`;

    try {
      const result = await inference({
        systemPrompt,
        userPrompt,
        level,
        timeout: 30000,
      });

      if (!result.success) {
        throw new Error(result.error || 'Inference failed');
      }

      const text = result.output;
      const results = this.parseResults(text, params.assertions);

      const passCount = results.filter(r => r.passed).length;
      const score = passCount / params.assertions.length;

      const passed = requireAll
        ? passCount === params.assertions.length
        : passCount > 0;

      return this.createResult(score, passed, performance.now() - start, {
        reasoning: `${passCount}/${params.assertions.length} assertions passed`,
        details: {
          results,
          require_all: requireAll,
          inference_level: level,
        },
      });
    } catch (e) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: `LLM assertion check error: ${e}`,
      });
    }
  }

  private parseResults(
    text: string,
    assertions: string[]
  ): { assertion: string; passed: boolean; explanation: string }[] {
    return assertions.map((assertion, i) => {
      const pattern = new RegExp(`${i + 1}\\.\\s*(TRUE|FALSE):\\s*(.*)`, 'i');
      const match = text.match(pattern);

      if (match) {
        return {
          assertion,
          passed: match[1].toUpperCase() === 'TRUE',
          explanation: match[2].trim(),
        };
      }

      // Try to find by content if numbered format didn't work
      const containsTrue = text.toLowerCase().includes(`assertion ${i + 1}`) &&
        text.toLowerCase().includes('true');

      return {
        assertion,
        passed: containsTrue,
        explanation: 'Could not parse result',
      };
    });
  }
}

registerGrader('natural_language_assert', NaturalLanguageAssertGrader);

/**
 * Pairwise Comparison Grader
 * Compare output against a reference with position swapping to reduce bias
 */

import { BaseGrader, registerGrader, type GraderContext } from '../Base.ts';
import type { GraderConfig, GraderResult, PairwiseComparisonParams } from '../../Types/index.ts';
import { inference, type InferenceLevel } from '../../../../PAI/TOOLS/Inference.ts';
import { readFileSync, existsSync } from 'fs';

export class PairwiseComparisonGrader extends BaseGrader {
  type = 'pairwise_comparison' as const;
  category = 'model_based' as const;

  async grade(context: GraderContext): Promise<GraderResult> {
    const start = performance.now();
    const params = this.config.params as PairwiseComparisonParams;

    // Load reference
    let reference = params.reference;
    if (existsSync(params.reference)) {
      reference = readFileSync(params.reference, 'utf-8');
    }

    if (!reference) {
      return this.createResult(0, false, performance.now() - start, {
        reasoning: 'No reference output available',
      });
    }

    // Map model preference to inference level (default to standard/Sonnet)
    const levelMap: Record<string, InferenceLevel> = {
      'claude-haiku-4-5-20251001': 'fast',
      'claude-sonnet-4-6': 'standard',
      'claude-opus-4-6': 'smart',
      'claude-sonnet-4-20250514': 'standard',
      'claude-opus-4-20250514': 'smart',
    };
    const level: InferenceLevel = levelMap[params.judge_model ?? ''] ?? 'standard';
    const positionSwap = params.position_swap ?? true;

    // Run comparison(s)
    const results: { position: string; winner: 'A' | 'B' | 'tie'; reasoning: string }[] = [];

    // First comparison: Output = A, Reference = B
    const result1 = await this.compare(context.output, reference, level, params.criteria);
    results.push({ position: 'output_first', ...result1 });

    if (positionSwap) {
      // Second comparison: Reference = A, Output = B
      const result2 = await this.compare(reference, context.output, level, params.criteria);
      // Flip winner since positions are swapped
      const flippedWinner = result2.winner === 'A' ? 'B' : result2.winner === 'B' ? 'A' : 'tie';
      results.push({
        position: 'reference_first',
        winner: flippedWinner as 'A' | 'B' | 'tie',
        reasoning: result2.reasoning,
      });
    }

    // Aggregate results
    const outputWins = results.filter(r => r.winner === 'A').length;
    const referenceWins = results.filter(r => r.winner === 'B').length;
    const ties = results.filter(r => r.winner === 'tie').length;

    let score: number;
    let aggregateWinner: string;

    if (outputWins > referenceWins) {
      score = 1.0;
      aggregateWinner = 'output';
    } else if (referenceWins > outputWins) {
      score = 0.0;
      aggregateWinner = 'reference';
    } else {
      score = 0.5;
      aggregateWinner = 'tie';
    }

    // For the score, also consider partial wins
    if (positionSwap && results.length === 2) {
      score = (outputWins + ties * 0.5) / 2;
    }

    const passed = score >= 0.5;

    return this.createResult(score, passed, performance.now() - start, {
      reasoning: `${aggregateWinner} wins (output: ${outputWins}, reference: ${referenceWins}, ties: ${ties})`,
      details: {
        results,
        position_swap: positionSwap,
        inference_level: level,
        criteria: params.criteria,
      },
    });
  }

  private async compare(
    outputA: string,
    outputB: string,
    level: InferenceLevel,
    criteria?: string[]
  ): Promise<{ winner: 'A' | 'B' | 'tie'; reasoning: string }> {
    const criteriaText = criteria?.length
      ? `Focus on these criteria:\n${criteria.map(c => `- ${c}`).join('\n')}`
      : 'Consider overall quality, accuracy, clarity, and helpfulness.';

    const systemPrompt = `You are comparing two outputs to determine which is better.

${criteriaText}

Respond in this format:
REASONING: <your analysis comparing A and B>
WINNER: A or B or TIE

Be objective. Consider both outputs fairly.`;

    const userPrompt = `## Output A

${outputA}

## Output B

${outputB}

Compare these outputs and determine which is better.`;

    try {
      const result = await inference({
        systemPrompt,
        userPrompt,
        level,
        timeout: 30000,
      });

      if (!result.success) {
        throw new Error(result.error || 'Inference failed');
      }

      const text = result.output;

      const winnerMatch = text.match(/WINNER:\s*(A|B|TIE)/i);
      const reasoningMatch = text.match(/REASONING:\s*([\s\S]*?)(?=WINNER:|$)/i);

      const winner = winnerMatch?.[1]?.toUpperCase() === 'A' ? 'A'
        : winnerMatch?.[1]?.toUpperCase() === 'B' ? 'B'
        : 'tie';

      return {
        winner,
        reasoning: reasoningMatch?.[1]?.trim() ?? text,
      };
    } catch (e) {
      return {
        winner: 'tie',
        reasoning: `Comparison error: ${e}`,
      };
    }
  }
}

registerGrader('pairwise_comparison', PairwiseComparisonGrader);

{
  "name": "@pai/skill-evals",
  "version": "1.1.0",
  "type": "module",
  "description": "AI agent evaluation framework for PAI — graders, pass@k scoring, multi-turn scenarios",
  "scripts": {
    "scenario": "bun run Tools/ScenarioRunner.ts"
  },
  "dependencies": {
    "@ai-sdk/anthropic": "2.0.74",
    "@langwatch/scenario": "0.4.10",
    "ai": "6.0.159"
  },
  "devDependencies": {
    "@types/bun": "latest",
    "typescript": "^5.0.0"
  }
}

{"type":"run.started","data":{"runId":"run_1763331985105_pjbi3p","timestamp":"2025-11-16T22:26:25.106Z","config":{"promptVersion":"v1","testCases":["test-001"],"criteria":["proper-markdown-links","summary-length-check","no-date-in-urls"],"model":"claude-sonnet-3.5","provider":"anthropic"}}}
{"type":"phase.started","data":{"phase":"loading","timestamp":"2025-11-16T22:26:25.107Z"}}
{"type":"phase.started","data":{"phase":"generating","timestamp":"2025-11-16T22:26:25.107Z"}}

{
  "runId": "run_1763331985105_pjbi3p",
  "useCaseId": "categorize-summarize-rate",
  "timestamp": "2025-11-16T22:26:25.108Z",
  "status": "failed",
  "config": {
    "promptVersion": "v1",
    "testCases": [
      "test-001"
    ],
    "criteria": [
      "proper-markdown-links",
      "summary-length-check",
      "no-date-in-urls"
    ],
    "model": "claude-sonnet-3.5",
    "provider": "anthropic"
  }
}

{
  "runId": "run_1763331985105_pjbi3p",
  "useCaseId": "categorize-summarize-rate",
  "timestamp": "2025-11-16T22:26:25.108Z",
  "status": "failed",
  "config": {
    "promptVersion": "v1",
    "testCases": [
      "test-001"
    ],
    "criteria": [
      "proper-markdown-links",
      "summary-length-check",
      "no-date-in-urls"
    ],
    "model": "claude-sonnet-3.5",
    "provider": "anthropic"
  },
  "results": [],
  "finalScore": 0,
  "weightedScore": 0,
  "summary": {
    "totalCriteria": 3,
    "passed": 0,
    "failed": 0,
    "deterministicScore": 0,
    "aiJudgeScore": 0
  }
}

{"type":"run.started","data":{"runId":"run_1763335202718_bh27iw","timestamp":"2025-11-16T23:20:02.719Z","config":{"promptVersion":"v1","testCaseId":"test-001","criteria":[{"id":"proper-markdown-links","type":"markdown-links","name":"Proper Markdown link formatting","description":"All links must be properly formatted as [TEXT](URL)","weight":1},{"id":"summary-length-check","type":"summary-length","name":"Summary sentence under 32 characters","description":"The summary sentence must be less than 32 characters","weight":1},{"id":"no-date-in-urls","type":"no-url-dates","name":"No dates appended to URLs","description":"URLs must not have arbitrary dates appended","weight":1}],"model":"claude-sonnet-3.5","provider":"anthropic"}}}
{"type":"phase.started","data":{"phase":"loading","timestamp":"2025-11-16T23:20:02.720Z"}}
{"type":"phase.started","data":{"phase":"generating","timestamp":"2025-11-16T23:20:02.720Z"}}

{
  "runId": "run_1763335202718_bh27iw",
  "useCaseId": "categorize-summarize-rate",
  "timestamp": "2025-11-16T23:20:02.720Z",
  "status": "failed",
  "config": {
    "promptVersion": "v1",
    "testCaseId": "test-001",
    "criteria": [
      {
        "id": "proper-markdown-links",
        "type": "markdown-links",
        "name": "Proper Markdown link formatting",
        "description": "All links must be properly formatted as [TEXT](URL)",
        "weight": 1
      },
      {
        "id": "summary-length-check",
        "type": "summary-length",
        "name": "Summary sentence under 32 characters",
        "description": "The summary sentence must be less than 32 characters",
        "weight": 1
      },
      {
        "id": "no-date-in-urls",
        "type": "no-url-dates",
        "name": "No dates appended to URLs",
        "description": "URLs must not have arbitrary dates appended",
        "weight": 1
      }
    ],
    "model": "claude-sonnet-3.5",
    "provider": "anthropic"
  }
}

{
  "runId": "run_1763335202718_bh27iw",
  "useCaseId": "categorize-summarize-rate",
  "timestamp": "2025-11-16T23:20:02.720Z",
  "status": "failed",
  "config": {
    "promptVersion": "v1",
    "testCaseId": "test-001",
    "criteria": [
      {
        "id": "proper-markdown-links",
        "type": "markdown-links",
        "name": "Proper Markdown link formatting",
        "description": "All links must be properly formatted as [TEXT](URL)",
        "weight": 1
      },
      {
        "id": "summary-length-check",
        "type": "summary-length",
        "name": "Summary sentence under 32 characters",
        "description": "The summary sentence must be less than 32 characters",
        "weight": 1
      },
      {
        "id": "no-date-in-urls",
        "type": "no-url-dates",
        "name": "No dates appended to URLs",
        "description": "URLs must not have arbitrary dates appended",
        "weight": 1
      }
    ],
    "model": "claude-sonnet-3.5",
    "provider": "anthropic"
  },
  "results": [],
  "finalScore": 0,
  "weightedScore": 0,
  "summary": {
    "totalCriteria": 3,
    "passed": 0,
    "failed": 0,
    "deterministicScore": 0,
    "aiJudgeScore": 0
  }
}

{"type":"run.started","data":{"runId":"run_1763335222974_nu7hud","timestamp":"2025-11-16T23:20:22.975Z","config":{"promptVersion":"v1","testCaseId":"test-001","criteria":[{"id":"proper-markdown-links","type":"markdown-links","name":"Proper Markdown link formatting","description":"All links must be properly formatted as [TEXT](URL)","weight":1},{"id":"summary-length-check","type":"summary-length","name":"Summary sentence under 32 characters","description":"The summary sentence must be less than 32 characters","weight":1},{"id":"no-date-in-urls","type":"no-url-dates","name":"No dates appended to URLs","description":"URLs must not have arbitrary dates appended","weight":1}],"model":"claude-sonnet-3.5","provider":"anthropic"}}}
{"type":"phase.started","data":{"phase":"loading","timestamp":"2025-11-16T23:20:22.976Z"}}
{"type":"phase.started","data":{"phase":"generating","timestamp":"2025-11-16T23:20:22.976Z"}}

{
  "runId": "run_1763335222974_nu7hud",
  "useCaseId": "categorize-summarize-rate",
  "timestamp": "2025-11-16T23:20:22.976Z",
  "status": "failed",
  "config": {
    "promptVersion": "v1",
    "testCaseId": "test-001",
    "criteria": [
      {
        "id": "proper-markdown-links",
        "type": "markdown-links",
        "name": "Proper Markdown link formatting",
        "description": "All links must be properly formatted as [TEXT](URL)",
        "weight": 1
      },
      {
        "id": "summary-length-check",
        "type": "summary-length",
        "name": "Summary sentence under 32 characters",
        "description": "The summary sentence must be less than 32 characters",
        "weight": 1
      },
      {
        "id": "no-date-in-urls",
        "type": "no-url-dates",
        "name": "No dates appended to URLs",
        "description": "URLs must not have arbitrary dates appended",
        "weight": 1
      }
    ],
    "model": "claude-sonnet-3.5",
    "provider": "anthropic"
  }
}