Benchmark Fetcher

{
  "version": "1.0.0",
  "lastUpdated": "2025-12-14",
  "description": "Maps website-specific model names to manifest IDs for benchmark data extraction",
  "mappings": {
    "swebench": {
      "websiteModels": {
        "Claude 4.5 Opus medium (20251101)": "claude-opus-4-1",
        "Claude 4.5 Sonnet (20250929)": "claude-sonnet-4-5",
        "Claude 4 Opus (20250514)": "claude-opus-4",
        "Claude 4 Sonnet (20250514)": "claude-sonnet-4",
        "Gemini 3 Pro Preview (2025-11-18)": "gemini-3-pro",
        "Gemini 2.5 Pro (2025-05-06)": "gemini-2-5-pro",
        "Gemini 2.5 Flash (2025-04-17)": "gemini-2-5-flash",
        "GPT-5.2 (2025-12-11) (high reasoning)": "gpt-5",
        "GPT-5.2 (2025-12-11)": "gpt-5",
        "GPT-5.1-codex (medium reasoning)": "gpt-5-1-codex",
        "GPT-5.1 (2025-11-13) (medium reasoning)": "gpt-5-1",
        "GPT-5 (2025-08-07) (medium reasoning)": "gpt-5",
        "GPT-4.1 (2025-04-14)": "gpt-4-1",
        "GPT-4o (2024-11-20)": "gpt-4o",
        "DeepSeek V3.2 Reasoner": "deepseek-r1",
        "DeepSeek V3 Terminus": "deepseek-v3-terminus",
        "GLM-4.6 (T=1)": "glm-4-6",
        "GLM-4.5 (2025-08-22)": "glm-4-6"
      }
    },
    "terminalBench": {
      "websiteModels": {
        "Claude Opus 4.5": "claude-opus-4-1",
        "Claude Sonnet 4.5": "claude-sonnet-4-5",
        "Claude Opus 4.1": "claude-opus-4-1",
        "Claude Haiku 4.5": "claude-haiku-4-5",
        "Gemini 3 Pro": "gemini-3-pro",
        "Gemini 2.5 Pro": "gemini-2-5-pro",
        "Gemini 2.5 Flash": "gemini-2-5-flash",
        "GPT-5.2": "gpt-5",
        "GPT-5.1-Codex-Max": "gpt-5-1-codex",
        "GPT-5.1-Codex": "gpt-5-1-codex",
        "GPT-5.1-Codex-Mini": "gpt-5-1-codex",
        "GPT-5.1": "gpt-5-1",
        "GPT-5-Codex": "gpt-5-codex",
        "GPT-5": "gpt-5",
        "GPT-5-Mini": "gpt-5",
        "GPT-5-Nano": "gpt-5",
        "GPT-4o": "gpt-4o",
        "Grok Code Fast 1": "grok-code-fast-1",
        "Grok 4": "grok-4",
        "GLM 4.6": "glm-4-6"
      }
    },
    "mmmu": {
      "websiteModels": {
        "Claude-Sonnet-4.5": "claude-sonnet-4-5",
        "Claude Sonnet 4.5": "claude-sonnet-4-5",
        "Claude Opus 4.1": "claude-opus-4-1",
        "Gemini-2.5-Pro": "gemini-2-5-pro",
        "Gemini 2.5 Pro": "gemini-2-5-pro",
        "GPT-4o": "gpt-4o",
        "GPT-5": "gpt-5"
      }
    },
    "sciCode": {
      "websiteModels": {
        "Claude Sonnet 4.5": "claude-sonnet-4-5",
        "GPT-4o": "gpt-4o",
        "GPT-5": "gpt-5",
        "DeepSeek R1": "deepseek-r1",
        "DeepSeek V3": "deepseek-v3-terminus"
      }
    },
    "liveCodeBench": {
      "websiteModels": {
        "claude-sonnet-4-5": "claude-sonnet-4-5",
        "claude-sonnet-4.5": "claude-sonnet-4-5",
        "gpt-4o": "gpt-4o",
        "gpt-5": "gpt-5",
        "gemini-2.5-pro": "gemini-2-5-pro",
        "gemini-2-5-flash": "gemini-2-5-flash"
      }
    },
    "webDevArena": {
      "websiteModels": {
        "Claude Sonnet 4.5": "claude-sonnet-4-5",
        "GPT-4o": "gpt-4o",
        "GPT-5": "gpt-5"
      }
    }
  }
}

#!/usr/bin/env node

/**
 * Benchmark Fetcher - Main Entry Point
 *
 * Usage:
 *   node fetch-benchmarks.mjs
 *   node fetch-benchmarks.mjs --benchmarks swebench,terminalBench
 *   node fetch-benchmarks.mjs --models claude-sonnet-4-5,gpt-4o
 *   node fetch-benchmarks.mjs --dry-run
 */

import fs from 'node:fs/promises'
import path from 'node:path'
import { extractors } from './lib/benchmark-extractors.mjs'
import { BENCHMARKS, DEBUG_CONFIG, MANIFEST_PATHS, RETRY_CONFIG } from './lib/config.mjs'
import { updateManifests } from './lib/manifest-updater.mjs'
import { getUnmappedModels } from './lib/model-name-mapper.mjs'
import { generateReport } from './lib/report-generator.mjs'

// Parse command-line arguments
const args = parseArgs(process.argv.slice(2))

// Validate arguments
validateArgs(args)

// Main execution
const startTime = Date.now()

main()
  .then(() => {
    console.log('\n✅ Benchmark fetch completed successfully')
    process.exit(0)
  })
  .catch(error => {
    console.error('\n❌ Benchmark fetch failed:', error.message)
    console.error(error.stack)
    process.exit(1)
  })

/**
 * Main workflow
 */
async function main() {
  console.log('🤖 Benchmark Fetcher')
  console.log('='.repeat(80))

  // 1. Load configuration
  console.log('\n📋 Loading configuration...')
  const mappings = await loadMappings()
  const manifests = await loadAllManifests()

  console.log(`  ✓ Loaded ${Object.keys(manifests).length} model manifests`)
  console.log(`  ✓ Loaded mapping configuration (version ${mappings.version})`)

  if (args.dryRun) {
    console.log('\n⚠️  DRY RUN MODE - No manifests will be modified')
  }

  // Filter benchmarks if specified
  const benchmarksToFetch = args.benchmarks
    ? BENCHMARKS.filter(b => args.benchmarks.includes(b.id))
    : BENCHMARKS

  console.log(`\n📊 Will fetch ${benchmarksToFetch.length} benchmarks`)

  // 2. Initialize MCP tools (placeholder - actual MCP tools will be injected)
  const mcpTools = await initializeMCPTools()

  // 3. Process each benchmark sequentially
  console.log(`\n${'='.repeat(80)}`)
  const benchmarkResults = {}

  for (const benchmark of benchmarksToFetch) {
    console.log(`\n📊 Fetching ${benchmark.name} (${benchmark.id})`)
    console.log('-'.repeat(80))

    try {
      const result = await extractWithRetry(extractors[benchmark.id], mcpTools, mappings, benchmark)
      benchmarkResults[benchmark.id] = {
        success: true,
        ...result,
      }

      const dataCount = result.data instanceof Map ? result.data.size : 0
      console.log(`  ✅ Success! Extracted ${dataCount} model scores`)
    } catch (error) {
      console.error(`  ❌ Failed: ${error.message}`)
      benchmarkResults[benchmark.id] = {
        success: false,
        error: error.message,
        data: new Map(),
        unmappedModels: [],
      }
    }
  }

  // 4. Update manifests (unless dry-run)
  let manifestUpdates = []

  if (!args.dryRun) {
    console.log(`\n${'='.repeat(80)}`)
    console.log('💾 Updating Manifests')
    console.log('='.repeat(80))

    // Filter manifests if specific models were requested
    let manifestsToUpdate = manifests
    if (args.models) {
      manifestsToUpdate = {}
      for (const modelId of args.models) {
        if (manifests[modelId]) {
          manifestsToUpdate[modelId] = manifests[modelId]
        }
      }
    }

    manifestUpdates = await updateManifests(
      manifestsToUpdate,
      benchmarkResults,
      MANIFEST_PATHS.modelsDir
    )

    console.log(`\n  ✅ Updated ${manifestUpdates.filter(u => u.success).length} manifests`)
  }

  // 5. Generate completion report
  const unmappedModels = getUnmappedModels(benchmarkResults, mappings)
  const executionTime = Date.now() - startTime

  generateReport(benchmarkResults, manifestUpdates, unmappedModels, executionTime, args.dryRun)

  // 6. Cleanup (if needed)
  await cleanupMCPTools(mcpTools)
}

/**
 * Extract benchmark data with retry logic
 */
async function extractWithRetry(extractor, mcpTools, mappings, benchmark) {
  let lastError = null

  for (let attempt = 1; attempt <= RETRY_CONFIG.maxAttempts; attempt++) {
    try {
      const result = await extractor(mcpTools, mappings)
      return result
    } catch (error) {
      lastError = error
      console.error(`  ⚠️  Attempt ${attempt}/${RETRY_CONFIG.maxAttempts} failed: ${error.message}`)

      if (attempt < RETRY_CONFIG.maxAttempts) {
        // Calculate exponential backoff delay
        const delay = Math.min(
          RETRY_CONFIG.initialDelay * RETRY_CONFIG.backoffFactor ** (attempt - 1),
          RETRY_CONFIG.maxDelay
        )
        console.log(`  ⏳ Retrying in ${delay / 1000}s...`)
        await sleep(delay)
      } else {
        // Final attempt failed - take debug screenshot if enabled
        if (DEBUG_CONFIG.saveScreenshots && mcpTools.take_screenshot) {
          const screenshotPath = path.join(DEBUG_CONFIG.screenshotDir, `${benchmark.id}-error.png`)
          try {
            await fs.mkdir(DEBUG_CONFIG.screenshotDir, { recursive: true })
            await mcpTools.take_screenshot({ filePath: screenshotPath })
            console.log(`  📸 Debug screenshot saved: ${screenshotPath}`)
          } catch (_screenshotError) {
            // Ignore screenshot errors
          }
        }
      }
    }
  }

  throw lastError
}

/**
 * Load model name mappings from configuration file
 */
async function loadMappings() {
  const content = await fs.readFile(MANIFEST_PATHS.mappings, 'utf8')
  return JSON.parse(content)
}

/**
 * Load all model manifests from manifests/models/
 */
async function loadAllManifests() {
  const manifestFiles = await fs.readdir(MANIFEST_PATHS.modelsDir)
  const manifests = {}

  for (const file of manifestFiles) {
    if (!file.endsWith('.json')) {
      continue
    }

    const manifestPath = path.join(MANIFEST_PATHS.modelsDir, file)
    const content = await fs.readFile(manifestPath, 'utf8')
    const manifest = JSON.parse(content)

    if (manifest.id) {
      manifests[manifest.id] = manifest
    }
  }

  return manifests
}

/**
 * Initialize MCP Chrome DevTools tools
 * NOTE: This is a placeholder. In actual execution, MCP tools will be available
 * through the Claude Code environment automatically.
 */
async function initializeMCPTools() {
  // In the Claude Code environment, MCP tools are available globally
  // This function is a placeholder for any initialization logic needed

  // Check if MCP tools are available
  if (typeof mcp__chrome_devtools__navigate_page === 'undefined') {
    console.warn('⚠️  Warning: MCP Chrome DevTools tools not detected')
    console.warn('   This script requires Chrome DevTools MCP to be enabled')
    console.warn('   Extractors will fail if MCP tools are not available')
  }

  return {
    navigate_page: mcp__chrome_devtools__navigate_page,
    wait_for: mcp__chrome_devtools__wait_for,
    take_snapshot: mcp__chrome_devtools__take_snapshot,
    take_screenshot: mcp__chrome_devtools__take_screenshot,
  }
}

/**
 * Cleanup MCP tools
 */
async function cleanupMCPTools(_mcpTools) {
  // No cleanup needed currently
  // MCP tools are managed by Claude Code environment
}

/**
 * Parse command-line arguments
 */
function parseArgs(argv) {
  const args = {
    benchmarks: null, // Array of benchmark IDs or null for all
    models: null, // Array of model IDs or null for all
    dryRun: false,
  }

  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i]

    if (arg === '--benchmarks' && i + 1 < argv.length) {
      args.benchmarks = argv[i + 1].split(',').map(s => s.trim())
      i++
    } else if (arg === '--models' && i + 1 < argv.length) {
      args.models = argv[i + 1].split(',').map(s => s.trim())
      i++
    } else if (arg === '--dry-run') {
      args.dryRun = true
    }
  }

  return args
}

/**
 * Validate command-line arguments
 */
function validateArgs(args) {
  if (args.benchmarks) {
    const validBenchmarkIds = BENCHMARKS.map(b => b.id)
    for (const benchmarkId of args.benchmarks) {
      if (!validBenchmarkIds.includes(benchmarkId)) {
        console.error(`❌ Invalid benchmark ID: ${benchmarkId}`)
        console.error(`   Valid IDs: ${validBenchmarkIds.join(', ')}`)
        process.exit(1)
      }
    }
  }
}

/**
 * Sleep utility
 */
function sleep(ms) {
  return new Promise(resolve => setTimeout(resolve, ms))
}

/**
 * Benchmark extractors for each leaderboard website
 * Each extractor navigates to the website, extracts benchmark data, and maps to manifest IDs
 */

import { mapModelName, trackUnmapped } from './model-name-mapper.mjs'

/**
 * Extract SWE-bench Verified scores
 * Website: https://www.swebench.com
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Map<string, number>>} Map of manifest IDs to scores
 */
async function extractSWEBench(mcpTools, mappings) {
  const url = 'https://www.swebench.com'
  console.log(`  🌐 Navigating to ${url}`)

  // Navigate to page
  await mcpTools.navigate_page({ url, type: 'url' })

  // Wait for leaderboard to load
  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Model', timeout: 10000 })

  // Take snapshot to analyze structure
  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  // Parse leaderboard data from snapshot
  console.log('  🔍 Parsing leaderboard data...')
  const benchmarkData = new Map()
  const unmapped = new Set()

  // Parse the snapshot content
  const lines = snapshot.content.split('\n')

  // Look for pattern: model name followed by % Resolved score
  // Pattern: checkbox "Select <ModelName>" ... StaticText "<Score>"
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]

    // Find checkbox lines with "Select" - these indicate model rows
    if (line.includes('checkbox "Select ') && !line.includes('Select all models')) {
      // Extract model name from checkbox label
      const selectMatch = line.match(/checkbox "Select (.+?)"/)
      if (selectMatch) {
        const modelName = selectMatch[1].trim()

        // Look ahead for the % Resolved score
        // The score appears a few lines after the model name
        for (let j = i + 1; j < Math.min(i + 15, lines.length); j++) {
          const scoreLine = lines[j]

          // Look for StaticText with a number pattern (e.g., "74.40", "21.62")
          const scoreMatch = scoreLine.match(/StaticText "(\d+\.\d+)"/)
          if (scoreMatch && !scoreLine.includes('StaticText "$')) {
            const score = parseFloat(scoreMatch[1])

            // Map model name to manifest ID
            const manifestId = mapModelName('swebench', modelName, mappings)

            if (manifestId) {
              benchmarkData.set(manifestId, score)
              console.log(`    ✓ Mapped: "${modelName}" → ${manifestId} (${score}%)`)
            } else {
              trackUnmapped(unmapped, modelName)
            }

            break // Found score for this model, move to next model
          }
        }
      }
    }
  }

  console.log(`  📊 Extracted ${benchmarkData.size} models, ${unmapped.size} unmapped`)

  return {
    data: benchmarkData,
    unmappedModels: Array.from(unmapped),
  }
}

/**
 * Extract TerminalBench 2.0 accuracy scores
 * Website: https://www.tbench.ai/leaderboard/terminal-bench/2.0
 * CRITICAL: Stores as decimal (0-1 scale), not percentage
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Map<string, number>>} Map of manifest IDs to scores (decimal format)
 */
async function extractTerminalBench(mcpTools, mappings) {
  const url = 'https://www.tbench.ai/leaderboard/terminal-bench/2.0'
  console.log(`  🌐 Navigating to ${url}`)

  await mcpTools.navigate_page({ url, type: 'url' })

  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Accuracy', timeout: 10000 })

  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  console.log('  🔍 Parsing leaderboard data...')
  const benchmarkData = new Map()
  const modelScores = new Map() // Track all scores per model to find max
  const unmapped = new Set()

  // Parse the snapshot content
  const lines = snapshot.content.split('\n')

  // Look for pattern: Rank → Agent → Model → Date → ... → Accuracy
  // After "Model" column, we find the model name, then accuracy appears later
  let currentModel = null

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i].trim()

    // Skip header rows and empty lines
    if (!line || line.includes('StaticText "Rank"') || line.includes('StaticText "Model"')) {
      continue
    }

    // Look for model names (they appear after rank numbers and agent names)
    // Pattern: StaticText "<Number>" (rank) → StaticText "<Agent>" → StaticText "<Model>"
    // Then much later: StaticText "<Accuracy>" StaticText "%"

    // Detect model name - it comes after agent name
    // We can identify it by looking for patterns that match model names
    const modelMatch = line.match(/StaticText "([^"]+)"/)
    if (modelMatch) {
      const text = modelMatch[1]

      // Check if this looks like a model name (contains key words or patterns)
      if (
        text.includes('Claude') ||
        text.includes('GPT') ||
        text.includes('Gemini') ||
        text.includes('Opus') ||
        text.includes('Sonnet') ||
        text.includes('Haiku') ||
        text.includes('Codex') ||
        text.includes('Kimi') ||
        text.includes('MiniMax') ||
        text.includes('Qwen') ||
        text.includes('GLM') ||
        text.includes('Grok') ||
        text.includes('Multiple')
      ) {
        currentModel = text
        continue
      }
    }

    // Look for accuracy percentage
    // Pattern: StaticText "<Number>.<Number>" followed by StaticText "%"
    const accuracyMatch = line.match(/StaticText "(\d+\.\d+)"/)
    if (accuracyMatch && currentModel && i + 1 < lines.length) {
      const nextLine = lines[i + 1]
      if (nextLine.includes('StaticText "%"')) {
        const percentage = parseFloat(accuracyMatch[1])
        const decimalScore = percentage / 100 // Convert to decimal (0-1)

        // Store the score for this model
        if (!modelScores.has(currentModel)) {
          modelScores.set(currentModel, [])
        }
        modelScores.get(currentModel).push(decimalScore)

        currentModel = null // Reset for next row
      }
    }
  }

  // For each model, take the highest score and map to manifest ID
  for (const [modelName, scores] of modelScores.entries()) {
    const maxScore = Math.max(...scores)
    const manifestId = mapModelName('terminalBench', modelName, mappings)

    if (manifestId) {
      benchmarkData.set(manifestId, maxScore)
      console.log(`    ✓ Mapped: "${modelName}" → ${manifestId} (${maxScore.toFixed(3)})`)
    } else {
      trackUnmapped(unmapped, modelName)
    }
  }

  console.log(`  📊 Extracted ${benchmarkData.size} models, ${unmapped.size} unmapped`)
  console.log(`  ⚠️  Note: Values stored in decimal format (0-1 scale)`)

  return {
    data: benchmarkData,
    unmappedModels: Array.from(unmapped),
  }
}

/**
 * Extract MMMU and MMMU Pro benchmark scores
 * Website: https://mmmu-benchmark.github.io/#leaderboard
 * Special: Returns both MMMU and MMMU Pro from single website
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Object>} Object with mmmu and mmmuPro Maps
 */
async function extractMMMU(mcpTools, mappings) {
  const url = 'https://mmmu-benchmark.github.io/#leaderboard'
  console.log(`  🌐 Navigating to ${url}`)

  await mcpTools.navigate_page({ url, type: 'url' })

  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Leaderboard', timeout: 10000 })

  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  console.log('  🔍 Parsing leaderboard data...')
  const mmmuData = new Map()
  const mmmuProData = new Map()
  const unmapped = new Set()

  // Parse snapshot - look for leaderboard tables
  const lines = snapshot.content.split('\n')

  // MMMU typically has sections for standard and Pro versions
  // Look for model names and scores in percentage format
  let inMmmuSection = false
  let inMmmuProSection = false

  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]

    // Detect section headers
    if (line.includes('MMMU Pro') || line.includes('mmmu-pro')) {
      inMmmuProSection = true
      inMmmuSection = false
      continue
    } else if (line.includes('MMMU') && !line.includes('Pro')) {
      inMmmuSection = true
      inMmmuProSection = false
      continue
    }

    // Look for model entries with scores
    const textMatch = line.match(/StaticText "([^"]+)"/)
    if (textMatch) {
      const text = textMatch[1]

      // Check if this is a model name
      if (text.includes('Claude') || text.includes('GPT') || text.includes('Gemini')) {
        // Look ahead for score
        for (let j = i + 1; j < Math.min(i + 10, lines.length); j++) {
          const scoreMatch = lines[j].match(/StaticText "(\d+\.?\d*)"/)
          if (scoreMatch) {
            const score = parseFloat(scoreMatch[1])

            const manifestId = mapModelName('mmmu', text, mappings)

            if (manifestId) {
              if (inMmmuProSection) {
                mmmuProData.set(manifestId, score)
                console.log(`    ✓ MMMU Pro: "${text}" → ${manifestId} (${score}%)`)
              } else if (inMmmuSection) {
                mmmuData.set(manifestId, score)
                console.log(`    ✓ MMMU: "${text}" → ${manifestId} (${score}%)`)
              }
            } else {
              trackUnmapped(unmapped, text)
            }

            break
          }
        }
      }
    }
  }

  console.log(
    `  📊 Extracted MMMU: ${mmmuData.size}, MMMU Pro: ${mmmuProData.size}, unmapped: ${unmapped.size}`
  )

  return {
    data: {
      mmmu: mmmuData,
      mmmuPro: mmmuProData,
    },
    unmappedModels: Array.from(unmapped),
  }
}

/**
 * Extract SciCode benchmark scores
 * Website: https://scicode-bench.github.io/leaderboard/
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Map<string, number>>} Map of manifest IDs to scores
 */
async function extractSciCode(mcpTools, mappings) {
  const url = 'https://scicode-bench.github.io/leaderboard/'
  console.log(`  🌐 Navigating to ${url}`)

  await mcpTools.navigate_page({ url, type: 'url' })

  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Model', timeout: 10000 })

  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  console.log('  🔍 Parsing leaderboard data...')
  const benchmarkData = new Map()
  const unmapped = new Set()

  // Generic extraction pattern - look for model names and nearby scores
  return extractGenericLeaderboard(snapshot, mappings, 'sciCode', benchmarkData, unmapped)
}

/**
 * Extract LiveCodeBench Pass@1 scores
 * Website: https://livecodebench.github.io/leaderboard.html
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Map<string, number>>} Map of manifest IDs to scores
 */
async function extractLiveCodeBench(mcpTools, mappings) {
  const url = 'https://livecodebench.github.io/leaderboard.html'
  console.log(`  🌐 Navigating to ${url}`)

  await mcpTools.navigate_page({ url, type: 'url' })

  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Pass@1', timeout: 10000 })

  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  console.log('  🔍 Parsing leaderboard data...')
  const benchmarkData = new Map()
  const unmapped = new Set()

  // Generic extraction pattern
  return extractGenericLeaderboard(snapshot, mappings, 'liveCodeBench', benchmarkData, unmapped)
}

/**
 * Extract WebDevArena scores
 * Website: https://web.lmarena.ai/leaderboard
 *
 * @param {Object} mcpTools - MCP Chrome DevTools tools
 * @param {Object} mappings - Model name mappings configuration
 * @returns {Promise<Map<string, number>>} Map of manifest IDs to scores
 */
async function extractWebDevArena(mcpTools, mappings) {
  const url = 'https://web.lmarena.ai/leaderboard'
  console.log(`  🌐 Navigating to ${url}`)

  await mcpTools.navigate_page({ url, type: 'url' })

  console.log('  ⏳ Waiting for leaderboard...')
  await mcpTools.wait_for({ text: 'Leaderboard', timeout: 10000 })

  console.log('  📸 Taking snapshot...')
  const snapshot = await mcpTools.take_snapshot()

  console.log('  🔍 Parsing leaderboard data...')
  const benchmarkData = new Map()
  const unmapped = new Set()

  // Generic extraction pattern
  return extractGenericLeaderboard(snapshot, mappings, 'webDevArena', benchmarkData, unmapped)
}

/**
 * Generic leaderboard extraction helper
 * Looks for model names and nearby numeric scores
 *
 * @param {Object} snapshot - Page snapshot
 * @param {Object} mappings - Model name mappings
 * @param {string} benchmarkId - Benchmark identifier
 * @param {Map} benchmarkData - Map to populate with data
 * @param {Set} unmapped - Set to track unmapped models
 * @returns {Object} Result object with data and unmapped models
 */
function extractGenericLeaderboard(snapshot, mappings, benchmarkId, benchmarkData, unmapped) {
  const lines = snapshot.content.split('\n')

  // Look for model names followed by scores
  for (let i = 0; i < lines.length; i++) {
    const line = lines[i]
    const textMatch = line.match(/StaticText "([^"]+)"/)

    if (textMatch) {
      const text = textMatch[1]

      // Check if this looks like a model name
      if (
        text.includes('Claude') ||
        text.includes('GPT') ||
        text.includes('Gemini') ||
        text.includes('DeepSeek') ||
        text.includes('Grok') ||
        text.includes('claude') ||
        text.includes('gpt') ||
        text.includes('gemini')
      ) {
        // Look ahead for a numeric score
        for (let j = i + 1; j < Math.min(i + 15, lines.length); j++) {
          const scoreMatch = lines[j].match(/StaticText "(\d+\.?\d*)"/)
          if (scoreMatch && !lines[j].includes('StaticText "$')) {
            const score = parseFloat(scoreMatch[1])

            // Only process reasonable scores (0-100 range)
            if (score >= 0 && score <= 100) {
              const manifestId = mapModelName(benchmarkId, text, mappings)

              if (manifestId) {
                benchmarkData.set(manifestId, score)
                console.log(`    ✓ Mapped: "${text}" → ${manifestId} (${score}%)`)
              } else {
                trackUnmapped(unmapped, text)
              }

              break
            }
          }
        }
      }
    }
  }

  console.log(`  📊 Extracted ${benchmarkData.size} models, ${unmapped.size} unmapped`)

  return {
    data: benchmarkData,
    unmappedModels: Array.from(unmapped),
  }
}

/**
 * Extractor registry
 * Maps benchmark IDs to their extractor functions
 */
export const extractors = {
  swebench: extractSWEBench,
  terminalBench: extractTerminalBench,
  mmmu: extractMMMU,
  sciCode: extractSciCode,
  liveCodeBench: extractLiveCodeBench,
  webDevArena: extractWebDevArena,
}

/**
 * Configuration constants for benchmark-fetcher skill
 */

import path from 'node:path'
import { fileURLToPath } from 'node:url'

const __filename = fileURLToPath(import.meta.url)
const __dirname = path.dirname(__filename)

// Project root (5 levels up from .claude/skills/benchmark-fetcher/scripts/lib/)
export const PROJECT_ROOT = path.resolve(__dirname, '../../../../..')

/**
 * Benchmark configurations
 * Each benchmark defines its website, target field in manifests, and data format
 */
export const BENCHMARKS = [
  {
    id: 'swebench',
    name: 'SWE-bench',
    url: 'https://www.swebench.com',
    field: 'sweBench',
    format: 'percentage',
    description: 'SWE-bench Verified score',
  },
  {
    id: 'terminalBench',
    name: 'TerminalBench',
    url: 'https://www.tbench.ai/leaderboard/terminal-bench/2.0',
    field: 'terminalBench',
    format: 'decimal', // CRITICAL: 0-1 scale, not percentage
    description: 'TerminalBench 2.0 accuracy score',
  },
  {
    id: 'mmmu',
    name: 'MMMU',
    url: 'https://mmmu-benchmark.github.io/#leaderboard',
    fields: ['mmmu', 'mmmuPro'], // Special case: two fields from one website
    format: 'percentage',
    description: 'MMMU and MMMU Pro benchmark scores',
  },
  {
    id: 'sciCode',
    name: 'SciCode',
    url: 'https://scicode-bench.github.io/leaderboard/',
    field: 'sciCode',
    format: 'percentage',
    description: 'SciCode benchmark score',
  },
  {
    id: 'liveCodeBench',
    name: 'LiveCodeBench',
    url: 'https://livecodebench.github.io/leaderboard.html',
    field: 'liveCodeBench',
    format: 'percentage',
    description: 'LiveCodeBench Pass@1 score',
  },
  {
    id: 'webDevArena',
    name: 'WebDevArena',
    url: 'https://web.lmarena.ai/leaderboard',
    field: 'webDevArena',
    format: 'percentage',
    description: 'WebDevArena score',
  },
]

/**
 * Retry configuration for benchmark extraction
 */
export const RETRY_CONFIG = {
  maxAttempts: 3,
  initialDelay: 1000, // 1 second
  maxDelay: 10000, // 10 seconds
  backoffFactor: 2, // Exponential backoff multiplier
}

/**
 * Timeout configuration for browser operations
 */
export const TIMEOUT_CONFIG = {
  navigation: 30000, // 30 seconds for page navigation
  waitFor: 10000, // 10 seconds for element to appear
  snapshot: 5000, // 5 seconds for taking snapshot
}

/**
 * File paths for manifests and configuration
 */
export const MANIFEST_PATHS = {
  modelsDir: path.join(PROJECT_ROOT, 'manifests/models'),
  schema: path.join(PROJECT_ROOT, 'manifests/$schemas/model.schema.json'),
  mappings: path.join(
    PROJECT_ROOT,
    '.claude/skills/benchmark-fetcher/references/model-name-mappings.json'
  ),
}

/**
 * Debug configuration
 */
export const DEBUG_CONFIG = {
  saveScreenshots: true,
  screenshotDir: '/tmp/benchmark-fetcher-debug',
  saveSnapshots: true,
  snapshotDir: '/tmp/benchmark-fetcher-snapshots',
}

/**
 * Benchmark field mapping for manifest updates
 * Maps benchmark IDs to manifest field names
 */
export const BENCHMARK_FIELD_MAP = {
  swebench: 'sweBench',
  terminalBench: 'terminalBench',
  sciCode: 'sciCode',
  liveCodeBench: 'liveCodeBench',
  webDevArena: 'webDevArena',
  // mmmu is handled specially (returns both mmmu and mmmuPro)
}

/**
 * Benchmark source URLs for reporting
 */
export const BENCHMARK_SOURCES = {
  swebench: 'swebench.com',
  terminalBench: 'tbench.ai',
  mmmu: 'mmmu-benchmark.github.io',
  sciCode: 'scicode-bench.github.io',
  liveCodeBench: 'livecodebench.github.io',
  webDevArena: 'web.lmarena.ai',
}

/**
 * Manifest updater utilities
 * Updates model manifests with latest benchmark data using atomic file writes
 */

import fs from 'node:fs/promises'
import path from 'node:path'
import { BENCHMARK_FIELD_MAP, BENCHMARK_SOURCES } from './config.mjs'

/**
 * Update all model manifests with benchmark results
 * Always overwrites existing benchmark values with latest data
 *
 * @param {Object} manifests - Map of manifest IDs to manifest objects
 * @param {Object} benchmarkResults - Results from all benchmark extractions
 * @param {string} manifestsDir - Directory containing manifest files
 * @returns {Promise<Array>} Array of update results
 */
export async function updateManifests(manifests, benchmarkResults, manifestsDir) {
  const updates = []

  for (const [manifestId, manifest] of Object.entries(manifests)) {
    const changes = collectChanges(manifestId, manifest, benchmarkResults)

    if (changes.length === 0) {
      continue // No updates for this model
    }

    // Apply changes to manifest object
    for (const change of changes) {
      if (!manifest.benchmarks) {
        manifest.benchmarks = {}
      }
      manifest.benchmarks[change.field] = change.newValue
    }

    // Write manifest to file
    const manifestPath = path.join(manifestsDir, `${manifestId}.json`)
    try {
      await writeManifestSafely(manifestPath, manifest)
      updates.push({
        manifestId,
        changes,
        success: true,
      })
    } catch (error) {
      updates.push({
        manifestId,
        changes,
        success: false,
        error: error.message,
      })
    }
  }

  return updates
}

/**
 * Collect all benchmark changes for a single model
 *
 * @param {string} manifestId - Manifest identifier
 * @param {Object} manifest - Manifest object
 * @param {Object} benchmarkResults - Results from all benchmark extractions
 * @returns {Array} Array of change objects
 */
function collectChanges(manifestId, manifest, benchmarkResults) {
  const changes = []

  for (const [benchmarkId, result] of Object.entries(benchmarkResults)) {
    if (!result.success || !result.data) {
      continue // Skip failed extractions
    }

    // Handle MMMU special case (two fields from one extraction)
    if (benchmarkId === 'mmmu') {
      // Process mmmu field
      if (result.data.mmmu?.has(manifestId)) {
        const newValue = result.data.mmmu.get(manifestId)
        const oldValue = manifest.benchmarks?.mmmu ?? null

        if (newValue !== oldValue) {
          changes.push({
            field: 'mmmu',
            oldValue,
            newValue,
            source: BENCHMARK_SOURCES.mmmu,
          })
        }
      }

      // Process mmmuPro field
      if (result.data.mmmuPro?.has(manifestId)) {
        const newValue = result.data.mmmuPro.get(manifestId)
        const oldValue = manifest.benchmarks?.mmmuPro ?? null

        if (newValue !== oldValue) {
          changes.push({
            field: 'mmmuPro',
            oldValue,
            newValue,
            source: BENCHMARK_SOURCES.mmmu,
          })
        }
      }
    } else {
      // Standard benchmark (single value)
      const fieldName = BENCHMARK_FIELD_MAP[benchmarkId]

      if (!fieldName) {
        console.warn(`⚠️  No field mapping for benchmark: ${benchmarkId}`)
        continue
      }

      if (result.data.has(manifestId)) {
        const newValue = result.data.get(manifestId)
        const oldValue = manifest.benchmarks?.[fieldName] ?? null

        if (newValue !== oldValue) {
          changes.push({
            field: fieldName,
            oldValue,
            newValue,
            source: BENCHMARK_SOURCES[benchmarkId],
          })
        }
      }
    }
  }

  return changes
}

/**
 * Write manifest to file safely using atomic writes
 * Validates JSON structure, writes to temp file, then renames atomically
 *
 * @param {string} manifestPath - Path to manifest file
 * @param {Object} manifest - Manifest object to write
 * @returns {Promise<void>}
 */
async function writeManifestSafely(manifestPath, manifest) {
  // 1. Validate JSON structure
  validateManifestStructure(manifest)

  // 2. Convert to JSON with 2-space indentation
  const json = JSON.stringify(manifest, null, 2)

  // 3. Write to temporary file
  const tempPath = `${manifestPath}.tmp`
  await fs.writeFile(tempPath, `${json}\n`, 'utf8')

  // 4. Atomic rename
  await fs.rename(tempPath, manifestPath)
}

/**
 * Validate manifest structure before writing
 * Ensures required fields exist
 *
 * @param {Object} manifest - Manifest object to validate
 * @throws {Error} If manifest structure is invalid
 */
function validateManifestStructure(manifest) {
  if (!manifest.id) {
    throw new Error('Manifest missing required field: id')
  }

  if (!manifest.name) {
    throw new Error('Manifest missing required field: name')
  }

  if (manifest.benchmarks && typeof manifest.benchmarks !== 'object') {
    throw new Error('Manifest benchmarks field must be an object')
  }

  // Ensure benchmarks is an object, not an array
  if (Array.isArray(manifest.benchmarks)) {
    throw new Error('Manifest benchmarks field must be an object, not an array')
  }
}

/**
 * Model name mapping utilities
 * Maps website-specific model names to manifest IDs using configurable mappings
 */

/**
 * Normalize model name for fuzzy matching
 * Removes spaces, hyphens, and special characters, converts to lowercase
 *
 * @param {string} name - Model name to normalize
 * @returns {string} Normalized model name
 */
function normalizeModelName(name) {
  return name
    .trim()
    .toLowerCase()
    .replace(/\s+/g, '') // Remove spaces
    .replace(/-/g, '') // Remove hyphens
    .replace(/[^a-z0-9]/g, '') // Remove special characters
}

/**
 * Map website model name to manifest ID using 3-tier fallback strategy
 *
 * Strategy:
 * 1. Exact match (case-sensitive)
 * 2. Case-insensitive match
 * 3. Fuzzy match (normalized)
 *
 * @param {string} benchmarkId - Benchmark identifier (e.g., 'swebench')
 * @param {string} websiteName - Model name as shown on website
 * @param {Object} mappings - Mapping configuration object
 * @returns {string|null} Manifest ID or null if not found
 */
export function mapModelName(benchmarkId, websiteName, mappings) {
  const benchmarkMappings = mappings.mappings[benchmarkId]

  if (!benchmarkMappings) {
    console.warn(`⚠️  No mappings configured for benchmark: ${benchmarkId}`)
    return null
  }

  const websiteModels = benchmarkMappings.websiteModels

  // Strategy 1: Exact match (case-sensitive)
  if (websiteModels[websiteName]) {
    return websiteModels[websiteName]
  }

  // Strategy 2: Case-insensitive match
  const lowerName = websiteName.toLowerCase()
  for (const [key, value] of Object.entries(websiteModels)) {
    if (key.toLowerCase() === lowerName) {
      return value
    }
  }

  // Strategy 3: Fuzzy match (normalized)
  const normalizedName = normalizeModelName(websiteName)
  for (const [key, value] of Object.entries(websiteModels)) {
    if (normalizeModelName(key) === normalizedName) {
      return value
    }
  }

  // Not found - log for manual mapping
  console.warn(`⚠️  Unmapped model on ${benchmarkId}: "${websiteName}"`)
  return null
}

/**
 * Get all unmapped models from extraction results
 *
 * @param {Object} benchmarkResults - Results from all benchmark extractions
 * @param {Object} mappings - Mapping configuration object
 * @returns {Array} Array of unmapped models grouped by benchmark
 */
export function getUnmappedModels(benchmarkResults, _mappings) {
  const unmapped = []

  for (const [benchmarkId, result] of Object.entries(benchmarkResults)) {
    if (!result.success || !result.unmappedModels) {
      continue
    }

    if (result.unmappedModels.length > 0) {
      unmapped.push({
        benchmarkId,
        benchmarkName: getBenchmarkName(benchmarkId),
        models: result.unmappedModels,
      })
    }
  }

  return unmapped
}

/**
 * Get benchmark display name from ID
 *
 * @param {string} benchmarkId - Benchmark identifier
 * @returns {string} Benchmark display name
 */
function getBenchmarkName(benchmarkId) {
  const names = {
    swebench: 'SWE-bench',
    terminalBench: 'TerminalBench',
    mmmu: 'MMMU',
    sciCode: 'SciCode',
    liveCodeBench: 'LiveCodeBench',
    webDevArena: 'WebDevArena',
  }
  return names[benchmarkId] || benchmarkId
}

/**
 * Track unmapped models during extraction
 * Call this when a model name cannot be mapped
 *
 * @param {Set} unmappedSet - Set to track unmapped model names
 * @param {string} websiteName - Model name that couldn't be mapped
 */
export function trackUnmapped(unmappedSet, websiteName) {
  unmappedSet.add(websiteName)
}

/**
 * Report generator for benchmark fetch completion
 * Generates formatted console output showing results, updates, and next steps
 */

import { BENCHMARK_SOURCES, BENCHMARKS } from './config.mjs'

/**
 * Generate and print completion report
 *
 * @param {Object} benchmarkResults - Results from all benchmark extractions
 * @param {Array} manifestUpdates - Array of manifest update results
 * @param {Array} unmappedModels - Array of unmapped models by benchmark
 * @param {number} executionTime - Total execution time in milliseconds
 * @param {boolean} isDryRun - Whether this was a dry run
 */
export function generateReport(
  benchmarkResults,
  manifestUpdates,
  unmappedModels,
  executionTime,
  isDryRun = false
) {
  console.log(`\n${'='.repeat(80)}`)
  console.log('📊 Benchmark Fetch Report')
  console.log('='.repeat(80))

  if (isDryRun) {
    console.log('\n⚠️  DRY RUN MODE - No manifests were modified\n')
  }

  // Summary section
  printSummary(benchmarkResults, manifestUpdates)

  // Benchmark results
  printBenchmarkResults(benchmarkResults)

  // Manifest updates
  if (!isDryRun && manifestUpdates.length > 0) {
    printManifestUpdates(manifestUpdates)
  }

  // Unmapped models
  if (unmappedModels.length > 0) {
    printUnmappedModels(unmappedModels)
  }

  // Statistics
  printStatistics(benchmarkResults, manifestUpdates, executionTime)

  // Next steps
  printNextSteps(isDryRun, unmappedModels, benchmarkResults)

  console.log(`${'='.repeat(80)}\n`)
}

/**
 * Print summary section
 */
function printSummary(benchmarkResults, manifestUpdates) {
  const totalBenchmarks = Object.keys(benchmarkResults).length
  const successful = Object.values(benchmarkResults).filter(r => r.success).length
  const failed = totalBenchmarks - successful
  const manifestsUpdated = manifestUpdates.filter(u => u.success).length

  console.log('\n📈 Summary')
  console.log('-'.repeat(80))
  console.log(`  Benchmarks fetched:  ${successful}/${totalBenchmarks}`)
  console.log(`  Manifests updated:   ${manifestsUpdated}`)
  console.log(`  Failed extractions:  ${failed}`)
}

/**
 * Print benchmark results
 */
function printBenchmarkResults(benchmarkResults) {
  console.log('\n📊 Benchmark Results')
  console.log('-'.repeat(80))

  const successful = []
  const failed = []

  for (const [benchmarkId, result] of Object.entries(benchmarkResults)) {
    const benchmark = BENCHMARKS.find(b => b.id === benchmarkId)
    const name = benchmark ? benchmark.name : benchmarkId

    if (result.success) {
      const dataCount = result.data instanceof Map ? result.data.size : 0
      successful.push({ name, source: BENCHMARK_SOURCES[benchmarkId], dataCount })
    } else {
      failed.push({
        name,
        source: BENCHMARK_SOURCES[benchmarkId],
        error: result.error,
      })
    }
  }

  if (successful.length > 0) {
    console.log('\n✅ Successfully Fetched:')
    for (const { name, source, dataCount } of successful) {
      console.log(`   ✓ ${name} (${source}) - ${dataCount} models`)
    }
  }

  if (failed.length > 0) {
    console.log('\n❌ Failed to Fetch:')
    for (const { name, source, error } of failed) {
      console.log(`   ✗ ${name} (${source})`)
      console.log(`     Reason: ${error}`)
    }
  }
}

/**
 * Print manifest updates
 */
function printManifestUpdates(manifestUpdates) {
  console.log('\n📝 Manifest Updates')
  console.log('-'.repeat(80))

  const successful = manifestUpdates.filter(u => u.success)
  const failed = manifestUpdates.filter(u => !u.success)

  if (successful.length > 0) {
    console.log(`\n✅ Updated: ${successful.length} manifests\n`)

    for (const update of successful) {
      if (update.changes.length > 0) {
        console.log(`   • ${update.manifestId}: ${update.changes.length} benchmarks updated`)

        for (const change of update.changes) {
          const oldDisplay = formatValue(change.oldValue)
          const newDisplay = formatValue(change.newValue)
          console.log(`     - ${change.field}: ${oldDisplay} → ${newDisplay}`)
        }
        console.log('')
      }
    }
  }

  if (failed.length > 0) {
    console.log(`\n❌ Failed Updates: ${failed.length} manifests\n`)

    for (const update of failed) {
      console.log(`   • ${update.manifestId}`)
      console.log(`     Error: ${update.error}`)
    }
  }
}

/**
 * Print unmapped models
 */
function printUnmappedModels(unmappedModels) {
  console.log('\n⚠️  Unmapped Models')
  console.log('-'.repeat(80))
  console.log('\nThe following models were found on leaderboards but could not be')
  console.log('mapped to manifest IDs. Add them to model-name-mappings.json:\n')

  for (const { benchmarkName, models } of unmappedModels) {
    if (models.length > 0) {
      console.log(`${benchmarkName}:`)
      for (const modelName of models) {
        console.log(`  • "${modelName}"`)
      }
      console.log('')
    }
  }

  console.log('Suggestion: Update references/model-name-mappings.json with these entries')
}

/**
 * Print statistics
 */
function printStatistics(benchmarkResults, manifestUpdates, executionTime) {
  console.log('\n📈 Statistics')
  console.log('-'.repeat(80))

  // Count total benchmark values fetched
  let totalValues = 0
  for (const result of Object.values(benchmarkResults)) {
    if (result.success && result.data instanceof Map) {
      totalValues += result.data.size
    }
  }

  // Count total changes
  let totalChanges = 0
  for (const update of manifestUpdates) {
    if (update.success) {
      totalChanges += update.changes.length
    }
  }

  const executionSeconds = (executionTime / 1000).toFixed(1)
  const avgTimePerBenchmark = (executionTime / 1000 / Object.keys(benchmarkResults).length).toFixed(
    1
  )

  console.log(`  Total benchmark values fetched:  ${totalValues}`)
  console.log(`  Total benchmark updates:         ${totalChanges}`)
  console.log(`  Manifests updated:               ${manifestUpdates.length}`)
  console.log(`  Execution time:                  ${executionSeconds}s`)
  console.log(`  Average time per benchmark:      ${avgTimePerBenchmark}s`)
}

/**
 * Print next steps
 */
function printNextSteps(isDryRun, unmappedModels, benchmarkResults) {
  console.log('\n✅ Next Steps')
  console.log('-'.repeat(80))

  const steps = []

  if (isDryRun) {
    steps.push('Run without --dry-run to apply updates to manifests')
  } else {
    steps.push('Review updated manifests in manifests/models/')
  }

  if (unmappedModels.length > 0) {
    steps.push('Add unmapped models to references/model-name-mappings.json')
  }

  const failedBenchmarks = Object.entries(benchmarkResults)
    .filter(([_, result]) => !result.success)
    .map(([id, _]) => id)

  if (failedBenchmarks.length > 0) {
    steps.push(`Retry failed benchmarks: --benchmarks ${failedBenchmarks.join(',')}`)
  }

  if (!isDryRun) {
    steps.push('Run validation: npm run test:validate')
    steps.push('Commit changes when satisfied')
  }

  for (let i = 0; i < steps.length; i++) {
    console.log(`  ${i + 1}. ${steps[i]}`)
  }
}

/**
 * Format value for display
 */
function formatValue(value) {
  if (value === null || value === undefined) {
    return 'null'
  }
  return String(value)
}