Nightly Eval Investigation

import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const BUCKET_NAME = 'guidance-evals';

// Resolve local directories
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, '../../../..');
const resultsDir = path.join(repoRoot, 'harness', 'results');

function getSlug(taskName: string): string {
  return taskName.toLowerCase().replace(/[^a-z0-9_-]/g, '').replace(/\s+/g, '-');
}

function findGuideDirs(dir: string, result: Record<string, string> = {}): Record<string, string> {
  if (!fs.existsSync(dir)) return result;
  const items = fs.readdirSync(dir, { withFileTypes: true });
  for (const item of items) {
    if (item.isDirectory() && item.name !== 'node_modules') {
      const fullPath = path.join(dir, item.name);
      if (fs.existsSync(path.join(fullPath, 'guide.md'))) {
        result[item.name] = fullPath;
      } else {
        findGuideDirs(fullPath, result);
      }
    }
  }
  return result;
}

function extractTaskPrompt(taskPath: string): string {
  if (!fs.existsSync(taskPath)) return '';
  const content = fs.readFileSync(taskPath, 'utf8');
  const parts = content.split('---');
  if (parts.length >= 3) {
    return parts.slice(2).join('---').trim();
  }
  return content.trim();
}

function extractGuideDescription(guidePath: string): string {
  if (!fs.existsSync(guidePath)) return '';
  const content = fs.readFileSync(guidePath, 'utf8');
  const parts = content.split('---');
  if (parts.length >= 3) {
    const frontmatter = parts[1];
    const match = frontmatter.match(/^description:\s*(.+)$/m);
    if (match) {
      return match[1].trim();
    }
  }
  return '';
}

function extractGraderTests(graderPath: string): string[] {
  if (!fs.existsSync(graderPath)) return [];
  const content = fs.readFileSync(graderPath, 'utf8');
  const lines = content.split('\n');
  const tests: string[] = [];
  const testRegex = /^(?:test|it|test\.only)\s*\(\s*(['"`])((?:[^\\]|\\.)*?)\1/;
  for (const line of lines) {
    const trimmed = line.trim();
    const match = trimmed.match(testRegex);
    if (match) {
      tests.push(match[2]);
    }
  }
  return tests;
}

interface TaskDetails {
  missingExpectedGuideCount: number;
  tooManyGuidesCount: number;
  guidedPassRates: { [agent: string]: number };
  unguidedPassRates: { [agent: string]: number };
  guidesConsumed: { [agent: string]: string[] };
  expectedGuide: string;
}

function runCommand(cmd: string): string {
  try {
    return execSync(cmd, { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 1024 * 1024 * 100 });
  } catch (err: any) {
    console.error(`Failed to execute command: ${cmd}`);
    console.error(err.stderr || err.message);
    throw err;
  }
}

interface FailedAssertion {
  testId: string;
  message: string;
}

function getFailedAssertions(agentData: any, taskName: string, mode: 'guided' | 'unguided'): FailedAssertion[] {
  const key = `task - ${taskName} - ${mode}`;
  const runs = agentData.results?.[key] || [];
  const failuresMap = new Map<string, string>();
  for (const run of runs) {
    if (run && Array.isArray(run.results)) {
      for (const res of run.results) {
        if (res.passed === false) {
          failuresMap.set(res.testId, res.message || '');
        }
      }
    }
  }
  return Array.from(failuresMap.entries()).map(([testId, message]) => ({ testId, message }));
}

function getCommonFailures(taskName: string, mode: 'guided' | 'unguided', parsedDataByAgent: Record<string, any>, activeAgents: string[]): FailedAssertion[] {
  if (activeAgents.length === 0) return [];
  
  const firstAgent = activeAgents[0];
  const firstFailures = getFailedAssertions(parsedDataByAgent[firstAgent], taskName, mode);
  let commonTestIds = new Set(firstFailures.map(f => f.testId));
  
  const messageMap = new Map<string, string>();
  for (const f of firstFailures) {
    messageMap.set(f.testId, f.message);
  }
  
  for (let i = 1; i < activeAgents.length; i++) {
    const agent = activeAgents[i];
    const agentFailures = getFailedAssertions(parsedDataByAgent[agent], taskName, mode);
    const agentTestIds = new Set(agentFailures.map(f => f.testId));
    
    const nextCommon = new Set<string>();
    for (const testId of commonTestIds) {
      if (agentTestIds.has(testId)) {
        nextCommon.add(testId);
      }
    }
    commonTestIds = nextCommon;
    
    for (const f of agentFailures) {
      messageMap.set(f.testId, f.message);
    }
  }
  
  return Array.from(commonTestIds).map(testId => ({
    testId,
    message: messageMap.get(testId) || ''
  }));
}

async function main() {
  console.log('🔍 Listing GCS bucket to locate the latest nightly runs...');
  let listOutput = '';
  try {
    listOutput = runCommand(`gcloud storage ls gs://${BUCKET_NAME}/`);
  } catch (e) {
    console.error('❌ Failed to list GCS bucket. Please make sure you are logged in with `gcloud auth login` and have access to gs://guidance-evals.');
    process.exit(1);
  }

  const lines = listOutput.split('\n');
  const nightlyRuns: { folderName: string; timestamp: string; agent: 'claude_code' | 'codex_cli' | 'jetski_cli' | null }[] = [];

  for (const line of lines) {
    const cleanLine = line.trim();
    if (!cleanLine) continue;

    // Expected formats:
    // gs://guidance-evals/nightly-YYYY-MM-DD_HH-MM-SS-agent/
    // gs://guidance-evals/nightly-YYYY-MM-DD_HH-MM-SS-agent-user/
    const match = cleanLine.match(/gs:\/\/guidance-evals\/(nightly-([0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{2}-[0-9]{2}-[0-9]{2})-(jetski_cli|codex_cli|claude_code|agy|claude|codex|jetski)(?:-([a-zA-Z0-9_]+))?)\//);
    if (!match) continue;

    const [_, folderName, timestampStr, agentStr] = match;
    let agent: 'claude_code' | 'codex_cli' | 'jetski_cli' | null = null;

    const agentLower = agentStr.toLowerCase();
    if (agentLower.includes('claude')) {
      agent = 'claude_code';
    } else if (agentLower.includes('codex')) {
      agent = 'codex_cli';
    } else if (agentLower.includes('jetski') || agentLower.includes('agy')) {
      agent = 'jetski_cli';
    }

    if (agent) {
      nightlyRuns.push({ folderName, timestamp: timestampStr, agent });
    }
  }

  if (nightlyRuns.length === 0) {
    console.error('❌ No nightly runs found in GCS!');
    process.exit(1);
  }

  // Find the latest run for each distinct agent
  const latestByAgent: Record<'claude_code' | 'codex_cli' | 'jetski_cli', string | null> = {
    claude_code: null,
    codex_cli: null,
    jetski_cli: null,
  };

  const latestTimestampByAgent: Record<'claude_code' | 'codex_cli' | 'jetski_cli', string | null> = {
    claude_code: null,
    codex_cli: null,
    jetski_cli: null,
  };

  for (const run of nightlyRuns) {
    if (!run.agent) continue;
    const currentLatestTs = latestTimestampByAgent[run.agent];
    // Compare timestamps as strings (e.g. 2026-06-01_17-00-02 vs 2026-05-21_03-30-02)
    if (!currentLatestTs || run.timestamp > currentLatestTs) {
      latestTimestampByAgent[run.agent] = run.timestamp;
      latestByAgent[run.agent] = run.folderName;
    }
  }

  console.log('\nSelected runs to download and investigate:');
  for (const [agent, folderName] of Object.entries(latestByAgent)) {
    if (folderName) {
      console.log(`  - ${agent}: ${folderName}`);
    } else {
      console.warn(`  ⚠️ No run found for agent: ${agent}`);
    }
  }

  // Ensure harness/results directory exists
  fs.mkdirSync(resultsDir, { recursive: true });

  const activeAgents = Object.keys(latestByAgent).filter(agent => latestByAgent[agent as keyof typeof latestByAgent]) as ('claude_code' | 'codex_cli' | 'jetski_cli')[];

  // Sync results for each run (excluding heavy binary zip/png files to save bandwidth)
  for (const agent of activeAgents) {
    const folderName = latestByAgent[agent]!;
    const localSuiteDir = path.join(resultsDir, folderName);

    if (fs.existsSync(localSuiteDir)) {
      console.log(`\n✅ [${agent}] Folder already exists in results, skipping download: ${folderName}`);
    } else {
      fs.mkdirSync(localSuiteDir, { recursive: true });
      console.log(`\n📥 [${agent}] Syncing results for ${folderName} (excluding heavy binary zip/png files)...`);
      try {
        runCommand(`gcloud storage rsync gs://${BUCKET_NAME}/${folderName} ${localSuiteDir} --recursive --exclude ".*\\.zip$|.*\\.png$"`);
        console.log(`✅ Sync completed successfully.`);
      } catch (err) {
        console.error(`❌ Failed to sync results for ${folderName}`);
        process.exit(1);
      }
    }
  }

  // Parse each run's data
  const parsedDataByAgent: Record<string, any> = {};
  for (const agent of activeAgents) {
    const folderName = latestByAgent[agent]!;
    const localEvalsPath = path.join(resultsDir, folderName, 'evals.json');
    try {
      const content = fs.readFileSync(localEvalsPath, 'utf8');
      parsedDataByAgent[agent] = JSON.parse(content);
    } catch (err: any) {
      console.error(`❌ Error reading/parsing local evals.json for ${agent}: ${err.message}`);
      process.exit(1);
    }
  }

  // Collect all unique tasks across all runs
  const allTasks = new Set<string>();
  for (const agent of activeAgents) {
    const data = parsedDataByAgent[agent];
    const results = data.results || {};
    for (const key of Object.keys(results)) {
      // Key is: "task - <guideName> - guided" or "task - <guideName> - unguided"
      const parts = key.split(' - ');
      if (parts.length >= 2) {
        allTasks.add(parts[1]);
      }
    }
  }

  console.log(`\nAnalyzing ${allTasks.size} distinct tasks...`);

  const taskAnalysis: Record<string, TaskDetails> = {};

  for (const taskName of allTasks) {
    taskAnalysis[taskName] = {
      missingExpectedGuideCount: 0,
      tooManyGuidesCount: 0,
      guidedPassRates: {},
      unguidedPassRates: {},
      guidesConsumed: {},
      expectedGuide: taskName,
    };

    for (const agent of activeAgents) {
      const data = parsedDataByAgent[agent];
      const results = data.results || {};
      const stats = data.stats || {};

      // Guided info
      const guidedKey = `task - ${taskName} - guided`;
      const guidedRuns = results[guidedKey] || [];
      const guidedStats = stats[guidedKey] || {};

      // Pass rate (default to 0 if no guided stats or no runs)
      const guidedPassRate = typeof guidedStats.medianPassRate === 'number' ? guidedStats.medianPassRate : 0;
      taskAnalysis[taskName].guidedPassRates[agent] = guidedPassRate;

      // Expected guide and consumed guides
      const guidesConsumed: string[] = [];
      if (guidedRuns.length > 0) {
        // For simple evaluation, look at the first run's guidesUsed
        const firstRun = guidedRuns[0];
        if (firstRun && Array.isArray(firstRun.guidesUsed)) {
          guidesConsumed.push(...firstRun.guidesUsed);
        }
      }
      taskAnalysis[taskName].guidesConsumed[agent] = guidesConsumed;

      // Flag: Missing expected guide (expected guide is taskName itself)
      if (!guidesConsumed.includes(taskName)) {
        taskAnalysis[taskName].missingExpectedGuideCount++;
      }

      // Flag: Too many guides consumed (3 or more)
      if (guidesConsumed.length >= 3) {
        taskAnalysis[taskName].tooManyGuidesCount++;
      }

      // Unguided info
      const unguidedKey = `task - ${taskName} - unguided`;
      const unguidedStats = stats[unguidedKey] || {};
      const unguidedPassRate = typeof unguidedStats.medianPassRate === 'number' ? unguidedStats.medianPassRate : 0;
      taskAnalysis[taskName].unguidedPassRates[agent] = unguidedPassRate;
    }
  }

  // Evaluate flags for each task
  const flaggedTasks: {
    taskName: string;
    flags: string[];
    details: string[];
    raw: TaskDetails;
  }[] = [];

  for (const [taskName, details] of Object.entries(taskAnalysis)) {
    const flags: string[] = [];
    const flagDetailsList: string[] = [];

    // 1. Missing expected guide in at least 2 runs
    if (details.missingExpectedGuideCount >= 2) {
      flags.push('MISSING_EXPECTED_GUIDE');
      flagDetailsList.push(`Expected guide was missing in ${details.missingExpectedGuideCount} out of ${activeAgents.length} agent runs.`);
    }

    // 2. Too many guides consumed (3 or more) in at least 2 runs
    if (details.tooManyGuidesCount >= 2) {
      flags.push('TOO_MANY_GUIDES_CONSUMED');
      flagDetailsList.push(`Consumed 3 or more guides in ${details.tooManyGuidesCount} out of ${activeAgents.length} agent runs.`);
    }

    // 3. Low guided percentage rate: Guided pass rate under 75% in all runs
    const allGuidedUnder75 = activeAgents.every(agent => details.guidedPassRates[agent] < 75);
    if (allGuidedUnder75) {
      flags.push('LOW_GUIDED_PASS_RATE');
      const rateStrings = activeAgents.map(agent => `${agent}: ${details.guidedPassRates[agent]}%`).join(', ');
      flagDetailsList.push(`Guided pass rate was under 75% in all nightly runs (${rateStrings}).`);
    }

    // 4. High unguided percentage rate: Unguided pass rate 90% or higher in all runs
    const allUnguidedOver90 = activeAgents.every(agent => details.unguidedPassRates[agent] >= 90);
    if (allUnguidedOver90) {
      flags.push('HIGH_UNGUIDED_PASS_RATE');
      const rateStrings = activeAgents.map(agent => `${agent}: ${details.unguidedPassRates[agent]}%`).join(', ');
      flagDetailsList.push(`Unguided pass rate was 90% or higher in all nightly runs (${rateStrings}).`);
    }

    if (flags.length > 0) {
      flaggedTasks.push({
        taskName,
        flags,
        details: flagDetailsList,
        raw: details,
      });
    }
  }

  console.log(`\nAnalysis completed. Flagged ${flaggedTasks.length} out of ${allTasks.size} tasks.`);

  // Generate reports
  const outputDir = path.join(__dirname, '../artifacts');
  fs.mkdirSync(outputDir, { recursive: true });

  const guidesDir = path.join(repoRoot, 'guides');
  const guideDirsMap = findGuideDirs(guidesDir);

  const markdownReportPath = path.join(outputDir, 'nightly_investigation_report.md');

  // 1. Build Markdown Report
  let md = `# Nightly Evaluation Investigation Report\n\n`;
  md += `**Generated:** ${new Date().toISOString().split('T')[0]}\n`;
  md += `**Suites Investigated:**\n`;
  for (const agent of activeAgents) {
    md += `- **${agent}**: \`${latestByAgent[agent]}\`\n`;
  }
  md += `\n---\n\n`;

  md += `## Summary of Flagged Tasks\n\n`;
  md += `- **Total distinct tasks analyzed:** ${allTasks.size}\n`;
  md += `- **Total flagged tasks:** ${flaggedTasks.length}\n\n`;

  if (flaggedTasks.length === 0) {
    md += `✅ **No flagged tasks! All nightly runs passed our health thresholds.**\n`;
  } else {
    md += `### Flagged Tasks Table\n\n`;
    md += `| Task / Guide | Flags | Details |\n`;
    md += `| :--- | :--- | :--- |\n`;
    for (const item of flaggedTasks) {
      md += `| \`${item.taskName}\` | ${item.flags.map(f => `\`${f}\``).join(', ')} | ${item.details.join(' <br> ')} |\n`;
    }
    md += `\n\n## Flagged Tasks Details\n\n`;

    for (const item of flaggedTasks) {
      md += `<a name="${getSlug(item.taskName)}"></a>\n### \`${item.taskName}\`\n\n`;
      md += `#### Flags:\n`;
      for (let i = 0; i < item.flags.length; i++) {
        md += `- **\`${item.flags[i]}\`**: ${item.details[i]}\n`;
      }
      md += `\n#### Run Details:\n\n`;
      md += `| Agent | Guided Pass Rate | Unguided Pass Rate | Guides Consumed |\n`;
      md += `| :--- | :---: | :---: | :--- |\n`;
      for (const agent of activeAgents) {
        const guidedRate = `${item.raw.guidedPassRates[agent]}%`;
        const unguidedRate = `${item.raw.unguidedPassRates[agent]}%`;
        const consumed = item.raw.guidesConsumed[agent].length > 0 
          ? item.raw.guidesConsumed[agent].map(g => `\`${g}\``).join(', ')
          : '*None*';
        md += `| **${agent}** | ${guidedRate} | ${unguidedRate} | ${consumed} |\n`;
      }
      
      const commonGuided = item.flags.includes('LOW_GUIDED_PASS_RATE')
        ? getCommonFailures(item.taskName, 'guided', parsedDataByAgent, activeAgents)
        : [];
      
      md += `\n#### Qualitative Diagnostic Summary:\n\n`;
      md += `- **Diagnostic Details**:\n`;
      
      if (commonGuided.length > 0) {
        md += `  - *Failed Assertions*:\n`;
        for (const f of commonGuided) {
          md += `    - "${f.message}" (Test ID: \`${f.testId}\`)\n`;
        }
      }
      
      for (const flag of item.flags) {
        md += `  - **${flag}**: TODO\n`;
      }

      const folderPath = guideDirsMap[item.taskName];
      const relativePath = folderPath ? path.relative(repoRoot, folderPath) : `guides/.../${item.taskName}`;

      md += `- **Actionable Recommendations**:\n`;
      md += `  - [ ] **Prompt** (\`${relativePath}/tasks/task.md\`): TODO\n`;
      md += `  - [ ] **Guide** (\`${relativePath}/guide.md\`): TODO\n`;
      md += `  - [ ] **Grader** (\`${relativePath}/grader.ts\`): TODO\n\n`;
      md += `---\n\n`;
    }
  }

  fs.writeFileSync(markdownReportPath, md);
  console.log(`💾 Saved Markdown report to: ${markdownReportPath}`);

  // 2. Build Flagged Tasks Context JSON
  if (flaggedTasks.length > 0) {
    console.log('📂 Gathering context details (task prompt, guide description, unit tests) for flagged tasks...');
    const contextMap: Record<string, {
      flags: string[];
      guidePath: string;
      prompt: string;
      guideDescription: string;
      testHeaders: string[];
    }> = {};

    for (const item of flaggedTasks) {
      const folderPath = guideDirsMap[item.taskName];
      if (!folderPath) {
        console.warn(`  ⚠️ Could not find guide folder for task: ${item.taskName}`);
        continue;
      }
      
      const relativeGuidePath = path.relative(repoRoot, folderPath);
      const taskPath = path.join(folderPath, 'tasks', 'task.md');
      const guidePath = path.join(folderPath, 'guide.md');
      const graderPath = path.join(folderPath, 'grader.ts');

      contextMap[item.taskName] = {
        flags: item.flags,
        guidePath: relativeGuidePath,
        prompt: extractTaskPrompt(taskPath),
        guideDescription: extractGuideDescription(guidePath),
        testHeaders: extractGraderTests(graderPath),
      };
    }

    const contextJsonPath = path.join(outputDir, 'flagged_tasks_context.json');
    fs.writeFileSync(contextJsonPath, JSON.stringify(contextMap, null, 2));
    console.log(`💾 Saved Flagged Tasks Context JSON to: ${contextJsonPath}`);
  }

  console.log('\n🎯 Investigation analysis finished successfully!');
}

main().catch(console.error);

import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(__dirname, '../../../../');
const reportPath = path.join(repoRoot, '.agents', 'skills', 'nightly-eval-investigation', 'artifacts', 'nightly_investigation_report.md');

function getSlug(taskName: string): string {
  return taskName.toLowerCase().replace(/[^a-z0-9_-]/g, '').replace(/\s+/g, '-');
}

function getOrCreateLabel(labelName: string, color: string, description: string) {
  try {
    console.log(`Checking/Creating label: "${labelName}"...`);
    execSync(`gh label create "${labelName}" --color "${color}" --description "${description}" || true`, { stdio: 'ignore' });
  } catch (e) {
    // Ignore error if it already exists or if we lack permissions but want to try anyway
  }
}

async function main() {
  const dateStr = new Date().toISOString().split('T')[0];
  const createdIssues: { title: string; url: string; type: string }[] = [];

  // 1. Read the generated markdown report
  if (!fs.existsSync(reportPath)) {
    console.error(`❌ Error: Report file does not exist at ${reportPath}`);
    process.exit(1);
  }

  const reportContent = fs.readFileSync(reportPath, 'utf-8');
  let parentBody = reportContent.replace(
    /- \*\*Actionable Recommendations\*\*:\n[\s\S]*?(?=\n---|$$)/g,
    ''
  );

  if (parentBody.length > 65536) {
    console.error(`❌ Error: The parent issue body size (${parentBody.length} characters) exceeds GitHub's limit of 65,536 characters.`);
    console.error(`Please adjust health thresholds in investigate.ts to reduce the number of flagged tasks.`);
    process.exit(1);
  }

  // 2. Create the parent issue on GitHub
  const parentTitle = `📋 Nightly Guidance Health Audit: ${dateStr}`;
  getOrCreateLabel('nightly-investigation', '1D76DB', 'Nightly Guidance Health Audits');
  console.log(`📦 Creating parent GitHub issue: "${parentTitle}"...`);
  
  let parentUrl = '';
  let parentNum = '';
  const tempReportPath = reportPath + '.parent.tmp';
  try {
    fs.writeFileSync(tempReportPath, parentBody, 'utf-8');
    const parentOutput = execSync(`gh issue create --title "${parentTitle}" --body-file "${tempReportPath}" --label "nightly-investigation"`, { encoding: 'utf-8' });
    parentUrl = parentOutput.trim();
    console.log(`✅ Parent issue created at: ${parentUrl}`);
    createdIssues.push({ title: parentTitle, url: parentUrl, type: 'parent' });
    const parentNumMatch = parentUrl.match(/\/issues\/(\d+)/);
    parentNum = parentNumMatch ? parentNumMatch[1] : '';
  } catch (err: any) {
    console.error('❌ Failed to create parent issue. Check GitHub CLI authentication.', err.message);
    if (fs.existsSync(tempReportPath)) fs.unlinkSync(tempReportPath);
    process.exit(1);
  } finally {
    if (fs.existsSync(tempReportPath)) fs.unlinkSync(tempReportPath);
  }

  // 3. Parse the report to find recommendations per task and create subissues
  const taskSections = reportContent.split(/(?=<a name="[^"]+"><\/a>\s*\n### `[^`]+`)/);
  if (taskSections.length <= 1) {
    console.log('ℹ️ No flagged tasks found or no task details generated.');
    return;
  }

  // Ensure labels exist
  getOrCreateLabel(`nightly-investigation-eng-${dateStr}`, '2B90B5', 'Engineering nightly investigation');
  getOrCreateLabel(`nightly-investigation-devrel-${dateStr}`, 'E27D60', 'Devrel nightly investigation');

  for (let i = 1; i < taskSections.length; i++) {
    const section = taskSections[i];
    const headerMatch = section.match(/### `([^`]+)`/);
    if (!headerMatch) continue;
    const taskName = headerMatch[1];
    
    // Parse metadata
    const runDetails: Record<string, { guided: string; unguided: string; guides: string }> = {};
    const agentLines = section.split('\n').filter(l => l.trim().startsWith('| **'));
    for (const line of agentLines) {
      const parts = line.split('|').map(p => p.trim());
      if (parts.length >= 5) {
        const agentName = parts[1].replace(/\*\*/g, '');
        runDetails[agentName] = {
          guided: parts[2],
          unguided: parts[3],
          guides: parts[4],
        };
      }
    }

    const flagsMatch = section.match(/#### Flags:\n([\s\S]*?)(?=\n#### Run Details:|\n#### Qualitative|\n- \*\*Actionable|$)/);
    const flags: string[] = [];
    if (flagsMatch) {
      const flagLines = flagsMatch[1].split('\n');
      for (const fl of flagLines) {
        const fm = fl.match(/-\s+\*\*`([^`]+)`\*\*/);
        if (fm) {
          flags.push(fm[1]);
        }
      }
    }

    // Parse recommendations
    const recommendationsMatch = section.match(/- \*\*Actionable Recommendations\*\*:\n([\s\S]*?)(?=\n---|$$)/);
    const recLines = (recommendationsMatch ? recommendationsMatch[1] : '').split('\n');
    let promptRec: { path?: string; detail: string } | null = null;
    let guideRec: { path?: string; detail: string } | null = null;
    let graderRec: { path?: string; detail: string } | null = null;

    for (const rl of recLines) {
      const m = rl.match(/^\s*-\s*\[[ x]?\]\s*\*\*(Prompt|Guide|Grader)\*\*\s*(?:\(`?([^`)]+)`?\))?:\s*(.*)$/i);
      if (m) {
        const type = m[1].toLowerCase();
        const filePath = m[2]?.trim();
        const detail = m[3]?.trim();
        if (detail && !detail.toUpperCase().includes('TODO')) {
          const rec = { path: filePath, detail };
          if (type === 'prompt') promptRec = rec;
          else if (type === 'guide') guideRec = rec;
          else if (type === 'grader') graderRec = rec;
        }
      }
    }

    const anchorUrl = `${parentUrl}#user-content-${getSlug(taskName)}`;

    // Create Grader Subissue (Engineering)
    if (graderRec) {
      const graderTitle = `Grader Fix: ${taskName} - ${dateStr}`;
      const graderBody = `### Grader Fix Recommendation for \`${taskName}\`

**Task / Guide**: \`${taskName}\`
**File to change**: \`${graderRec.path || 'Unknown'}\`
**Audit Report**: [Nightly Audit Report Issue #${parentNum}](${anchorUrl})

#### Recommended Grader Changes:
${graderRec.detail}

#### Run Details & Metadata:
- **Flags Triggered**: ${flags.map(f => `\`${f}\``).join(', ')}
${Object.entries(runDetails).map(([agent, details]) => `- **${agent}**: Guided Pass Rate: ${details.guided}, Unguided Pass Rate: ${details.unguided}, Guides Consumed: ${details.guides}`).join('\n')}`;

      console.log(`Creating Grader subissue: "${graderTitle}"...`);
      const tempGraderPath = path.join(repoRoot, `.agents/skills/nightly-eval-investigation/artifacts/grader_subissue_${taskName}.tmp`);
      try {
        fs.writeFileSync(tempGraderPath, graderBody, 'utf-8');
        const graderIssueUrl = execSync(
          `gh issue create --title "${graderTitle}" --body-file "${tempGraderPath}" --label "nightly-investigation-eng-${dateStr}"`,
          { encoding: 'utf-8' }
        ).trim();
        console.log(`✅ Grader subissue created: ${graderIssueUrl}`);
        createdIssues.push({ title: graderTitle, url: graderIssueUrl, type: 'eng-subissue' });
      } catch (err: any) {
        console.error(`❌ Failed to create Grader subissue:`, err.message);
      } finally {
        if (fs.existsSync(tempGraderPath)) fs.unlinkSync(tempGraderPath);
      }
    }

    // Create Task & Guide Subissue (Devrel)
    if (promptRec || guideRec) {
      const devrelTitle = `Task/Guide Fix: ${taskName} - ${dateStr}`;
      const devrelBody = `### Task & Guide Fix Recommendation for \`${taskName}\`

**Task / Guide**: \`${taskName}\`
**Files to change**:
${promptRec ? `- **Prompt**: \`${promptRec.path || 'Unknown'}\`` : ''}
${guideRec ? `- **Guide**: \`${guideRec.path || 'Unknown'}\`` : ''}
**Audit Report**: [Nightly Audit Report Issue #${parentNum}](${anchorUrl})

${promptRec ? `#### Recommended Prompt Changes:\n${promptRec.detail}\n` : ''}
${guideRec ? `#### Recommended Guide Changes:\n${guideRec.detail}\n` : ''}

#### Run Details & Metadata:
- **Flags Triggered**: ${flags.map(f => `\`${f}\``).join(', ')}
${Object.entries(runDetails).map(([agent, details]) => `- **${agent}**: Guided Pass Rate: ${details.guided}, Unguided Pass Rate: ${details.unguided}, Guides Consumed: ${details.guides}`).join('\n')}`;

      console.log(`Creating Devrel subissue: "${devrelTitle}"...`);
      const tempDevrelPath = path.join(repoRoot, `.agents/skills/nightly-eval-investigation/artifacts/devrel_subissue_${taskName}.tmp`);
      try {
        fs.writeFileSync(tempDevrelPath, devrelBody, 'utf-8');
        const devrelIssueUrl = execSync(
          `gh issue create --title "${devrelTitle}" --body-file "${tempDevrelPath}" --label "nightly-investigation-devrel-${dateStr}"`,
          { encoding: 'utf-8' }
        ).trim();
        console.log(`✅ Devrel subissue created: ${devrelIssueUrl}`);
        createdIssues.push({ title: devrelTitle, url: devrelIssueUrl, type: 'devrel-subissue' });

        // Add to Project 30 (Modern Web Guidance) and set Status to needs-investigation
        try {
          console.log(`Adding subissue to Project 30 (GoogleChrome)...`);
          const addOutput = execSync(
            `gh project item-add 30 --owner GoogleChrome --url "${devrelIssueUrl}" --format json`,
            { encoding: 'utf-8' }
          ).trim();
          const item = JSON.parse(addOutput);
          const itemId = item.id;

          // Fetch Project ID
          const projOutput = execSync(`gh project view 30 --owner GoogleChrome --format json`, { encoding: 'utf-8' }).trim();
          const proj = JSON.parse(projOutput);
          const projectId = proj.id;

          // Fetch fields and options to set Status field to needs-investigation
          const fieldsOutput = execSync(`gh project field-list 30 --owner GoogleChrome --format json`, { encoding: 'utf-8' }).trim();
          const fields = JSON.parse(fieldsOutput);
          const fieldsList = Array.isArray(fields) ? fields : fields.fields;
          const statusField = fieldsList.find((f: any) => f.name?.toLowerCase() === 'status');
          
          if (statusField) {
            const fieldId = statusField.id;
            const options = statusField.options || statusField.settings?.options || [];
            const needsInvestigationOption = options.find(
              (o: any) => o.name?.toLowerCase() === 'needs-investigation' || o.name?.toLowerCase() === 'needs investigation'
            );
            if (needsInvestigationOption) {
              const optionId = needsInvestigationOption.id;
              execSync(
                `gh project item-edit --id "${itemId}" --field-id "${fieldId}" --project-id "${projectId}" --single-select-option-id "${optionId}"`,
                { stdio: 'inherit' }
              );
              console.log(`✅ Successfully added to Project 30 and set Status to needs-investigation.`);
            } else {
              console.warn(`⚠️ Could not find "needs-investigation" option in Project 30 Status options.`);
            }
          } else {
            console.warn(`⚠️ Could not find "Status" field in Project 30 fields.`);
          }
        } catch (projErr: any) {
          console.error(`❌ Failed to integrate with Project 30:`, projErr.message);
        }
      } catch (err: any) {
        console.error(`❌ Failed to create Devrel subissue:`, err.message);
      } finally {
        if (fs.existsSync(tempDevrelPath)) fs.unlinkSync(tempDevrelPath);
      }
    }
  }

  // 4. Link all created subissues to the parent issue natively
  const subissues = createdIssues.filter(i => i.type !== 'parent');
  if (subissues.length > 0) {
    try {
      console.log(`🔗 Fetching GraphQL ID for parent issue...`);
      const parentNodeId = execSync(`gh issue view ${parentUrl} --json id --jq .id`, { encoding: 'utf-8' }).trim();
      
      for (const issue of subissues) {
        console.log(`🔗 Linking subissue "${issue.title}" to parent natively...`);
        try {
          const subIssueNodeId = execSync(`gh issue view ${issue.url} --json id --jq .id`, { encoding: 'utf-8' }).trim();
          const mutation = 'mutation($issueId: ID!, $subIssueId: ID!) { addSubIssue(input: { issueId: $issueId, subIssueId: $subIssueId }) { clientMutationId } }';
          execSync(`gh api graphql -f query='${mutation}' -f issueId="${parentNodeId}" -f subIssueId="${subIssueNodeId}"`, { stdio: 'ignore' });
        } catch (linkErr: any) {
          console.error(`❌ Failed to link subissue "${issue.title}":`, linkErr.message);
        }
      }
      console.log(`✅ Native sub-issue relationships established.`);
    } catch (parentErr: any) {
      console.error(`❌ Failed to fetch parent GraphQL ID:`, parentErr.message);
    }
  }

  console.log('\n📦 Summary of Created Issues:');
  for (const issue of createdIssues) {
    console.log(`- [${issue.type.toUpperCase()}] ${issue.title}`);
    console.log(`  Url: ${issue.url}`);
  }

  console.log('\n🎉 Nightly investigation publisher script completed!');
}

main().catch(console.error);