Usmetrics

#!/usr/bin/env bun
/**
 * fetch-fred-series.ts
 *
 * Fetches historical data from FRED (Federal Reserve Economic Data) API
 * for use in US Metrics analysis.
 *
 * Usage:
 *   bun run fetch-fred-series.ts <series_id> [--years=10]
 *   bun run fetch-fred-series.ts UNRATE --years=10
 *   bun run fetch-fred-series.ts --all --years=10
 *
 * Environment:
 *   FRED_API_KEY - Required API key from https://fred.stlouisfed.org/docs/api/api_key.html
 */

import { parseArgs } from "util";

// Core economic series for US-Common-Metrics
const CORE_SERIES: Record<string, { name: string; category: string; unit: string }> = {
  // Economic Output & Growth
  "GDPC1": { name: "Real GDP", category: "Economic Output", unit: "Billions of Chained 2017 Dollars" },
  "A191RL1Q225SBEA": { name: "Real GDP Growth Rate", category: "Economic Output", unit: "Percent Change" },
  "INDPRO": { name: "Industrial Production Index", category: "Economic Output", unit: "Index 2017=100" },
  "RSXFS": { name: "Retail Sales", category: "Economic Output", unit: "Millions of Dollars" },

  // Inflation & Prices
  "CPIAUCSL": { name: "CPI-U All Items", category: "Inflation", unit: "Index 1982-84=100" },
  "CPILFESL": { name: "Core CPI (ex Food/Energy)", category: "Inflation", unit: "Index 1982-84=100" },
  "PCEPI": { name: "PCE Price Index", category: "Inflation", unit: "Index 2017=100" },
  "PCEPILFE": { name: "Core PCE", category: "Inflation", unit: "Index 2017=100" },
  "DCOILWTICO": { name: "WTI Crude Oil", category: "Inflation", unit: "Dollars per Barrel" },

  // Employment & Labor
  "UNRATE": { name: "Unemployment Rate (U-3)", category: "Employment", unit: "Percent" },
  "U6RATE": { name: "Underemployment Rate (U-6)", category: "Employment", unit: "Percent" },
  "PAYEMS": { name: "Nonfarm Payrolls", category: "Employment", unit: "Thousands of Persons" },
  "ICSA": { name: "Initial Jobless Claims", category: "Employment", unit: "Number" },
  "CCSA": { name: "Continuing Claims", category: "Employment", unit: "Number" },
  "JTSJOL": { name: "Job Openings", category: "Employment", unit: "Level in Thousands" },
  "JTSQUR": { name: "Quit Rate", category: "Employment", unit: "Percent" },
  "CIVPART": { name: "Labor Force Participation", category: "Employment", unit: "Percent" },
  "CES0500000003": { name: "Average Hourly Earnings", category: "Employment", unit: "Dollars per Hour" },

  // Housing
  "MSPUS": { name: "Median Sales Price of Houses", category: "Housing", unit: "Dollars" },
  "EXHOSLUSM495S": { name: "Existing Home Sales", category: "Housing", unit: "Number of Units" },
  "HSN1F": { name: "New Home Sales", category: "Housing", unit: "Thousands" },
  "HOUST": { name: "Housing Starts", category: "Housing", unit: "Thousands of Units" },
  "PERMIT": { name: "Building Permits", category: "Housing", unit: "Thousands of Units" },
  "RHORUSQ156N": { name: "Homeownership Rate", category: "Housing", unit: "Percent" },
  "MORTGAGE30US": { name: "30-Year Mortgage Rate", category: "Housing", unit: "Percent" },
  "CSUSHPINSA": { name: "Case-Shiller Home Price Index", category: "Housing", unit: "Index Jan 2000=100" },

  // Consumer & Personal Finance
  "UMCSENT": { name: "Consumer Sentiment (UMich)", category: "Consumer", unit: "Index 1966:Q1=100" },
  "PI": { name: "Personal Income", category: "Consumer", unit: "Billions of Dollars" },
  "DSPI": { name: "Disposable Personal Income", category: "Consumer", unit: "Billions of Dollars" },
  "PSAVERT": { name: "Personal Saving Rate", category: "Consumer", unit: "Percent" },
  "TOTALSL": { name: "Consumer Credit Outstanding", category: "Consumer", unit: "Billions of Dollars" },
  "DRCCLACBS": { name: "Credit Card Delinquency Rate", category: "Consumer", unit: "Percent" },
  "TDSP": { name: "Debt Service Ratio", category: "Consumer", unit: "Percent" },

  // Financial Markets
  "DGS10": { name: "10-Year Treasury Yield", category: "Financial", unit: "Percent" },
  "DGS2": { name: "2-Year Treasury Yield", category: "Financial", unit: "Percent" },
  "FEDFUNDS": { name: "Fed Funds Rate", category: "Financial", unit: "Percent" },
  "VIXCLS": { name: "VIX Volatility Index", category: "Financial", unit: "Index" },
  "STLFSI4": { name: "Financial Stress Index", category: "Financial", unit: "Index" },

  // Trade & International
  "BOPGSTB": { name: "Trade Balance", category: "Trade", unit: "Millions of Dollars" },
  "BOPGEXP": { name: "Exports", category: "Trade", unit: "Millions of Dollars" },
  "BOPGIMP": { name: "Imports", category: "Trade", unit: "Millions of Dollars" },
  "DTWEXBGS": { name: "Trade Weighted U.S. Dollar Index", category: "Trade", unit: "Index Jan 2006=100" },

  // Government & Fiscal
  "GFDEBTN": { name: "Federal Debt Total", category: "Fiscal", unit: "Millions of Dollars" },
  "FYFSD": { name: "Federal Surplus/Deficit", category: "Fiscal", unit: "Millions of Dollars" },
  "FGEXPND": { name: "Federal Spending", category: "Fiscal", unit: "Millions of Dollars" },
  "FGRECPT": { name: "Federal Receipts", category: "Fiscal", unit: "Millions of Dollars" },

  // Demographics (Annual)
  "POPTHM": { name: "U.S. Population", category: "Demographics", unit: "Thousands" },
  "SIPOVGINIUSA": { name: "GINI Index", category: "Demographics", unit: "Index" },
};

interface FredObservation {
  date: string;
  value: string;
}

interface FredResponse {
  observations: FredObservation[];
  seriess?: Array<{ title: string; units: string; frequency: string }>;
}

interface SeriesData {
  series_id: string;
  name: string;
  category: string;
  unit: string;
  observations: Array<{ date: string; value: number | null }>;
  latest: { date: string; value: number | null } | null;
  stats: {
    min: number;
    max: number;
    mean: number;
    count: number;
  } | null;
}

async function fetchFredSeries(
  seriesId: string,
  years: number = 10
): Promise<SeriesData | null> {
  const apiKey = process.env.FRED_API_KEY;

  if (!apiKey) {
    console.error("Error: FRED_API_KEY environment variable not set");
    console.error("Get your free API key at: https://fred.stlouisfed.org/docs/api/api_key.html");
    process.exit(1);
  }

  const endDate = new Date();
  const startDate = new Date();
  startDate.setFullYear(startDate.getFullYear() - years);

  const startStr = startDate.toISOString().split('T')[0];
  const endStr = endDate.toISOString().split('T')[0];

  const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${seriesId}&api_key=${apiKey}&file_type=json&observation_start=${startStr}&observation_end=${endStr}`;

  try {
    const response = await fetch(url);

    if (!response.ok) {
      console.error(`Error fetching ${seriesId}: HTTP ${response.status}`);
      return null;
    }

    const data: FredResponse = await response.json();

    if (!data.observations || data.observations.length === 0) {
      console.error(`No data found for ${seriesId}`);
      return null;
    }

    const seriesInfo = CORE_SERIES[seriesId] || {
      name: seriesId,
      category: "Unknown",
      unit: "Unknown"
    };

    const observations = data.observations
      .map(obs => ({
        date: obs.date,
        value: obs.value === "." ? null : parseFloat(obs.value)
      }))
      .filter(obs => obs.value !== null);

    const values = observations.map(o => o.value).filter((v): v is number => v !== null);

    const stats = values.length > 0 ? {
      min: Math.min(...values),
      max: Math.max(...values),
      mean: values.reduce((a, b) => a + b, 0) / values.length,
      count: values.length
    } : null;

    const latest = observations.length > 0
      ? observations[observations.length - 1]
      : null;

    return {
      series_id: seriesId,
      name: seriesInfo.name,
      category: seriesInfo.category,
      unit: seriesInfo.unit,
      observations,
      latest,
      stats
    };

  } catch (error) {
    console.error(`Error fetching ${seriesId}:`, error);
    return null;
  }
}

function calculateTrendStats(data: SeriesData, periodYears: number): {
  startValue: number | null;
  endValue: number | null;
  absoluteChange: number | null;
  percentChange: number | null;
  cagr: number | null;
  direction: "↑" | "↓" | "→";
} | null {
  if (!data.observations || data.observations.length === 0) return null;

  const cutoffDate = new Date();
  cutoffDate.setFullYear(cutoffDate.getFullYear() - periodYears);

  const periodData = data.observations.filter(
    obs => new Date(obs.date) >= cutoffDate && obs.value !== null
  );

  if (periodData.length < 2) return null;

  const startValue = periodData[0].value;
  const endValue = periodData[periodData.length - 1].value;

  if (startValue === null || endValue === null) return null;

  const absoluteChange = endValue - startValue;
  const percentChange = ((endValue - startValue) / Math.abs(startValue)) * 100;

  // CAGR calculation
  const years = periodYears;
  const cagr = startValue !== 0
    ? (Math.pow(endValue / startValue, 1 / years) - 1) * 100
    : null;

  // Direction determination
  let direction: "↑" | "↓" | "→";
  if (Math.abs(percentChange) < 2) {
    direction = "→";
  } else if (percentChange > 0) {
    direction = "↑";
  } else {
    direction = "↓";
  }

  return {
    startValue,
    endValue,
    absoluteChange,
    percentChange,
    cagr,
    direction
  };
}

async function main() {
  const { values, positionals } = parseArgs({
    args: Bun.argv.slice(2),
    options: {
      years: { type: "string", default: "10" },
      all: { type: "boolean", default: false },
      json: { type: "boolean", default: false },
      trends: { type: "boolean", default: false },
      help: { type: "boolean", short: "h", default: false },
    },
    allowPositionals: true,
  });

  if (values.help) {
    console.log(`
fetch-fred-series.ts - Fetch economic data from FRED API

Usage:
  bun run fetch-fred-series.ts <series_id> [options]
  bun run fetch-fred-series.ts --all [options]

Options:
  --years=N     Number of years of history (default: 10)
  --all         Fetch all core series
  --json        Output as JSON
  --trends      Include trend calculations
  -h, --help    Show this help

Examples:
  bun run fetch-fred-series.ts UNRATE
  bun run fetch-fred-series.ts GDPC1 --years=20 --trends
  bun run fetch-fred-series.ts --all --json > data.json

Environment:
  FRED_API_KEY  Your FRED API key (required)
`);
    process.exit(0);
  }

  const years = parseInt(values.years || "10");
  const seriesIds = values.all ? Object.keys(CORE_SERIES) : positionals;

  if (seriesIds.length === 0) {
    console.error("Error: Specify a series ID or use --all");
    console.error("Run with --help for usage information");
    process.exit(1);
  }

  const results: SeriesData[] = [];

  for (const seriesId of seriesIds) {
    console.error(`Fetching ${seriesId}...`);
    const data = await fetchFredSeries(seriesId, years);

    if (data) {
      if (values.trends) {
        const trends = {
          "10y": calculateTrendStats(data, 10),
          "5y": calculateTrendStats(data, 5),
          "2y": calculateTrendStats(data, 2),
          "1y": calculateTrendStats(data, 1),
        };
        (data as any).trends = trends;
      }
      results.push(data);
    }
  }

  if (values.json) {
    console.log(JSON.stringify(results, null, 2));
  } else {
    for (const result of results) {
      console.log(`\n${"=".repeat(60)}`);
      console.log(`${result.name} (${result.series_id})`);
      console.log(`Category: ${result.category}`);
      console.log(`Unit: ${result.unit}`);

      if (result.latest) {
        console.log(`Latest: ${result.latest.value} (${result.latest.date})`);
      }

      if (result.stats) {
        console.log(`Range: ${result.stats.min.toFixed(2)} - ${result.stats.max.toFixed(2)}`);
        console.log(`Mean: ${result.stats.mean.toFixed(2)}`);
        console.log(`Observations: ${result.stats.count}`);
      }

      if ((result as any).trends) {
        console.log(`\nTrend Analysis:`);
        const trends = (result as any).trends;
        for (const [period, trend] of Object.entries(trends)) {
          if (trend) {
            const t = trend as any;
            console.log(`  ${period}: ${t.startValue?.toFixed(2)} → ${t.endValue?.toFixed(2)} (${t.percentChange?.toFixed(1)}% ${t.direction})`);
          }
        }
      }
    }
  }
}

main();

#!/usr/bin/env bun
/**
 * GenerateAnalysis.ts
 *
 * Generates the US Economic State Analysis document
 * by fetching data and producing the structured markdown report.
 *
 * Usage:
 *   bun run GenerateAnalysis.ts [--output=path]
 *
 * Environment:
 *   FRED_API_KEY - Required for FRED data
 *   EIA_API_KEY - Required for energy data
 */

import { parseArgs } from "util";

// ============================================================================
// CONFIGURATION
// ============================================================================

const FRED_API_KEY = process.env.FRED_API_KEY;
const EIA_API_KEY = process.env.EIA_API_KEY;

// Priority series for the analysis (most impactful metrics)
const PRIORITY_SERIES = {
  economic: ["GDPC1", "A191RL1Q225SBEA", "INDPRO", "RSXFS"],
  inflation: ["CPIAUCSL", "CPILFESL", "PCEPI", "PCEPILFE"],
  employment: ["UNRATE", "PAYEMS", "ICSA", "CIVPART", "CES0500000003"],
  housing: ["MSPUS", "MORTGAGE30US", "HOUST", "CSUSHPINSA"],
  consumer: ["UMCSENT", "PSAVERT", "TOTALSL", "DRCCLACBS"],
  financial: ["FEDFUNDS", "DGS10", "DGS2", "VIXCLS"],
  trade: ["BOPGSTB", "DTWEXBGS"],
  fiscal: ["GFDEBTN", "FYFSD"],
};

const SERIES_NAMES: Record<string, string> = {
  "GDPC1": "Real GDP",
  "A191RL1Q225SBEA": "GDP Growth Rate",
  "INDPRO": "Industrial Production",
  "RSXFS": "Retail Sales",
  "CPIAUCSL": "CPI All Items",
  "CPILFESL": "Core CPI",
  "PCEPI": "PCE Price Index",
  "PCEPILFE": "Core PCE",
  "UNRATE": "Unemployment Rate",
  "PAYEMS": "Nonfarm Payrolls",
  "ICSA": "Initial Jobless Claims",
  "CIVPART": "Labor Force Participation",
  "CES0500000003": "Average Hourly Earnings",
  "MSPUS": "Median Home Price",
  "MORTGAGE30US": "30-Year Mortgage Rate",
  "HOUST": "Housing Starts",
  "CSUSHPINSA": "Case-Shiller Index",
  "UMCSENT": "Consumer Sentiment",
  "PSAVERT": "Personal Saving Rate",
  "TOTALSL": "Consumer Credit",
  "DRCCLACBS": "Credit Card Delinquency",
  "FEDFUNDS": "Fed Funds Rate",
  "DGS10": "10-Year Treasury",
  "DGS2": "2-Year Treasury",
  "VIXCLS": "VIX",
  "BOPGSTB": "Trade Balance",
  "DTWEXBGS": "USD Index",
  "GFDEBTN": "Federal Debt",
  "FYFSD": "Federal Deficit",
};

// ============================================================================
// DATA FETCHING
// ============================================================================

interface Observation {
  date: string;
  value: number;
}

interface SeriesResult {
  id: string;
  name: string;
  observations: Observation[];
  latest: Observation | null;
  trends: {
    "10y": TrendStat | null;
    "5y": TrendStat | null;
    "2y": TrendStat | null;
    "1y": TrendStat | null;
  };
}

interface TrendStat {
  start: number;
  end: number;
  change: number;
  pctChange: number;
  direction: "↑" | "↓" | "→";
}

async function fetchFredSeries(seriesId: string, years: number = 10): Promise<SeriesResult | null> {
  if (!FRED_API_KEY) {
    console.error("FRED_API_KEY not set");
    return null;
  }

  const endDate = new Date();
  const startDate = new Date();
  startDate.setFullYear(startDate.getFullYear() - years);

  const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${seriesId}&api_key=${FRED_API_KEY}&file_type=json&observation_start=${startDate.toISOString().split('T')[0]}&observation_end=${endDate.toISOString().split('T')[0]}`;

  try {
    const response = await fetch(url);
    if (!response.ok) return null;

    const data = await response.json();
    if (!data.observations?.length) return null;

    const observations: Observation[] = data.observations
      .filter((o: any) => o.value !== ".")
      .map((o: any) => ({ date: o.date, value: parseFloat(o.value) }));

    const latest = observations[observations.length - 1] || null;

    // Calculate trends
    const trends = {
      "10y": calculateTrend(observations, 10),
      "5y": calculateTrend(observations, 5),
      "2y": calculateTrend(observations, 2),
      "1y": calculateTrend(observations, 1),
    };

    return {
      id: seriesId,
      name: SERIES_NAMES[seriesId] || seriesId,
      observations,
      latest,
      trends,
    };
  } catch (e) {
    console.error(`Error fetching ${seriesId}:`, e);
    return null;
  }
}

function calculateTrend(observations: Observation[], years: number): TrendStat | null {
  const cutoff = new Date();
  cutoff.setFullYear(cutoff.getFullYear() - years);

  const filtered = observations.filter(o => new Date(o.date) >= cutoff);
  if (filtered.length < 2) return null;

  const start = filtered[0].value;
  const end = filtered[filtered.length - 1].value;
  const change = end - start;
  const pctChange = (change / Math.abs(start)) * 100;

  let direction: "↑" | "↓" | "→";
  if (Math.abs(pctChange) < 2) direction = "→";
  else if (pctChange > 0) direction = "↑";
  else direction = "↓";

  return { start, end, change, pctChange, direction };
}

async function fetchEIAGasPrice(): Promise<{ value: number; date: string } | null> {
  if (!EIA_API_KEY) return null;

  const url = `https://api.eia.gov/v2/petroleum/pri/gnd/data/?api_key=${EIA_API_KEY}&frequency=weekly&data[0]=value&facets[product][]=EPMR&facets[duession][]=Y&sort[0][column]=period&sort[0][direction]=desc&length=1`;

  try {
    const response = await fetch(url);
    if (!response.ok) return null;

    const data = await response.json();
    const item = data.response?.data?.[0];
    if (!item) return null;

    return { value: parseFloat(item.value), date: item.period };
  } catch (e) {
    return null;
  }
}

// ============================================================================
// ANALYSIS GENERATION
// ============================================================================

function formatValue(value: number, seriesId: string): string {
  // Format based on series type
  if (["UNRATE", "CIVPART", "PSAVERT", "MORTGAGE30US", "FEDFUNDS", "DGS10", "DGS2", "DRCCLACBS"].includes(seriesId)) {
    return `${value.toFixed(1)}%`;
  }
  if (["GDPC1", "GFDEBTN", "BOPGSTB", "TOTALSL"].includes(seriesId)) {
    return value >= 1000000 ? `$${(value / 1000000).toFixed(2)}T` : `$${(value / 1000).toFixed(1)}B`;
  }
  if (["MSPUS"].includes(seriesId)) {
    return `$${(value / 1000).toFixed(0)}K`;
  }
  if (["PAYEMS", "ICSA"].includes(seriesId)) {
    return value >= 1000 ? `${(value / 1000).toFixed(1)}M` : `${value.toFixed(0)}K`;
  }
  return value.toFixed(2);
}

function generateMarkdown(results: Map<string, SeriesResult>, gasPrice: { value: number; date: string } | null): string {
  const now = new Date().toISOString().replace('T', ' ').split('.')[0];

  let md = `# US Economic State Analysis

**Generated:** ${now}
**Data Period:** 10 years through present
**Sources:** Federal Reserve Economic Data (FRED), Energy Information Administration (EIA)

---

## Executive Summary

`;

  // Build summary based on key metrics
  const unrate = results.get("UNRATE");
  const cpi = results.get("CPIAUCSL");
  const gdp = results.get("GDPC1");
  const fedfunds = results.get("FEDFUNDS");

  if (unrate?.latest) {
    md += `- **Unemployment** at ${formatValue(unrate.latest.value, "UNRATE")} (${unrate.trends["1y"]?.direction || "→"} YoY)\n`;
  }
  if (cpi?.trends["1y"]) {
    md += `- **Inflation (CPI)** running ${cpi.trends["1y"].pctChange.toFixed(1)}% YoY\n`;
  }
  if (gdp?.trends["1y"]) {
    md += `- **Real GDP** ${gdp.trends["1y"].direction} ${Math.abs(gdp.trends["1y"].pctChange).toFixed(1)}% over past year\n`;
  }
  if (fedfunds?.latest) {
    md += `- **Fed Funds Rate** at ${formatValue(fedfunds.latest.value, "FEDFUNDS")}\n`;
  }
  if (gasPrice) {
    md += `- **Gas Prices** at $${gasPrice.value.toFixed(2)}/gallon (as of ${gasPrice.date})\n`;
  }

  md += `
---

## Current Snapshot

| Category | Metric | Current | 1Y Change | Trend |
|----------|--------|---------|-----------|-------|
`;

  // Snapshot table
  const categories = [
    { name: "Economy", ids: ["GDPC1", "A191RL1Q225SBEA"] },
    { name: "Inflation", ids: ["CPIAUCSL", "PCEPILFE"] },
    { name: "Employment", ids: ["UNRATE", "PAYEMS"] },
    { name: "Housing", ids: ["MSPUS", "MORTGAGE30US"] },
    { name: "Consumer", ids: ["UMCSENT", "PSAVERT"] },
    { name: "Markets", ids: ["FEDFUNDS", "DGS10"] },
  ];

  for (const cat of categories) {
    for (const id of cat.ids) {
      const r = results.get(id);
      if (r?.latest && r.trends["1y"]) {
        const changeStr = r.trends["1y"].pctChange >= 0
          ? `+${r.trends["1y"].pctChange.toFixed(1)}%`
          : `${r.trends["1y"].pctChange.toFixed(1)}%`;
        md += `| ${cat.name} | ${r.name} | ${formatValue(r.latest.value, id)} | ${changeStr} | ${r.trends["1y"].direction} |\n`;
      }
    }
  }

  md += `
---

## Detailed Trend Analysis

`;

  // Detailed analysis by category
  const categoryDetails = [
    { title: "Economic Output & Growth", ids: PRIORITY_SERIES.economic },
    { title: "Inflation & Prices", ids: PRIORITY_SERIES.inflation },
    { title: "Employment & Labor", ids: PRIORITY_SERIES.employment },
    { title: "Housing", ids: PRIORITY_SERIES.housing },
    { title: "Consumer & Personal Finance", ids: PRIORITY_SERIES.consumer },
    { title: "Financial Markets", ids: PRIORITY_SERIES.financial },
    { title: "Trade & International", ids: PRIORITY_SERIES.trade },
    { title: "Government & Fiscal", ids: PRIORITY_SERIES.fiscal },
  ];

  for (const cat of categoryDetails) {
    md += `### ${cat.title}\n\n`;
    md += `| Metric | Current | 10Y | 5Y | 2Y | 1Y |\n`;
    md += `|--------|---------|-----|----|----|----|\n`;

    for (const id of cat.ids) {
      const r = results.get(id);
      if (r?.latest) {
        const t10 = r.trends["10y"] ? `${r.trends["10y"].pctChange >= 0 ? "+" : ""}${r.trends["10y"].pctChange.toFixed(1)}%` : "—";
        const t5 = r.trends["5y"] ? `${r.trends["5y"].pctChange >= 0 ? "+" : ""}${r.trends["5y"].pctChange.toFixed(1)}%` : "—";
        const t2 = r.trends["2y"] ? `${r.trends["2y"].pctChange >= 0 ? "+" : ""}${r.trends["2y"].pctChange.toFixed(1)}%` : "—";
        const t1 = r.trends["1y"] ? `${r.trends["1y"].pctChange >= 0 ? "+" : ""}${r.trends["1y"].pctChange.toFixed(1)}%` : "—";
        md += `| ${r.name} | ${formatValue(r.latest.value, id)} | ${t10} | ${t5} | ${t2} | ${t1} |\n`;
      }
    }
    md += `\n`;
  }

  md += `---

## Cross-Metric Analysis

### Inflation-Employment Dynamics

`;

  if (unrate?.latest && cpi?.trends["1y"]) {
    md += `Current unemployment (${formatValue(unrate.latest.value, "UNRATE")}) with CPI change of ${cpi.trends["1y"].pctChange.toFixed(1)}% YoY suggests `;
    if (unrate.latest.value < 4.5 && cpi.trends["1y"].pctChange > 3) {
      md += `a tight labor market with persistent inflationary pressure.\n`;
    } else if (unrate.latest.value < 4.5 && cpi.trends["1y"].pctChange < 3) {
      md += `the economy is approaching a "soft landing" scenario.\n`;
    } else {
      md += `moderate labor market conditions.\n`;
    }
  }

  md += `
### Yield Curve Status

`;

  const dgs10 = results.get("DGS10");
  const dgs2 = results.get("DGS2");
  if (dgs10?.latest && dgs2?.latest) {
    const spread = dgs10.latest.value - dgs2.latest.value;
    md += `- 10Y Treasury: ${formatValue(dgs10.latest.value, "DGS10")}\n`;
    md += `- 2Y Treasury: ${formatValue(dgs2.latest.value, "DGS2")}\n`;
    md += `- Spread: ${spread.toFixed(2)}pp (${spread < 0 ? "INVERTED - recessionary signal" : "Normal"})\n`;
  }

  md += `
### Housing Affordability

`;

  const homePrice = results.get("MSPUS");
  const mortgage = results.get("MORTGAGE30US");
  if (homePrice?.latest && mortgage?.latest) {
    md += `With median home price at ${formatValue(homePrice.latest.value, "MSPUS")} and mortgage rates at ${formatValue(mortgage.latest.value, "MORTGAGE30US")}, `;
    if (homePrice.latest.value > 400000 && mortgage.latest.value > 6) {
      md += `housing affordability remains severely stressed.\n`;
    } else {
      md += `housing affordability is challenging but stabilizing.\n`;
    }
  }

  md += `
---

## Pattern Detection

### Historical Extremes

`;

  // Check for extremes
  const extremes: string[] = [];
  for (const [id, r] of results) {
    if (r.trends["10y"]) {
      if (Math.abs(r.trends["10y"].pctChange) > 50) {
        extremes.push(`- **${r.name}**: ${r.trends["10y"].pctChange > 0 ? "+" : ""}${r.trends["10y"].pctChange.toFixed(0)}% over 10 years (significant move)`);
      }
    }
  }

  if (extremes.length > 0) {
    md += extremes.join("\n") + "\n";
  } else {
    md += "No extreme outliers detected in current data.\n";
  }

  md += `
### Recent Momentum Shifts

`;

  // Look for acceleration/deceleration
  const shifts: string[] = [];
  for (const [id, r] of results) {
    if (r.trends["5y"] && r.trends["1y"]) {
      const fiveYrAnnual = r.trends["5y"].pctChange / 5;
      const oneYr = r.trends["1y"].pctChange;
      if (Math.abs(oneYr) > Math.abs(fiveYrAnnual) * 2 && Math.abs(oneYr) > 5) {
        shifts.push(`- **${r.name}**: Accelerating (1Y: ${oneYr.toFixed(1)}% vs 5Y avg: ${fiveYrAnnual.toFixed(1)}%/yr)`);
      } else if (Math.abs(oneYr) < Math.abs(fiveYrAnnual) / 2 && Math.abs(fiveYrAnnual) > 5) {
        shifts.push(`- **${r.name}**: Decelerating (1Y: ${oneYr.toFixed(1)}% vs 5Y avg: ${fiveYrAnnual.toFixed(1)}%/yr)`);
      }
    }
  }

  if (shifts.length > 0) {
    md += shifts.join("\n") + "\n";
  } else {
    md += "No significant momentum shifts detected.\n";
  }

  md += `
---

## Research Recommendations

### High Priority Investigations

1. **Labor Market Dynamics**: Examine the relationship between job openings, quit rate, and wage growth
2. **Inflation Persistence**: Analyze components of CPI to identify sticky inflation drivers
3. **Housing Market**: Investigate regional variations in home prices vs. mortgage rate sensitivity

### Risks to Monitor

1. **Credit Conditions**: Watch credit card delinquency and consumer credit growth rates
2. **Yield Curve**: Monitor 10Y-2Y spread for recession signals
3. **Consumer Sentiment**: Track sentiment vs. actual spending divergence

### Data Gaps

1. Add regional breakdowns for key metrics
2. Include leading economic indicators (LEI)
3. Add wage growth by sector data

---

## Sources

- **FRED (Federal Reserve Economic Data)**: Primary source for most indicators
- **EIA (Energy Information Administration)**: Gas and oil prices
- **Treasury FiscalData**: Federal debt and deficit data
- **BLS (Bureau of Labor Statistics)**: Employment statistics
- **Census Bureau**: Housing data

---

*Analysis generated by US-Metrics skill using Substrate US-Common-Metrics dataset*
`;

  return md;
}

// ============================================================================
// MAIN
// ============================================================================

async function main() {
  const { values } = parseArgs({
    args: Bun.argv.slice(2),
    options: {
      output: { type: "string" },
      help: { type: "boolean", short: "h" },
    },
    allowPositionals: true,
  });

  if (values.help) {
    console.log(`
GenerateAnalysis.ts - Generate US Economic State Analysis

Usage:
  bun run GenerateAnalysis.ts [--output=path]

Options:
  --output=PATH  Save to file instead of stdout
  -h, --help     Show this help

Environment:
  FRED_API_KEY   Required for FRED data
  EIA_API_KEY    Optional for gas prices
`);
    process.exit(0);
  }

  console.error("Fetching data from FRED API...");

  const results = new Map<string, SeriesResult>();

  // Fetch all priority series
  const allSeries = Object.values(PRIORITY_SERIES).flat();
  for (const id of allSeries) {
    console.error(`  Fetching ${id}...`);
    const result = await fetchFredSeries(id, 10);
    if (result) {
      results.set(id, result);
    }
    // Small delay to be nice to the API
    await new Promise(r => setTimeout(r, 100));
  }

  console.error("Fetching gas prices from EIA...");
  const gasPrice = await fetchEIAGasPrice();

  console.error("Generating analysis...");
  const markdown = generateMarkdown(results, gasPrice);

  if (values.output) {
    await Bun.write(values.output, markdown);
    console.error(`Analysis saved to ${values.output}`);
  } else {
    console.log(markdown);
  }
}

main();

#!/usr/bin/env bun
/**
 * update-substrate-metrics.ts
 *
 * Fetches current data from all sources (FRED, EIA, Treasury) and updates
 * the Substrate US-Common-Metrics dataset files.
 *
 * Usage:
 *   bun run update-substrate-metrics.ts [--dry-run]
 *
 * Environment:
 *   FRED_API_KEY - Required for most metrics
 *   EIA_API_KEY - Required for gas prices
 *
 * Output files:
 *   - ${PROJECTS_DIR}/Substrate/Data/US-Common-Metrics/US-Common-Metrics.md (updated values)
 *   - ${PROJECTS_DIR}/Substrate/Data/US-Common-Metrics/us-metrics-current.csv (current snapshot)
 *   - ${PROJECTS_DIR}/Substrate/Data/US-Common-Metrics/us-metrics-historical.csv (appended)
 */

import { parseArgs } from "util";
import { readFileSync, writeFileSync, existsSync, appendFileSync } from "fs";
import { join } from "path";

// ============================================================================
// CONFIGURATION
// ============================================================================

const SUBSTRATE_PATH = join(process.env.HOME || "", "Projects/Substrate/Data/US-Common-Metrics");
const FRED_API_KEY = process.env.FRED_API_KEY;
const EIA_API_KEY = process.env.EIA_API_KEY;

// All metrics with their configuration
interface MetricConfig {
  name: string;
  category: string;
  fredId?: string;
  source: string;
  format: "number" | "percent" | "currency" | "billions" | "trillions" | "thousands" | "millions" | "index";
  decimals?: number;
  special?: "eia-gas" | "eia-oil" | "treasury-debt" | "yoy-calc";
  baseId?: string; // For calculated metrics (YoY change)
}

const METRICS: Record<string, MetricConfig> = {
  // Economic Output & Growth
  "GDPC1": { name: "Real GDP", category: "Economic Output", fredId: "GDPC1", source: "BEA/FRED", format: "billions", decimals: 2 },
  "GDP": { name: "Nominal GDP", category: "Economic Output", fredId: "GDP", source: "BEA/FRED", format: "billions", decimals: 2 },
  "A191RL1Q225SBEA": { name: "GDP Growth Rate (QoQ)", category: "Economic Output", fredId: "A191RL1Q225SBEA", source: "BEA/FRED", format: "percent", decimals: 1 },
  "A191RO1Q156NBEA": { name: "GDP Growth Rate (YoY)", category: "Economic Output", fredId: "A191RO1Q156NBEA", source: "BEA/FRED", format: "percent", decimals: 1 },
  "INDPRO": { name: "Industrial Production", category: "Economic Output", fredId: "INDPRO", source: "Fed/FRED", format: "index", decimals: 2 },
  "TCU": { name: "Capacity Utilization", category: "Economic Output", fredId: "TCU", source: "Fed/FRED", format: "percent", decimals: 1 },
  "DGORDER": { name: "Durable Goods Orders", category: "Economic Output", fredId: "DGORDER", source: "Census/FRED", format: "millions", decimals: 0 },
  "RSAFS": { name: "Retail Sales", category: "Economic Output", fredId: "RSAFS", source: "Census/FRED", format: "billions", decimals: 1 },

  // Inflation & Prices
  "CPIAUCSL": { name: "CPI-U All Items", category: "Inflation", fredId: "CPIAUCSL", source: "BLS/FRED", format: "index", decimals: 3 },
  "CPILFESL": { name: "Core CPI", category: "Inflation", fredId: "CPILFESL", source: "BLS/FRED", format: "index", decimals: 3 },
  "PCEPI": { name: "PCE Price Index", category: "Inflation", fredId: "PCEPI", source: "BEA/FRED", format: "index", decimals: 3 },
  "PCEPILFE": { name: "Core PCE", category: "Inflation", fredId: "PCEPILFE", source: "BEA/FRED", format: "index", decimals: 3 },
  "PPIACO": { name: "Producer Price Index", category: "Inflation", fredId: "PPIACO", source: "BLS/FRED", format: "index", decimals: 2 },
  "DCOILWTICO": { name: "WTI Crude Oil", category: "Inflation", fredId: "DCOILWTICO", source: "EIA/FRED", format: "currency", decimals: 2 },
  "GAS_PRICE": { name: "Gas Price (Regular)", category: "Inflation", source: "EIA", format: "currency", decimals: 3, special: "eia-gas" },

  // Employment & Labor
  "UNRATE": { name: "Unemployment Rate (U-3)", category: "Employment", fredId: "UNRATE", source: "BLS/FRED", format: "percent", decimals: 1 },
  "U6RATE": { name: "Underemployment Rate (U-6)", category: "Employment", fredId: "U6RATE", source: "BLS/FRED", format: "percent", decimals: 1 },
  "PAYEMS": { name: "Nonfarm Payrolls", category: "Employment", fredId: "PAYEMS", source: "BLS/FRED", format: "thousands", decimals: 0 },
  "ICSA": { name: "Initial Jobless Claims", category: "Employment", fredId: "ICSA", source: "DOL/FRED", format: "thousands", decimals: 0 },
  "CCSA": { name: "Continuing Claims", category: "Employment", fredId: "CCSA", source: "DOL/FRED", format: "thousands", decimals: 0 },
  "JTSJOL": { name: "Job Openings (JOLTS)", category: "Employment", fredId: "JTSJOL", source: "BLS/FRED", format: "thousands", decimals: 0 },
  "JTSQUR": { name: "Quit Rate", category: "Employment", fredId: "JTSQUR", source: "BLS/FRED", format: "percent", decimals: 1 },
  "JTSHIR": { name: "Hire Rate", category: "Employment", fredId: "JTSHIR", source: "BLS/FRED", format: "percent", decimals: 1 },
  "CIVPART": { name: "Labor Force Participation", category: "Employment", fredId: "CIVPART", source: "BLS/FRED", format: "percent", decimals: 1 },
  "EMRATIO": { name: "Employment-Population Ratio", category: "Employment", fredId: "EMRATIO", source: "BLS/FRED", format: "percent", decimals: 1 },
  "CES0500000003": { name: "Average Hourly Earnings", category: "Employment", fredId: "CES0500000003", source: "BLS/FRED", format: "currency", decimals: 2 },
  "AWHAETP": { name: "Average Weekly Hours", category: "Employment", fredId: "AWHAETP", source: "BLS/FRED", format: "number", decimals: 1 },

  // Housing
  "MSPUS": { name: "Median Home Sales Price", category: "Housing", fredId: "MSPUS", source: "Census/FRED", format: "currency", decimals: 0 },
  "CSUSHPINSA": { name: "Case-Shiller Index", category: "Housing", fredId: "CSUSHPINSA", source: "S&P/FRED", format: "index", decimals: 2 },
  "EXHOSLUSM495S": { name: "Existing Home Sales", category: "Housing", fredId: "EXHOSLUSM495S", source: "NAR/FRED", format: "thousands", decimals: 0 },
  "HSN1F": { name: "New Home Sales", category: "Housing", fredId: "HSN1F", source: "Census/FRED", format: "thousands", decimals: 0 },
  "HOUST": { name: "Housing Starts", category: "Housing", fredId: "HOUST", source: "Census/FRED", format: "thousands", decimals: 0 },
  "PERMIT": { name: "Building Permits", category: "Housing", fredId: "PERMIT", source: "Census/FRED", format: "thousands", decimals: 0 },
  "MORTGAGE30US": { name: "30-Year Mortgage Rate", category: "Housing", fredId: "MORTGAGE30US", source: "Freddie Mac/FRED", format: "percent", decimals: 2 },
  "MORTGAGE15US": { name: "15-Year Mortgage Rate", category: "Housing", fredId: "MORTGAGE15US", source: "Freddie Mac/FRED", format: "percent", decimals: 2 },

  // Consumer & Personal Finance
  "UMCSENT": { name: "Consumer Sentiment", category: "Consumer", fredId: "UMCSENT", source: "UMich/FRED", format: "index", decimals: 1 },
  "PI": { name: "Personal Income", category: "Consumer", fredId: "PI", source: "BEA/FRED", format: "billions", decimals: 1 },
  "DSPI": { name: "Disposable Personal Income", category: "Consumer", fredId: "DSPI", source: "BEA/FRED", format: "billions", decimals: 1 },
  "PSAVERT": { name: "Personal Saving Rate", category: "Consumer", fredId: "PSAVERT", source: "BEA/FRED", format: "percent", decimals: 1 },
  "TOTALSL": { name: "Consumer Credit Outstanding", category: "Consumer", fredId: "TOTALSL", source: "Fed/FRED", format: "billions", decimals: 1 },
  "DRCCLACBS": { name: "Credit Card Delinquency", category: "Consumer", fredId: "DRCCLACBS", source: "Fed/FRED", format: "percent", decimals: 2 },
  "TDSP": { name: "Debt Service Ratio", category: "Consumer", fredId: "TDSP", source: "Fed/FRED", format: "percent", decimals: 2 },
  "TOTALSA": { name: "Auto Sales", category: "Consumer", fredId: "TOTALSA", source: "BEA/FRED", format: "millions", decimals: 2 },

  // Financial Markets
  "FEDFUNDS": { name: "Fed Funds Rate", category: "Financial", fredId: "FEDFUNDS", source: "Fed/FRED", format: "percent", decimals: 2 },
  "DFEDTARU": { name: "Fed Funds Target (Upper)", category: "Financial", fredId: "DFEDTARU", source: "Fed/FRED", format: "percent", decimals: 2 },
  "DGS10": { name: "10-Year Treasury", category: "Financial", fredId: "DGS10", source: "Treasury/FRED", format: "percent", decimals: 2 },
  "DGS2": { name: "2-Year Treasury", category: "Financial", fredId: "DGS2", source: "Treasury/FRED", format: "percent", decimals: 2 },
  "T10Y2Y": { name: "10Y-2Y Spread", category: "Financial", fredId: "T10Y2Y", source: "FRED", format: "percent", decimals: 2 },
  "DGS30": { name: "30-Year Treasury", category: "Financial", fredId: "DGS30", source: "Treasury/FRED", format: "percent", decimals: 2 },
  "DTB3": { name: "3-Month T-Bill", category: "Financial", fredId: "DTB3", source: "Treasury/FRED", format: "percent", decimals: 2 },
  "STLFSI4": { name: "Financial Stress Index", category: "Financial", fredId: "STLFSI4", source: "StL Fed/FRED", format: "index", decimals: 3 },
  "SP500": { name: "S&P 500", category: "Financial", fredId: "SP500", source: "S&P/FRED", format: "number", decimals: 2 },
  "VIXCLS": { name: "VIX", category: "Financial", fredId: "VIXCLS", source: "CBOE/FRED", format: "index", decimals: 2 },

  // Trade & International
  "BOPGSTB": { name: "Trade Balance", category: "Trade", fredId: "BOPGSTB", source: "Census/BEA/FRED", format: "billions", decimals: 1 },
  "BOPGEXP": { name: "Exports", category: "Trade", fredId: "BOPGEXP", source: "Census/FRED", format: "billions", decimals: 1 },
  "BOPGIMP": { name: "Imports", category: "Trade", fredId: "BOPGIMP", source: "Census/FRED", format: "billions", decimals: 1 },
  "DTWEXBGS": { name: "USD Index", category: "Trade", fredId: "DTWEXBGS", source: "Fed/FRED", format: "index", decimals: 2 },
  "DEXUSEU": { name: "USD/EUR", category: "Trade", fredId: "DEXUSEU", source: "Fed/FRED", format: "number", decimals: 4 },

  // Government & Fiscal
  "GFDEBTN": { name: "Federal Debt Total", category: "Fiscal", fredId: "GFDEBTN", source: "Treasury/FRED", format: "trillions", decimals: 3 },
  "GFDEGDQ188S": { name: "Debt-to-GDP Ratio", category: "Fiscal", fredId: "GFDEGDQ188S", source: "FRED", format: "percent", decimals: 1 },
  "FGRECPT": { name: "Federal Receipts", category: "Fiscal", fredId: "FGRECPT", source: "Treasury/FRED", format: "billions", decimals: 1 },
  "FGEXPND": { name: "Federal Expenditures", category: "Fiscal", fredId: "FGEXPND", source: "Treasury/FRED", format: "billions", decimals: 1 },
  "TREASURY_DEBT": { name: "Total Public Debt", category: "Fiscal", source: "Treasury", format: "trillions", decimals: 3, special: "treasury-debt" },

  // Demographics
  "POPTHM": { name: "US Population", category: "Demographics", fredId: "POPTHM", source: "Census/FRED", format: "millions", decimals: 1 },
  "SIPOVGINIUSA": { name: "GINI Index", category: "Demographics", fredId: "SIPOVGINIUSA", source: "Census/FRED", format: "number", decimals: 3 },
  "MEHOINUSA672N": { name: "Median Household Income", category: "Demographics", fredId: "MEHOINUSA672N", source: "Census/FRED", format: "currency", decimals: 0 },
};

// ============================================================================
// DATA FETCHING
// ============================================================================

interface FetchResult {
  id: string;
  name: string;
  value: number;
  formattedValue: string;
  period: string;
  updated: string;
  source: string;
}

async function fetchFredSeries(seriesId: string): Promise<{ value: number; date: string } | null> {
  if (!FRED_API_KEY) {
    console.error("FRED_API_KEY not set");
    return null;
  }

  const url = `https://api.stlouisfed.org/fred/series/observations?series_id=${seriesId}&api_key=${FRED_API_KEY}&file_type=json&sort_order=desc&limit=1`;

  try {
    const response = await fetch(url);
    if (!response.ok) return null;

    const data = await response.json();
    const obs = data.observations?.[0];
    if (!obs || obs.value === ".") return null;

    return { value: parseFloat(obs.value), date: obs.date };
  } catch (e) {
    console.error(`Error fetching FRED ${seriesId}:`, e);
    return null;
  }
}

async function fetchEIAGasPrice(): Promise<{ value: number; date: string } | null> {
  if (!EIA_API_KEY) {
    console.error("EIA_API_KEY not set - skipping gas prices");
    return null;
  }

  const url = `https://api.eia.gov/v2/petroleum/pri/gnd/data/?api_key=${EIA_API_KEY}&frequency=weekly&data[0]=value&facets[product][]=EPMR&facets[duoarea][]=NUS&sort[0][column]=period&sort[0][direction]=desc&length=1`;

  try {
    const response = await fetch(url);
    if (!response.ok) return null;

    const data = await response.json();
    const item = data.response?.data?.[0];
    if (!item) return null;

    return { value: parseFloat(item.value), date: item.period };
  } catch (e) {
    console.error("Error fetching EIA gas price:", e);
    return null;
  }
}

async function fetchTreasuryDebt(): Promise<{ value: number; date: string } | null> {
  const url = "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/debt_to_penny?sort=-record_date&page[size]=1";

  try {
    const response = await fetch(url);
    if (!response.ok) return null;

    const data = await response.json();
    const item = data.data?.[0];
    if (!item) return null;

    // Value is in dollars, convert to trillions
    const valueInTrillions = parseFloat(item.tot_pub_debt_out_amt) / 1e12;
    return { value: valueInTrillions, date: item.record_date };
  } catch (e) {
    console.error("Error fetching Treasury debt:", e);
    return null;
  }
}

function formatValue(value: number, config: MetricConfig): string {
  const decimals = config.decimals ?? 2;

  switch (config.format) {
    case "percent":
      return `${value.toFixed(decimals)}%`;
    case "currency":
      return `$${value.toLocaleString("en-US", { minimumFractionDigits: decimals, maximumFractionDigits: decimals })}`;
    case "billions":
      return `$${value.toFixed(decimals)}B`;
    case "trillions":
      return `$${value.toFixed(decimals)}T`;
    case "thousands":
      return `${value.toLocaleString("en-US", { maximumFractionDigits: 0 })}K`;
    case "millions":
      return `${value.toFixed(decimals)}M`;
    case "index":
      return value.toFixed(decimals);
    case "number":
    default:
      return value.toLocaleString("en-US", { minimumFractionDigits: decimals, maximumFractionDigits: decimals });
  }
}

async function fetchAllMetrics(): Promise<Map<string, FetchResult>> {
  const results = new Map<string, FetchResult>();
  const errors: string[] = [];

  console.log("Fetching metrics from APIs...\n");

  for (const [id, config] of Object.entries(METRICS)) {
    let data: { value: number; date: string } | null = null;

    // Handle special fetches
    if (config.special === "eia-gas") {
      console.log(`  [EIA] ${config.name}...`);
      data = await fetchEIAGasPrice();
    } else if (config.special === "treasury-debt") {
      console.log(`  [Treasury] ${config.name}...`);
      data = await fetchTreasuryDebt();
    } else if (config.fredId) {
      console.log(`  [FRED] ${config.name} (${config.fredId})...`);
      data = await fetchFredSeries(config.fredId);
    }

    if (data) {
      results.set(id, {
        id,
        name: config.name,
        value: data.value,
        formattedValue: formatValue(data.value, config),
        period: data.date,
        updated: new Date().toISOString().split("T")[0],
        source: config.source,
      });
      console.log(`    ✓ ${formatValue(data.value, config)} (${data.date})`);
    } else {
      errors.push(id);
      console.log(`    ✗ Failed`);
    }

    // Small delay to be nice to APIs
    await new Promise((r) => setTimeout(r, 100));
  }

  console.log(`\nFetched ${results.size}/${Object.keys(METRICS).length} metrics`);
  if (errors.length > 0) {
    console.log(`Failed: ${errors.join(", ")}`);
  }

  return results;
}

// ============================================================================
// FILE UPDATES
// ============================================================================

function updateMarkdownFile(results: Map<string, FetchResult>): string {
  const mdPath = join(SUBSTRATE_PATH, "US-Common-Metrics.md");
  let content = readFileSync(mdPath, "utf-8");

  // Update the "Last Updated" timestamp
  const now = new Date().toISOString().split("T")[0];
  content = content.replace(
    /\*\*Last Updated:\*\* .*/,
    `**Last Updated:** ${now}`
  );

  // Update Quick Reference Dashboard
  const dashboardUpdates: Record<string, { key: string; value: string; trend: string }> = {
    "Economy": { key: "A191RL1Q225SBEA", value: "--", trend: "--" },
    "Inflation": { key: "CPIAUCSL", value: "--", trend: "--" },
    "Employment": { key: "UNRATE", value: "--", trend: "--" },
    "Housing": { key: "MORTGAGE30US", value: "--", trend: "--" },
    "Markets": { key: "FEDFUNDS", value: "--", trend: "--" },
    "Consumer": { key: "UMCSENT", value: "--", trend: "--" },
    "Fiscal": { key: "TREASURY_DEBT", value: "--", trend: "--" },
    "Energy": { key: "GAS_PRICE", value: "--", trend: "--" },
  };

  for (const [cat, info] of Object.entries(dashboardUpdates)) {
    const result = results.get(info.key);
    if (result) {
      info.value = result.formattedValue;
      info.trend = "→"; // Would need historical data to calculate actual trend
    }
  }

  // Build updated dashboard
  const dashboardRegex = /(\| Category \| Key Metric \| Value \| Updated \| Trend \|\n\|[^\n]+\n)([\s\S]*?)(\n\*Values updated)/;
  const dashboardMatch = content.match(dashboardRegex);

  if (dashboardMatch) {
    let newDashboard = "";
    for (const [cat, info] of Object.entries(dashboardUpdates)) {
      const result = results.get(info.key);
      const metricName = result?.name || METRICS[info.key]?.name || info.key;
      const value = result?.formattedValue || "--";
      const updated = result?.updated || "--";
      newDashboard += `| ${cat} | ${metricName} | ${value} | ${updated} | ${info.trend} |\n`;
    }
    content = content.replace(dashboardRegex, `$1${newDashboard}$3`);
  }

  // Update individual metric tables
  // Match pattern: | MetricName | -- | Period | -- | Source | ID |
  // and update the Value and Updated columns

  for (const [id, result] of results) {
    const config = METRICS[id];
    if (!config?.fredId && !config?.special) continue;

    // Try to find and update the row in the markdown
    // Pattern matches: | Metric Name | value | period | updated | source | FRED_ID |
    const escapedName = config.name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
    const fredIdPattern = config.fredId || id;

    // Match the full table row
    const rowRegex = new RegExp(
      `(\\| ${escapedName}[^|]*\\| )--( \\|[^|]+\\| )--( \\|[^|]+\\| )(${fredIdPattern}[^|]* \\|)`,
      "g"
    );

    const replacement = `$1${result.formattedValue}$2${result.updated}$3$4`;
    content = content.replace(rowRegex, replacement);

    // Also try simpler pattern for metrics without exact name match
    const simpleRegex = new RegExp(
      `(\\|[^|]+\\| )--( \\|[^|]+\\| )--( \\|[^|]+\\| ${fredIdPattern} \\|)`,
      "g"
    );
    content = content.replace(simpleRegex, `$1${result.formattedValue}$2${result.updated}$3`);
  }

  return content;
}

function generateCurrentCSV(results: Map<string, FetchResult>): string {
  const lines = ["metric_id,metric_name,value,formatted_value,period,updated,source"];

  for (const [id, result] of results) {
    const line = [
      id,
      `"${result.name}"`,
      result.value,
      `"${result.formattedValue}"`,
      result.period,
      result.updated,
      `"${result.source}"`,
    ].join(",");
    lines.push(line);
  }

  return lines.join("\n") + "\n";
}

function generateHistoricalCSV(results: Map<string, FetchResult>): string {
  const timestamp = new Date().toISOString();
  const lines: string[] = [];

  for (const [id, result] of results) {
    lines.push(`${timestamp},${id},${result.value},${result.period}`);
  }

  return lines.join("\n") + "\n";
}

// ============================================================================
// MAIN
// ============================================================================

async function main() {
  const { values } = parseArgs({
    args: Bun.argv.slice(2),
    options: {
      "dry-run": { type: "boolean", default: false },
      help: { type: "boolean", short: "h", default: false },
    },
    allowPositionals: true,
  });

  if (values.help) {
    console.log(`
update-substrate-metrics.ts - Update Substrate US-Common-Metrics dataset

Usage:
  bun run update-substrate-metrics.ts [--dry-run]

Options:
  --dry-run    Fetch data but don't write files
  -h, --help   Show this help

Environment:
  FRED_API_KEY   Required for most metrics
  EIA_API_KEY    Required for gas prices

Output:
  ${PROJECTS_DIR}/Substrate/Data/US-Common-Metrics/
    - US-Common-Metrics.md (updated)
    - us-metrics-current.csv (current snapshot)
    - us-metrics-historical.csv (appended)
`);
    process.exit(0);
  }

  // Check API keys
  if (!FRED_API_KEY) {
    console.error("Error: FRED_API_KEY environment variable not set");
    console.error("Get your free key at: https://fred.stlouisfed.org/docs/api/api_key.html");
    process.exit(1);
  }

  // Verify Substrate path exists
  if (!existsSync(SUBSTRATE_PATH)) {
    console.error(`Error: Substrate path not found: ${SUBSTRATE_PATH}`);
    process.exit(1);
  }

  console.log("=".repeat(60));
  console.log("US-Common-Metrics Update");
  console.log("=".repeat(60));
  console.log(`Substrate path: ${SUBSTRATE_PATH}`);
  console.log(`Timestamp: ${new Date().toISOString()}`);
  console.log("");

  // Fetch all metrics
  const results = await fetchAllMetrics();

  if (results.size === 0) {
    console.error("No metrics fetched successfully. Aborting.");
    process.exit(1);
  }

  // Generate updates
  console.log("\nGenerating file updates...");
  const updatedMd = updateMarkdownFile(results);
  const currentCsv = generateCurrentCSV(results);
  const historicalCsv = generateHistoricalCSV(results);

  if (values["dry-run"]) {
    console.log("\n[DRY RUN] Would write:");
    console.log(`  - US-Common-Metrics.md (${updatedMd.length} bytes)`);
    console.log(`  - us-metrics-current.csv (${currentCsv.length} bytes)`);
    console.log(`  - us-metrics-historical.csv (append ${historicalCsv.length} bytes)`);
    console.log("\nSample CSV output:");
    console.log(currentCsv.split("\n").slice(0, 5).join("\n"));
    process.exit(0);
  }

  // Write files
  const mdPath = join(SUBSTRATE_PATH, "US-Common-Metrics.md");
  const currentCsvPath = join(SUBSTRATE_PATH, "us-metrics-current.csv");
  const historicalCsvPath = join(SUBSTRATE_PATH, "us-metrics-historical.csv");

  console.log("\nWriting files...");

  writeFileSync(mdPath, updatedMd);
  console.log(`  ✓ ${mdPath}`);

  writeFileSync(currentCsvPath, currentCsv);
  console.log(`  ✓ ${currentCsvPath}`);

  // Append to historical (create header if new file)
  if (!existsSync(historicalCsvPath)) {
    writeFileSync(historicalCsvPath, "fetch_timestamp,metric_id,value,period\n");
  }
  appendFileSync(historicalCsvPath, historicalCsv);
  console.log(`  ✓ ${historicalCsvPath} (appended)`);

  console.log("\n" + "=".repeat(60));
  console.log(`Update complete. ${results.size} metrics updated.`);
  console.log("=".repeat(60));
}

main();