Market Top Detector

#!/usr/bin/env python3
"""
Breadth CSV Client - Fetch 200DMA breadth from TraderMonty's public CSV.

Lightweight client for auto-fetching S&P 500 breadth data (% above 200DMA)
from the market-breadth-analysis GitHub Pages CSV. No API key required.

Data source:
  https://tradermonty.github.io/market-breadth-analysis/market_breadth_data.csv
"""

import csv
import io
from datetime import datetime
from typing import Optional

import requests

DEFAULT_DETAIL_URL = "https://tradermonty.github.io/market-breadth-analysis/market_breadth_data.csv"

TIMEOUT = 30


def fetch_breadth_200dma(url: str = DEFAULT_DETAIL_URL) -> Optional[dict]:
    """
    Fetch the latest 200DMA breadth value from TraderMonty CSV.

    Returns:
        Dict with keys: value (0-100%), date (str), is_fresh (bool),
        days_old (int), source (str).
        None if fetch fails or CSV is empty.
    """
    rows = _fetch_detail_csv(url)
    if not rows:
        return None

    latest = rows[-1]
    raw_value = latest["Breadth_Index_Raw"]
    value = round(raw_value * 100, 2)
    date_str = latest["Date"]

    freshness = _check_data_freshness(date_str)

    return {
        "value": value,
        "date": date_str,
        "is_fresh": freshness["is_fresh"],
        "days_old": freshness["days_old"],
        "source": "TraderMonty CSV",
    }


def _fetch_detail_csv(url: str) -> list:
    """Fetch and parse the detail CSV. Returns rows sorted by date ascending."""
    try:
        resp = requests.get(url, timeout=TIMEOUT)
        resp.raise_for_status()
    except (requests.RequestException, OSError, ValueError):
        return []

    reader = csv.DictReader(io.StringIO(resp.text))
    rows = []

    for raw_row in reader:
        try:
            row = _parse_detail_row(raw_row)
            rows.append(row)
        except (ValueError, KeyError):
            continue

    if not rows:
        return []

    rows.sort(key=lambda r: r["Date"])
    return rows


def _check_data_freshness(date_str: str, max_stale_days: int = 5) -> dict:
    """Check if data date is within acceptable freshness window.

    Uses business days so Friday data remains fresh on Monday.
    Future dates return is_fresh=False, days_old=None.
    """
    from utils import count_business_days

    try:
        data_date = datetime.strptime(date_str, "%Y-%m-%d").date()
    except ValueError:
        return {"is_fresh": False, "days_old": None}

    today = datetime.now().date()

    # Reject future dates
    if data_date > today:
        return {"is_fresh": False, "days_old": None}

    biz_days = count_business_days(data_date, today)
    return {
        "is_fresh": biz_days <= max_stale_days,
        "days_old": biz_days,
    }


def _parse_detail_row(raw: dict) -> dict:
    """Convert a raw CSV row dict to properly typed values."""
    return {
        "Date": raw["Date"].strip(),
        "S&P500_Price": float(raw["S&P500_Price"]),
        "Breadth_Index_Raw": float(raw["Breadth_Index_Raw"]),
        "Breadth_Index_200MA": float(raw["Breadth_Index_200MA"]),
        "Breadth_Index_8MA": float(raw["Breadth_Index_8MA"]),
        "Breadth_200MA_Trend": int(raw["Breadth_200MA_Trend"]),
        "Bearish_Signal": _parse_bool(raw["Bearish_Signal"]),
        "Is_Peak": _parse_bool(raw["Is_Peak"]),
        "Is_Trough": _parse_bool(raw["Is_Trough"]),
        "Is_Trough_8MA_Below_04": _parse_bool(raw["Is_Trough_8MA_Below_04"]),
    }


def _parse_bool(val: str) -> bool:
    """Parse boolean from CSV string."""
    return val.strip().lower() in ("true", "1", "yes")

"""Market Top Detector Calculator Modules"""

from .breadth_calculator import calculate_breadth_divergence
from .defensive_rotation_calculator import calculate_defensive_rotation
from .distribution_day_calculator import calculate_distribution_days
from .index_technical_calculator import calculate_index_technical
from .leading_stock_calculator import calculate_leading_stock_health
from .math_utils import calc_ema, calc_sma
from .sentiment_calculator import calculate_sentiment

__all__ = [
    "calculate_distribution_days",
    "calculate_leading_stock_health",
    "calculate_defensive_rotation",
    "calculate_breadth_divergence",
    "calculate_index_technical",
    "calculate_sentiment",
    "calc_ema",
    "calc_sma",
]

#!/usr/bin/env python3
"""
Component 4: Market Breadth Divergence (Weight: 15%)

Data source: CLI arguments (collected via WebSearch before script execution)
- breadth_200dma: % of S&P 500 stocks above 200DMA
- breadth_50dma: % of S&P 500 stocks above 50DMA

Scoring logic:
  When index is near highs (within -5% of 52-week high):
    200DMA_above < 40% -> 100 (Critical divergence)
    200DMA_above < 50% -> 80
    200DMA_above < 60% -> 60
    200DMA_above < 70% -> 35
    200DMA_above >= 70% -> 10

  When index is NOT near highs (> -5%):
    Score is halved (breadth divergence less meaningful when index is correcting)

  50DMA breadth is used as a supplementary signal.
"""

from typing import Optional


def calculate_breadth_divergence(
    breadth_200dma: Optional[float],
    breadth_50dma: Optional[float],
    index_distance_from_high_pct: float,
) -> dict:
    """
    Calculate market breadth divergence score.

    Args:
        breadth_200dma: % of S&P 500 stocks above 200DMA (0-100)
        breadth_50dma: % of S&P 500 stocks above 50DMA (0-100)
        index_distance_from_high_pct: S&P 500 distance from 52-week high (negative %)

    Returns:
        Dict with score (0-100), signal, details
    """
    if breadth_200dma is None:
        return {
            "score": 50,
            "signal": "NO DATA: Breadth data not provided (neutral default)",
            "data_available": False,
            "breadth_200dma": None,
            "breadth_50dma": breadth_50dma,
            "index_near_highs": False,
            "divergence_detected": False,
        }

    index_near_highs = index_distance_from_high_pct >= -5.0

    # Primary score based on 200DMA breadth
    raw_score = _score_200dma_breadth(breadth_200dma)

    # Halve score if index is NOT near highs
    if not index_near_highs:
        raw_score = raw_score * 0.5

    # 50DMA breadth supplementary adjustment
    if breadth_50dma is not None:
        if breadth_50dma < 30 and index_near_highs:
            raw_score = min(100, raw_score + 10)  # Extra warning
        elif breadth_50dma > 70:
            raw_score = max(0, raw_score - 5)  # Slight relief

    score = round(min(100, max(0, raw_score)))

    # Detect divergence
    divergence_detected = index_near_highs and breadth_200dma < 60

    if score >= 80:
        signal = "CRITICAL: Severe breadth divergence - index at highs but breadth collapsing"
    elif score >= 60:
        signal = "WARNING: Significant breadth divergence"
    elif score >= 35:
        signal = "CAUTION: Mild breadth weakness"
    elif divergence_detected:
        signal = "WATCH: Some breadth deterioration despite index strength"
    else:
        signal = "HEALTHY: Breadth supports index level"

    return {
        "score": score,
        "signal": signal,
        "data_available": True,
        "breadth_200dma": breadth_200dma,
        "breadth_50dma": breadth_50dma,
        "index_near_highs": index_near_highs,
        "index_distance_from_high_pct": round(index_distance_from_high_pct, 1),
        "divergence_detected": divergence_detected,
    }


def _score_200dma_breadth(breadth_200dma: float) -> int:
    """Score based on % of stocks above 200DMA.

    Calibrated thresholds (linearly interpolated):
      < 40%  -> 100 (Critical divergence)
      40-50% -> 100 to 55 (Severe)
      50-60% -> 55 to 30 (Moderate)
      60-70% -> 30 to 5  (Mild - still relatively healthy)
      >= 70% -> 5         (Healthy breadth)

    Calibration note: 62.26% should score ~24 (healthy/green signal)
    """
    if breadth_200dma < 40:
        return 100
    elif breadth_200dma < 50:
        return round(100 - (breadth_200dma - 40) / 10 * 45)
    elif breadth_200dma < 60:
        return round(55 - (breadth_200dma - 50) / 10 * 25)
    elif breadth_200dma < 70:
        return round(30 - (breadth_200dma - 60) / 10 * 25)
    else:
        return 5

#!/usr/bin/env python3
"""
Component 3: Defensive Sector Rotation (Weight: 15%)

Compares defensive ETF performance vs offensive/growth ETF performance
using multi-period weighted analysis: 10d(0.2) + 20d(0.5) + 40d(0.3).

Defensive: XLU, XLP, XLV, VNQ
Offensive:  XLK, XLC, XLY, QQQ

Scoring (defensive_return - offensive_return):
  +5.0% or more -> 100 (Strong rotation into defensives)
  +3.0% or more -> 80
  +1.5% or more -> 60
  +0.5% or more -> 40
  +0.0% or more -> 20
  Negative       ->  0 (Growth leading = healthy)

Multi-period confirmation:
  All periods defensive -> "confirmed" (max 100)
  Some periods only     -> "unconfirmed" (capped at 80)
"""

from typing import Optional

DEFENSIVE_ETFS = ["XLU", "XLP", "XLV", "VNQ"]
OFFENSIVE_ETFS = ["XLK", "XLC", "XLY", "QQQ"]

MULTI_PERIOD_WEIGHTS = {10: 0.2, 20: 0.5, 40: 0.3}


def _calc_return(symbol_hist: list[dict], days: int) -> Optional[float]:
    if not symbol_hist or len(symbol_hist) < days + 1:
        return None
    recent = symbol_hist[0].get("close", symbol_hist[0].get("adjClose", 0))
    past = symbol_hist[days].get("close", symbol_hist[days].get("adjClose", 0))
    if past == 0:
        return None
    return (recent - past) / past * 100


def _compute_period_rotation(historical: dict[str, list[dict]], lookback: int) -> Optional[dict]:
    """Compute defensive vs offensive rotation for a single period."""
    def_returns = []
    off_returns = []

    for symbol in DEFENSIVE_ETFS:
        hist = historical.get(symbol, [])
        ret = _calc_return(hist, lookback)
        if ret is not None:
            def_returns.append(ret)

    for symbol in OFFENSIVE_ETFS:
        hist = historical.get(symbol, [])
        ret = _calc_return(hist, lookback)
        if ret is not None:
            off_returns.append(ret)

    if not def_returns or not off_returns:
        return None

    def_avg = sum(def_returns) / len(def_returns)
    off_avg = sum(off_returns) / len(off_returns)
    relative = def_avg - off_avg
    score = _score_rotation(relative)

    return {
        "lookback": lookback,
        "defensive_avg": round(def_avg, 2),
        "offensive_avg": round(off_avg, 2),
        "relative": round(relative, 2),
        "score": score,
        "defensive_leading": relative > 0,
    }


def calculate_defensive_rotation(historical: dict[str, list[dict]], lookback: int = 20) -> dict:
    """
    Calculate defensive vs offensive sector rotation score using multi-period analysis.

    Args:
        historical: Dict of symbol -> list of daily OHLCV (most recent first, 50+ days)
        lookback: Primary lookback period (default 20, used for backward compat)

    Returns:
        Dict with score (0-100), relative_performance, multi_period details
    """
    # Track fetch success
    all_etfs = DEFENSIVE_ETFS + OFFENSIVE_ETFS
    total_attempted = len(all_etfs)
    fetch_successes = 0

    def_details = {}
    off_details = {}
    for symbol in DEFENSIVE_ETFS:
        hist = historical.get(symbol, [])
        ret = _calc_return(hist, lookback)
        if ret is not None:
            def_details[symbol] = round(ret, 2)
            fetch_successes += 1

    for symbol in OFFENSIVE_ETFS:
        hist = historical.get(symbol, [])
        ret = _calc_return(hist, lookback)
        if ret is not None:
            off_details[symbol] = round(ret, 2)
            fetch_successes += 1

    fetch_success_rate = fetch_successes / total_attempted if total_attempted > 0 else 0.0

    if not def_details or not off_details:
        return {
            "score": 50,
            "signal": "INSUFFICIENT DATA (neutral default)",
            "data_available": False,
            "relative_performance": 0,
            "defensive_avg_return": 0,
            "offensive_avg_return": 0,
            "defensive_details": def_details,
            "offensive_details": off_details,
            "lookback_days": lookback,
            "fetch_success_rate": round(fetch_success_rate, 2),
        }

    # Multi-period analysis
    period_results = {}
    weighted_score = 0.0
    total_weight = 0.0
    all_defensive = True
    periods_computed = 0

    for period, weight in MULTI_PERIOD_WEIGHTS.items():
        result = _compute_period_rotation(historical, period)
        if result is not None:
            period_results[period] = result
            weighted_score += result["score"] * weight
            total_weight += weight
            periods_computed += 1
            if not result["defensive_leading"]:
                all_defensive = False
        else:
            all_defensive = False

    # Fall back to single-period if multi-period unavailable
    if total_weight > 0:
        raw_score = weighted_score / total_weight
    else:
        # Fallback: single period
        def_avg = sum(def_details.values()) / len(def_details)
        off_avg = sum(off_details.values()) / len(off_details)
        relative = def_avg - off_avg
        raw_score = _score_rotation(relative)

    # Multi-period confirmation
    if periods_computed >= 2 and all_defensive:
        confirmation = "confirmed"
        score = round(min(100, raw_score))
    elif periods_computed >= 2:
        confirmation = "unconfirmed"
        score = round(min(80, raw_score))
    else:
        confirmation = "single_period"
        score = round(min(100, raw_score))

    score = max(0, score)

    # Primary period relative performance (for backward compat)
    primary = period_results.get(lookback)
    if primary:
        relative = primary["relative"]
        def_avg = primary["defensive_avg"]
        off_avg = primary["offensive_avg"]
    else:
        def_avg = sum(def_details.values()) / len(def_details)
        off_avg = sum(off_details.values()) / len(off_details)
        relative = def_avg - off_avg

    if score >= 80:
        signal = "CRITICAL: Strong defensive rotation"
    elif score >= 60:
        signal = "WARNING: Defensive outperformance"
    elif score >= 40:
        signal = "CAUTION: Mild defensive rotation"
    elif score >= 20:
        signal = "MIXED: Slight defensive tilt"
    else:
        signal = "HEALTHY: Growth leading"

    return {
        "score": score,
        "signal": signal,
        "data_available": fetch_success_rate >= 0.75,
        "relative_performance": round(relative, 2),
        "defensive_avg_return": round(def_avg, 2),
        "offensive_avg_return": round(off_avg, 2),
        "defensive_details": def_details,
        "offensive_details": off_details,
        "lookback_days": lookback,
        "fetch_success_rate": round(fetch_success_rate, 2),
        "confirmation": confirmation,
        "multi_period": period_results,
    }


def _score_rotation(relative: float) -> int:
    """Convert relative performance (defensive - offensive) to 0-100 score"""
    if relative >= 5.0:
        return 100
    elif relative >= 3.0:
        return round(80 + (relative - 3.0) / 2.0 * 20)
    elif relative >= 1.5:
        return round(60 + (relative - 1.5) / 1.5 * 20)
    elif relative >= 0.5:
        return round(40 + (relative - 0.5) / 1.0 * 20)
    elif relative >= 0.0:
        return round(20 + relative / 0.5 * 20)
    else:
        if relative >= -2.0:
            return round(max(0, 20 + relative / 2.0 * 20))
        return 0

#!/usr/bin/env python3
"""
Component 1: Distribution Day Count (Weight: 25%)

O'Neil's Distribution Day Rules:
- Distribution Day: Index drops >= 0.2% on higher volume than previous day
- Stalling Day: Volume increases but price gain < 0.1% (half weight)
- Days expire after 25 trading days
- Count the higher of S&P 500 or NASDAQ

Scoring:
  6+ distribution days -> 100 (Critical)
  5 days -> 90
  4 days -> 75  (O'Neil's warning threshold)
  3 days -> 55
  2 days -> 30
  1 day  -> 15
  0 days -> 0
"""


def calculate_distribution_days(sp500_history: list[dict], nasdaq_history: list[dict]) -> dict:
    """
    Calculate distribution day count for S&P 500 and NASDAQ.

    Args:
        sp500_history: List of daily OHLCV dicts (most recent first), at least 30 days
        nasdaq_history: List of daily OHLCV dicts (most recent first), at least 30 days

    Returns:
        Dict with score (0-100), distribution_days, stalling_days, details
    """
    sp500_result = _count_distribution_days(sp500_history, "S&P 500")
    nasdaq_result = _count_distribution_days(nasdaq_history, "NASDAQ")

    # Use the higher (worse) effective count
    sp500_effective = sp500_result["distribution_days"] + 0.5 * sp500_result["stalling_days"]
    nasdaq_effective = nasdaq_result["distribution_days"] + 0.5 * nasdaq_result["stalling_days"]

    if sp500_effective >= nasdaq_effective:
        primary = sp500_result
        primary_name = "S&P 500"
    else:
        primary = nasdaq_result
        primary_name = "NASDAQ"

    effective_count = max(sp500_effective, nasdaq_effective)
    raw_score = _score_distribution_days(effective_count)

    # Clustering analysis: check if distribution days are concentrated recently
    primary_details = primary["details"]
    clustering = _calculate_clustering_factor(primary_details)
    clustering_applied = effective_count >= 2 and clustering["factor"] > 0.5
    if clustering_applied:
        score = min(100, round(raw_score * 1.15))
    else:
        score = raw_score

    # Build signal description
    if effective_count >= 5:
        signal = "CRITICAL: Heavy distribution detected"
    elif effective_count >= 4:
        signal = "WARNING: O'Neil's threshold reached"
    elif effective_count >= 3:
        signal = "CAUTION: Moderate distribution"
    elif effective_count >= 1:
        signal = "MINOR: Some distribution present"
    else:
        signal = "CLEAR: No distribution"

    return {
        "score": score,
        "raw_score": raw_score,
        "effective_count": effective_count,
        "signal": signal,
        "clustering": clustering,
        "clustering_applied": clustering_applied,
        "primary_index": primary_name,
        "sp500": {
            "distribution_days": sp500_result["distribution_days"],
            "stalling_days": sp500_result["stalling_days"],
            "effective_count": sp500_effective,
            "details": sp500_result["details"],
        },
        "nasdaq": {
            "distribution_days": nasdaq_result["distribution_days"],
            "stalling_days": nasdaq_result["stalling_days"],
            "effective_count": nasdaq_effective,
            "details": nasdaq_result["details"],
        },
    }


def _count_distribution_days(history: list[dict], index_name: str) -> dict:
    """Count distribution and stalling days in the last 25 trading days"""
    if not history or len(history) < 2:
        return {"distribution_days": 0, "stalling_days": 0, "details": []}

    # We need at least 26 days to check 25 days of change
    # history[0] = most recent, history[1] = day before, etc.
    window = min(25, len(history) - 1)  # 25 trading day window

    distribution_days = 0
    stalling_days = 0
    details = []

    for i in range(window):
        today = history[i]
        yesterday = history[i + 1]

        today_close = today.get("close", today.get("adjClose", 0))
        yesterday_close = yesterday.get("close", yesterday.get("adjClose", 0))
        today_volume = today.get("volume", 0)
        yesterday_volume = yesterday.get("volume", 0)

        if yesterday_close == 0 or yesterday_volume == 0:
            continue

        pct_change = (today_close - yesterday_close) / yesterday_close * 100
        volume_increase = today_volume > yesterday_volume

        date = today.get("date", f"day-{i}")

        # Distribution day: price drops >= 0.2% AND volume increases
        if pct_change <= -0.2 and volume_increase:
            distribution_days += 1
            details.append(
                {
                    "date": date,
                    "type": "distribution",
                    "pct_change": round(pct_change, 2),
                    "volume_change": round((today_volume / yesterday_volume - 1) * 100, 1),
                    "window_index": i,
                }
            )

        # Stalling day: volume increases but price gain < 0.1%
        elif volume_increase and 0 <= pct_change < 0.1:
            stalling_days += 1
            details.append(
                {
                    "date": date,
                    "type": "stalling",
                    "pct_change": round(pct_change, 2),
                    "volume_change": round((today_volume / yesterday_volume - 1) * 100, 1),
                    "window_index": i,
                }
            )

    return {
        "distribution_days": distribution_days,
        "stalling_days": stalling_days,
        "details": details,
    }


def _calculate_clustering_factor(details: list) -> dict:
    """
    Calculate how clustered distribution/stalling events are in recent days.

    Looks at window_index of events: index 0-4 = most recent 5 days.
    Returns factor = recent_count / total_count (0.0-1.0).
    """
    if not details:
        return {"factor": 0.0, "recent_count": 0, "total_count": 0}

    total_count = len(details)
    recent_count = sum(1 for d in details if d.get("window_index", 99) < 5)
    factor = recent_count / total_count if total_count > 0 else 0.0

    return {
        "factor": round(factor, 2),
        "recent_count": recent_count,
        "total_count": total_count,
    }


def _score_distribution_days(effective_count: float) -> int:
    """Convert effective distribution day count to 0-100 score"""
    if effective_count >= 6:
        return 100
    elif effective_count >= 5:
        return 90
    elif effective_count >= 4:
        return 75
    elif effective_count >= 3:
        return 55
    elif effective_count >= 2:
        return 30
    elif effective_count >= 1:
        return 15
    else:
        return 0

#!/usr/bin/env python3
"""
Component 5: Index Technical Condition (Weight: 15%)

Evaluates technical health of S&P 500 and NASDAQ.
Uses shared historical data from Component 1 (no additional API calls).

Checkpoints (per index, S&P 500 + NASDAQ):
1. Price < 21 EMA (short-term weakness)        -> +8pt
2. Price < 50 EMA (medium-term breakdown)       -> +12pt
3. 21 EMA < 50 EMA (MA bearish crossover)       -> +10pt
4. Price < 200 SMA (long-term trend breakdown)   -> +15pt
5. Failed rally pattern                          -> +10pt
6. Lower highs pattern                           -> +10pt
7. Gap down on volume                            -> +10pt

Max per index: ~75 points (not all conditions fire at once in early stages)
Final: average of both indices, scaled to 0-100
"""

from typing import Optional

from calculators.math_utils import calc_ema


def calculate_index_technical(
    sp500_history: list[dict],
    nasdaq_history: list[dict],
    sp500_quote: Optional[dict] = None,
    nasdaq_quote: Optional[dict] = None,
) -> dict:
    """
    Calculate index technical condition score.

    Args:
        sp500_history: S&P 500 daily OHLCV (most recent first, 250+ days preferred)
        nasdaq_history: NASDAQ daily OHLCV (most recent first, 250+ days preferred)
        sp500_quote: Current S&P 500 quote (optional, for real-time price)
        nasdaq_quote: Current NASDAQ/QQQ quote (optional)

    Returns:
        Dict with score (0-100), sp500_details, nasdaq_details, signal
    """
    sp500_result = _evaluate_index("S&P 500", sp500_history, sp500_quote)
    nasdaq_result = _evaluate_index("NASDAQ", nasdaq_history, nasdaq_quote)

    # Average only indices with available data
    scores = []
    if sp500_result.get("data_available", False):
        scores.append(sp500_result["raw_score"])
    if nasdaq_result.get("data_available", False):
        scores.append(nasdaq_result["raw_score"])

    if not scores:
        final_score = 50  # Neutral when no data
        signal = "NO DATA: Index data unavailable"
        return {
            "score": final_score,
            "signal": signal,
            "sp500": sp500_result,
            "nasdaq": nasdaq_result,
            "data_available": False,
        }

    avg_score = sum(scores) / len(scores)
    final_score = round(min(100, max(0, avg_score)))

    if final_score >= 70:
        signal = "CRITICAL: Major technical breakdown"
    elif final_score >= 50:
        signal = "WARNING: Significant technical deterioration"
    elif final_score >= 35:
        signal = "CAUTION: Short-term weakness detected"
    elif final_score >= 15:
        signal = "MIXED: Minor technical concerns"
    else:
        signal = "HEALTHY: Technical structure intact"

    return {
        "score": final_score,
        "signal": signal,
        "sp500": sp500_result,
        "nasdaq": nasdaq_result,
        "data_available": True,
    }


def _evaluate_index(name: str, history: list[dict], quote: Optional[dict] = None) -> dict:
    """Evaluate a single index's technical condition"""
    if not history or len(history) < 21:
        return {"raw_score": 0, "flags": ["Insufficient data"], "mas": {}, "data_available": False}

    closes = [d.get("close", d.get("adjClose", 0)) for d in history]
    highs = [d.get("high", d.get("close", 0)) for d in history]
    [d.get("low", d.get("close", 0)) for d in history]
    volumes = [d.get("volume", 0) for d in history]

    # Current price (from quote or most recent close)
    if quote:
        price = quote.get("price", closes[0])
    else:
        price = closes[0]

    score = 0
    flags = []
    mas = {}

    # Calculate moving averages
    ema21 = calc_ema(closes, 21)
    mas["ema21"] = round(ema21, 2)

    if len(closes) >= 50:
        ema50 = calc_ema(closes, 50)
        mas["ema50"] = round(ema50, 2)
    else:
        ema50 = None

    if len(closes) >= 200:
        sma200 = sum(closes[:200]) / 200
        mas["sma200"] = round(sma200, 2)
    else:
        sma200 = None

    # Check 1: Price < 21 EMA (short-term weakness)
    if price < ema21:
        score += 8
        pct_below = (price - ema21) / ema21 * 100
        flags.append(f"Below 21 EMA ({pct_below:+.1f}%)")

    # Check 2: Price < 50 EMA (medium-term breakdown)
    if ema50 and price < ema50:
        score += 12
        pct_below = (price - ema50) / ema50 * 100
        flags.append(f"Below 50 EMA ({pct_below:+.1f}%)")

    # Check 3: 21 EMA < 50 EMA (bearish crossover)
    if ema50 and ema21 < ema50:
        score += 10
        flags.append("21 EMA < 50 EMA (bearish crossover)")

    # Check 4: Price < 200 SMA (long-term trend breakdown)
    if sma200 and price < sma200:
        score += 15
        pct_below = (price - sma200) / sma200 * 100
        flags.append(f"Below 200 SMA ({pct_below:+.1f}%)")

    # Check 5: Failed rally pattern
    if _detect_failed_rally(closes, volumes):
        score += 10
        flags.append("Failed rally pattern detected")

    # Check 6: Lower highs pattern (20 day)
    if _detect_lower_highs(highs, lookback=20):
        score += 10
        flags.append("Lower highs pattern (20 day)")

    # Check 7: Gap down on volume
    if _detect_gap_down(history):
        score += 10
        flags.append("Recent gap down on volume")

    return {
        "raw_score": min(100, score),
        "price": round(price, 2),
        "flags": flags,
        "mas": mas,
        "data_available": True,
    }


def _detect_failed_rally(closes: list[float], volumes: list[int], lookback: int = 15) -> bool:
    """
    Detect failed rally: price bounces then fails to make new high
    within recent lookback period.
    """
    if len(closes) < lookback:
        return False

    recent = closes[:lookback]
    # Find the peak in recent data
    peak_idx = recent.index(max(recent))

    # Failed rally: peak occurred 3-10 days ago, current price is below that peak
    if 3 <= peak_idx <= 10:
        # Price fell from peak
        drop = (recent[0] - recent[peak_idx]) / recent[peak_idx] * 100
        if drop < -2.0:  # Dropped >2% from recent peak
            return True
    return False


def _detect_lower_highs(highs: list[float], lookback: int = 20) -> bool:
    """Detect lower highs pattern in price action"""
    if len(highs) < lookback:
        return False

    recent_highs = highs[:lookback]

    # Find swing highs (local maxima)
    swing_highs = []
    for i in range(1, len(recent_highs) - 1):
        if recent_highs[i] > recent_highs[i - 1] and recent_highs[i] > recent_highs[i + 1]:
            swing_highs.append(recent_highs[i])

    if len(swing_highs) < 2:
        return False

    # Most recent swing high lower than previous
    return swing_highs[0] < swing_highs[1]


def _detect_gap_down(history: list[dict], lookback: int = 5) -> bool:
    """Detect gap down on increased volume in recent sessions"""
    if len(history) < lookback + 1:
        return False

    for i in range(lookback):
        today = history[i]
        yesterday = history[i + 1]

        today_open = today.get("open", 0)
        yesterday_close = yesterday.get("close", yesterday.get("adjClose", 0))
        today_volume = today.get("volume", 0)
        yesterday_volume = yesterday.get("volume", 0)

        if yesterday_close == 0 or yesterday_volume == 0:
            continue

        gap_pct = (today_open - yesterday_close) / yesterday_close * 100
        volume_increase = today_volume > yesterday_volume * 1.1

        # Gap down >= 0.5% on higher volume
        if gap_pct <= -0.5 and volume_increase:
            return True

    return False

#!/usr/bin/env python3
"""
Component 2: Leading Stock Health (Weight: 20%)

Evaluates health of leading/growth ETFs as proxy for market leadership.
Uses ETF baskets instead of individual stocks for API efficiency.

Default ETF Basket: ARKK, WCLD, IGV, XBI, SOXX, SMH, KWEB, TAN
Dynamic mode: Selects top-N ETFs from a 20-ETF candidate pool by 52-week high proximity.

Evaluation per ETF:
- Distance from 52-week high
- Position vs 50DMA and 200DMA
- Lower highs pattern detection

Scoring: Weighted average of ETF deterioration signals.
If 60%+ ETFs are deteriorating, apply 1.3x amplification.
"""

from typing import Optional

# Default Leading/Growth ETF basket
DEFAULT_LEADING_ETFS = ["ARKK", "WCLD", "IGV", "XBI", "SOXX", "SMH", "KWEB", "TAN"]
LEADING_ETFS = DEFAULT_LEADING_ETFS  # backward compat

# Expanded candidate pool for dynamic basket selection
CANDIDATE_POOL = [
    "SMH",
    "SOXX",
    "PSI",
    "SOXQ",  # Semiconductors
    "IGV",
    "WCLD",
    "CLOU",
    "BUG",  # Software / Cloud / Cyber
    "XBI",
    "ARKG",  # Biotech / Genomics
    "ARKK",
    "ARKW",
    "KOMP",  # Innovation / Disruptive
    "TAN",
    "ICLN",  # Clean Energy
    "KWEB",
    "FDN",  # Internet / China Tech
    "FINX",
    "IPAY",  # Fintech / Payments
    "BOTZ",  # Robotics / AI
]


def select_dynamic_basket(quotes: dict[str, dict], top_n: int = 10) -> list[str]:
    """
    Select top-N ETFs from candidate pool by proximity to 52-week high.

    Args:
        quotes: Dict of symbol -> quote data with 'price' and 'yearHigh'
        top_n: Number of ETFs to select (default 10)

    Returns:
        List of selected symbols, sorted by proximity to high (closest first).
        Falls back to DEFAULT_LEADING_ETFS if fewer than 5 valid candidates.
    """
    scored = []
    for symbol in CANDIDATE_POOL:
        q = quotes.get(symbol)
        if not q:
            continue
        price = q.get("price", 0)
        year_high = q.get("yearHigh", 0)
        if year_high <= 0 or price <= 0:
            continue
        proximity = price / year_high  # 1.0 = at high, 0.5 = 50% below
        scored.append((symbol, proximity))

    if len(scored) < 5:
        return list(DEFAULT_LEADING_ETFS)

    # Sort by proximity descending (closest to high = strongest leader)
    scored.sort(key=lambda x: x[1], reverse=True)
    return [s[0] for s in scored[:top_n]]


def calculate_leading_stock_health(
    quotes: dict[str, dict], historical: dict[str, list[dict]], etf_list: Optional[list[str]] = None
) -> dict:
    """
    Calculate leading stock health score.

    Args:
        quotes: Dict of symbol -> quote data (from FMP batch quote)
        historical: Dict of symbol -> list of daily OHLCV (most recent first, ~60 days)
        etf_list: Optional explicit list of ETF symbols to evaluate.
                  If None, uses quotes.keys() or DEFAULT_LEADING_ETFS.

    Returns:
        Dict with score (0-100), etf_details, signal, basket_mode, basket
    """
    etf_scores = []
    etf_details = {}
    if etf_list is not None:
        basket_mode = "dynamic" if etf_list != list(DEFAULT_LEADING_ETFS) else "static"
        eval_list = list(etf_list)
    elif quotes:
        basket_mode = "static"
        eval_list = list(quotes.keys())
    else:
        basket_mode = "static"
        eval_list = list(DEFAULT_LEADING_ETFS)
    total_attempted = len(eval_list)
    fetch_successes = 0
    partial_data_count = 0

    # Minimum history bars for full analysis (50DMA calculation)
    MIN_HISTORY_BARS = 50

    for symbol in eval_list:
        quote = quotes.get(symbol)
        hist = historical.get(symbol, [])

        if not quote:
            continue

        # Full success requires quote + sufficient history
        if len(hist) >= MIN_HISTORY_BARS:
            fetch_successes += 1
        else:
            partial_data_count += 1

        detail = _evaluate_etf(symbol, quote, hist)
        etf_scores.append(detail["deterioration_score"])
        etf_details[symbol] = detail

    fetch_success_rate = fetch_successes / total_attempted if total_attempted > 0 else 0.0

    if not etf_scores:
        return {
            "score": 50,
            "signal": "INSUFFICIENT DATA",
            "etf_details": {},
            "etfs_evaluated": 0,
            "etfs_deteriorating": 0,
            "data_available": False,
            "fetch_success_rate": fetch_success_rate,
            "partial_data_count": partial_data_count,
            "basket_mode": basket_mode,
            "basket": eval_list,
        }

    # Mark data as unavailable if fetch success rate is below 75%
    data_available = fetch_success_rate >= 0.75

    avg_deterioration = sum(etf_scores) / len(etf_scores)

    # Count deteriorating ETFs (score >= 50)
    deteriorating_count = sum(1 for s in etf_scores if s >= 50)
    deteriorating_pct = deteriorating_count / len(etf_scores)

    # Amplification: if 60%+ ETFs deteriorating, multiply by 1.3
    if deteriorating_pct >= 0.60:
        final_score = min(100, avg_deterioration * 1.3)
    else:
        final_score = avg_deterioration

    final_score = round(min(100, max(0, final_score)))

    # Signal
    if final_score >= 70:
        signal = "CRITICAL: Leadership broadly deteriorating"
    elif final_score >= 50:
        signal = "WARNING: Multiple leaders weakening"
    elif final_score >= 30:
        signal = "CAUTION: Some leaders showing strain"
    else:
        signal = "HEALTHY: Leadership intact"

    return {
        "score": final_score,
        "signal": signal,
        "avg_deterioration": round(avg_deterioration, 1),
        "etfs_evaluated": len(etf_scores),
        "etfs_deteriorating": deteriorating_count,
        "deteriorating_pct": round(deteriorating_pct * 100, 1),
        "amplified": deteriorating_pct >= 0.60,
        "etf_details": etf_details,
        "data_available": data_available,
        "fetch_success_rate": round(fetch_success_rate, 2),
        "partial_data_count": partial_data_count,
        "basket_mode": basket_mode,
        "basket": eval_list,
    }


def _evaluate_etf(symbol: str, quote: dict, history: list[dict]) -> dict:
    """Evaluate a single ETF's deterioration level"""
    score = 0
    flags = []

    price = quote.get("price", 0)
    year_high = quote.get("yearHigh", 0)
    quote.get("yearLow", 0)

    # 1. Distance from 52-week high (0-40 points)
    if year_high > 0:
        distance_pct = (price - year_high) / year_high * 100
        if distance_pct <= -25:
            score += 40
            flags.append(f">{abs(distance_pct):.0f}% below 52wk high (bear territory)")
        elif distance_pct <= -15:
            score += 30
            flags.append(f"{abs(distance_pct):.0f}% below 52wk high (correction)")
        elif distance_pct <= -10:
            score += 20
            flags.append(f"{abs(distance_pct):.0f}% below 52wk high")
        elif distance_pct <= -5:
            score += 10
            flags.append(f"{abs(distance_pct):.0f}% below 52wk high")
    else:
        distance_pct = 0

    # 2. Position vs moving averages (0-40 points)
    if history and len(history) >= 50:
        closes = [d.get("close", d.get("adjClose", 0)) for d in history]

        # 50DMA
        sma50 = sum(closes[:50]) / 50
        if price < sma50:
            score += 20
            flags.append(f"Below 50DMA (${sma50:.2f})")

        # 200DMA (if enough data)
        if len(closes) >= 200:
            sma200 = sum(closes[:200]) / 200
            if price < sma200:
                score += 20
                flags.append(f"Below 200DMA (${sma200:.2f})")
        elif len(closes) >= 50:
            # Estimate 200DMA position from available data
            if price < sma50 * 0.95:  # >5% below 50DMA suggests below 200DMA
                score += 10
                flags.append("Likely below 200DMA (estimated)")

    elif history and len(history) >= 20:
        closes = [d.get("close", d.get("adjClose", 0)) for d in history]
        sma20 = sum(closes[:20]) / 20
        if price < sma20:
            score += 15
            flags.append(f"Below 20DMA (${sma20:.2f})")

    # 3. Lower highs pattern (0-20 points)
    if history and len(history) >= 20:
        lower_highs = _detect_lower_highs(history)
        if lower_highs:
            score += 20
            flags.append("Lower highs pattern detected")

    score = min(100, score)

    return {
        "deterioration_score": score,
        "price": price,
        "year_high": year_high,
        "distance_from_high_pct": round(distance_pct, 1),
        "flags": flags,
    }


def _detect_lower_highs(history: list[dict], lookback: int = 20) -> bool:
    """
    Detect lower highs pattern in recent price action.

    Look for at least 2 consecutive lower swing highs in the last 20 days.
    """
    if len(history) < lookback:
        return False

    highs = [d.get("high", d.get("close", 0)) for d in history[:lookback]]

    # Find local maxima (swing highs)
    swing_highs = []
    for i in range(1, len(highs) - 1):
        if highs[i] > highs[i - 1] and highs[i] > highs[i + 1]:
            swing_highs.append(highs[i])

    # Need at least 2 swing highs to compare
    if len(swing_highs) < 2:
        return False

    # Check if most recent swing highs are declining
    # swing_highs[0] is earliest (since history is most-recent-first, reversed)
    # Actually history[0] = most recent, so highs[0] = most recent high
    # So swing_highs are in reverse chronological order
    return swing_highs[0] < swing_highs[1]

#!/usr/bin/env python3
"""
Shared EMA/SMA calculation utilities for Market Top Detector.

All functions expect prices in most-recent-first order.
"""


def calc_ema(prices: list[float], period: int) -> float:
    """
    Calculate Exponential Moving Average from prices (most recent first).

    Args:
        prices: List of prices, most recent first.
        period: EMA period (must be >= 1).

    Returns:
        EMA value as float.

    Raises:
        ValueError: If prices is empty or period < 1.
    """
    if not prices:
        raise ValueError("prices must not be empty")
    if period < 1:
        raise ValueError("period must be >= 1")

    if len(prices) < period:
        return sum(prices) / len(prices)

    prices_rev = prices[::-1]
    sma = sum(prices_rev[:period]) / period
    ema = sma
    k = 2 / (period + 1)
    for p in prices_rev[period:]:
        ema = p * k + ema * (1 - k)
    return ema


def calc_sma(prices: list[float], period: int) -> float:
    """
    Calculate Simple Moving Average from prices (most recent first).

    Args:
        prices: List of prices, most recent first.
        period: SMA period (must be >= 1).

    Returns:
        SMA value as float.

    Raises:
        ValueError: If prices is empty or period < 1.
    """
    if not prices:
        raise ValueError("prices must not be empty")
    if period < 1:
        raise ValueError("period must be >= 1")

    if len(prices) < period:
        return sum(prices) / len(prices)
    return sum(prices[:period]) / period

#!/usr/bin/env python3
"""
Component 6: Sentiment & Speculation (Weight: 10%)

Data sources:
- VIX: FMP API quote
- Put/Call ratio: CLI argument (from WebSearch)
- VIX term structure: CLI argument (from WebSearch) or auto-detected
- Margin Debt YoY: CLI argument (from WebSearch)

Scoring (additive, max 100):
  Put/Call < 0.60 -> +35pt (extreme call buying = complacency)
  Put/Call < 0.70 -> +25pt
  Put/Call < 0.80 -> +12pt
  Put/Call >= 0.80 -> +0pt (healthy caution)

  VIX < 12       -> +25pt (extreme low fear)
  VIX < 14       -> +17pt
  VIX < 16       -> +8pt
  VIX 16-20      -> +0pt (normal)
  VIX > 25       -> -8pt (fear already present, top less likely)

  VIX term structure:
    Steep contango (normal) -> +25pt (complacency)
    Normal contango        -> +12pt
    Flat                   -> +0pt
    Backwardation          -> -8pt (hedging demand = fear)

  Margin Debt YoY:
    > 30%  -> +15pt (extreme leverage)
    > 20%  -> +10pt
    > 10%  -> +5pt
    <= 10% -> +0pt
"""

from typing import Optional

# VIX term structure states
STEEP_CONTANGO = "steep_contango"
NORMAL_CONTANGO = "contango"
FLAT = "flat"
BACKWARDATION = "backwardation"


def calculate_sentiment(
    vix_level: Optional[float] = None,
    put_call_ratio: Optional[float] = None,
    vix_term_structure: Optional[str] = None,
    margin_debt_yoy_pct: Optional[float] = None,
) -> dict:
    """
    Calculate sentiment and speculation score.

    Args:
        vix_level: Current VIX value (from FMP API)
        put_call_ratio: CBOE equity put/call ratio (from WebSearch/CLI)
        vix_term_structure: One of 'steep_contango', 'contango', 'flat', 'backwardation'
        margin_debt_yoy_pct: Year-over-year margin debt change % (optional extra context)

    Returns:
        Dict with score (0-100), signal, component details
    """
    # All inputs missing -> neutral default
    if vix_level is None and put_call_ratio is None and vix_term_structure is None:
        return {
            "score": 50,
            "signal": "NO DATA: All sentiment inputs missing (neutral default)",
            "data_available": False,
            "total_points": 0,
            "details": {},
        }

    total_points = 0
    details = {}

    # 1. Put/Call Ratio scoring (max 35pt)
    pc_points = 0
    if put_call_ratio is not None:
        if put_call_ratio < 0.60:
            pc_points = 35
        elif put_call_ratio < 0.70:
            pc_points = 25
        elif put_call_ratio < 0.80:
            pc_points = 12
        else:
            pc_points = 0
        details["put_call_ratio"] = {
            "value": put_call_ratio,
            "points": pc_points,
            "interpretation": _interpret_put_call(put_call_ratio),
        }
    else:
        details["put_call_ratio"] = {"value": None, "points": 0, "interpretation": "No data"}

    total_points += pc_points

    # 2. VIX Level scoring (max 25pt)
    vix_points = 0
    if vix_level is not None:
        if vix_level < 12:
            vix_points = 25
        elif vix_level < 14:
            vix_points = 17
        elif vix_level < 16:
            vix_points = 8
        elif vix_level <= 25:
            vix_points = 0
        else:
            vix_points = -8  # High fear = top less likely
        details["vix_level"] = {
            "value": round(vix_level, 2),
            "points": vix_points,
            "interpretation": _interpret_vix(vix_level),
        }
    else:
        details["vix_level"] = {"value": None, "points": 0, "interpretation": "No data"}

    total_points += vix_points

    # 3. VIX Term Structure scoring (max 25pt)
    vts_points = 0
    if vix_term_structure:
        if vix_term_structure == STEEP_CONTANGO:
            vts_points = 25
        elif vix_term_structure == NORMAL_CONTANGO:
            vts_points = 12
        elif vix_term_structure == FLAT:
            vts_points = 0
        elif vix_term_structure == BACKWARDATION:
            vts_points = -8
        details["vix_term_structure"] = {
            "value": vix_term_structure,
            "points": vts_points,
            "interpretation": _interpret_vix_term(vix_term_structure),
        }
    else:
        details["vix_term_structure"] = {"value": None, "points": 0, "interpretation": "No data"}

    total_points += vts_points

    # 4. Margin Debt YoY scoring (max 15pt)
    md_points = 0
    if margin_debt_yoy_pct is not None:
        if margin_debt_yoy_pct > 30:
            md_points = 15
        elif margin_debt_yoy_pct > 20:
            md_points = 10
        elif margin_debt_yoy_pct > 10:
            md_points = 5
        else:
            md_points = 0
        details["margin_debt"] = {
            "yoy_pct": margin_debt_yoy_pct,
            "points": md_points,
            "interpretation": _interpret_margin_debt(margin_debt_yoy_pct),
        }

    total_points += md_points

    # Clamp to 0-100
    score = max(0, min(100, total_points))

    # Signal
    if score >= 70:
        signal = "CRITICAL: Extreme complacency / speculation"
    elif score >= 50:
        signal = "WARNING: Elevated complacency"
    elif score >= 30:
        signal = "CAUTION: Some speculative excess"
    elif score >= 10:
        signal = "MIXED: Moderate sentiment"
    else:
        signal = "HEALTHY: No excessive speculation"

    return {
        "score": score,
        "signal": signal,
        "data_available": True,
        "total_points": total_points,
        "details": details,
    }


def _interpret_put_call(ratio: float) -> str:
    if ratio < 0.60:
        return "Extreme call buying - maximum complacency"
    elif ratio < 0.70:
        return "Elevated call buying - notable optimism"
    elif ratio < 0.80:
        return "Slightly optimistic"
    elif ratio < 0.90:
        return "Normal range"
    else:
        return "Elevated put buying - caution/fear present"


def _interpret_vix(level: float) -> str:
    if level < 12:
        return "Extreme low fear - complacency zone"
    elif level < 14:
        return "Low fear"
    elif level < 16:
        return "Mildly low volatility"
    elif level <= 20:
        return "Normal range"
    elif level <= 25:
        return "Elevated fear"
    else:
        return "High fear - panic present"


def _interpret_vix_term(structure: str) -> str:
    mapping = {
        STEEP_CONTANGO: "Market expects calm - complacency in term structure",
        NORMAL_CONTANGO: "Normal term structure",
        FLAT: "Uncertainty - no clear expectation",
        BACKWARDATION: "Hedging demand elevated - fear in term structure",
    }
    return mapping.get(structure, "Unknown")


def _interpret_margin_debt(yoy_pct: float) -> str:
    if yoy_pct >= 30:
        return f"DANGER: Margin debt surging +{yoy_pct:.0f}% YoY"
    elif yoy_pct >= 20:
        return f"WARNING: Rapid leverage increase +{yoy_pct:.0f}% YoY"
    elif yoy_pct >= 10:
        return f"Elevated: +{yoy_pct:.0f}% YoY"
    elif yoy_pct >= 0:
        return f"Normal: +{yoy_pct:.0f}% YoY"
    else:
        return f"Deleveraging: {yoy_pct:.0f}% YoY"

#!/usr/bin/env python3
# GENERATED by scripts/generate_fmp_client.py — do not edit.
# Source of truth: scripts/fmp_client/specials/ and scripts/fmp_client/registry.py.
# Regenerate: python3 scripts/generate_fmp_client.py
"""
FMP API Client for Market Top Detector

Provides rate-limited access to Financial Modeling Prep API endpoints
for market top detection analysis.

Features:
- Rate limiting (0.3s between requests)
- Automatic retry on 429 errors
- Session caching for duplicate requests
- Batch quote support for ETF baskets
"""

import os
import sys
import time
from datetime import date, timedelta
from typing import Optional

try:
    import requests
except ImportError:
    print("ERROR: requests library not found. Install with: pip install requests", file=sys.stderr)
    sys.exit(1)


# --- FMP endpoint fallback: stable (new users) -> v3 (legacy users) ---


def _stable_quote_url(base, symbols_str, params):
    """stable/quote?symbol=^GSPC"""
    params["symbol"] = symbols_str
    return base, params


def _v3_quote_url(base, symbols_str, params):
    """api/v3/quote/^GSPC"""
    return f"{base}/{symbols_str}", params


def _stable_hist_url(base, symbols_str, params):
    """stable/historical-price-eod/full?symbol=^GSPC&from=...&to=..."""
    params["symbol"] = symbols_str
    # New stable EOD endpoint ignores `timeseries`; convert to from/to range
    # to bound the payload. Use 2x calendar days to cover N trading days
    # (trading-day/calendar-day ratio ~252/365 ~0.69, so *2 leaves headroom).
    days = params.pop("timeseries", None)
    if days is not None:
        today = date.today()
        params["from"] = (today - timedelta(days=int(days) * 2)).isoformat()
        params["to"] = today.isoformat()
    return base, params


def _v3_hist_url(base, symbols_str, params):
    """api/v3/historical-price-full/^GSPC?timeseries=80"""
    return f"{base}/{symbols_str}", params


_FMP_ENDPOINTS = {
    "quote": [
        ("https://financialmodelingprep.com/stable/quote", _stable_quote_url),
        ("https://financialmodelingprep.com/api/v3/quote", _v3_quote_url),
    ],
    "historical": [
        ("https://financialmodelingprep.com/stable/historical-price-eod/full", _stable_hist_url),
        ("https://financialmodelingprep.com/api/v3/historical-price-full", _v3_hist_url),
    ],
}


def _normalize_eod_flat_list(data, symbols_str: str, limit: Optional[int] = None):
    """Convert stable/historical-price-eod/full flat list to v3-compatible dict.

    Input  : [{"symbol": "SPY", "date": "...", "open": ..., ...}, ...]
    Output : {"symbol": "SPY", "historical": [{"date": ..., "open": ..., ...}, ...]}

    Returns the input unchanged if not a list (passthrough for v3 dict /
    historicalStockList responses). Returns None when no row matches the
    requested symbol; the caller will record the failure and try the next
    endpoint.

    If `limit` is provided (the original `timeseries=N` request), the
    `historical` list is truncated to the first `limit` entries. The new
    EOD endpoint ignores `timeseries` and returns the full available history,
    so the caller's date-range bounding plus this truncation together preserve
    the legacy "most-recent N rows" contract. Truncation assumes descending
    date order, which the FMP EOD endpoint provides (verified live).

    Note: empty list ``[]`` does not reach this normalizer because the caller's
    ``if not data: continue`` falsy check handles it earlier in
    ``_request_with_fallback``.
    """
    if not isinstance(data, list):
        return data
    if not data:
        return None
    norm_target = symbols_str.replace("-", ".")
    matched_symbol = None
    historical = []
    for row in data:
        if not isinstance(row, dict):
            continue
        # Be permissive: single-symbol endpoint may omit per-row "symbol".
        # Treat missing symbol as belonging to the requested symbols_str.
        row_sym = row.get("symbol") or symbols_str
        if row_sym.replace("-", ".") != norm_target:
            continue
        matched_symbol = matched_symbol or row_sym
        historical.append({k: v for k, v in row.items() if k != "symbol"})
    if not historical:
        return None
    if limit is not None and limit > 0:
        historical = historical[:limit]
    return {"symbol": matched_symbol or symbols_str, "historical": historical}


def _has_usable_history(data) -> bool:
    """True only when `data` is a dict carrying a non-empty `historical` list.

    A dict with an empty `historical` list (e.g. an index/ETF unavailable on the
    caller's FMP plan) is treated as unusable so the caller can fall back to
    yfinance instead of caching an empty result.
    """
    return bool(
        isinstance(data, dict) and isinstance(data.get("historical"), list) and data["historical"]
    )


class FMPClient:
    """Client for Financial Modeling Prep API with rate limiting and caching"""

    BASE_URL = "https://financialmodelingprep.com/api/v3"
    RATE_LIMIT_DELAY = 0.3  # 300ms between requests

    _ENDPOINT_FAILURE_THRESHOLD = 3  # disable endpoint after N consecutive failures

    def __init__(self, api_key: Optional[str] = None):
        self.api_key = api_key or os.getenv("FMP_API_KEY")
        if not self.api_key:
            raise ValueError(
                "FMP API key required. Set FMP_API_KEY environment variable "
                "or pass api_key parameter."
            )
        self.session = requests.Session()
        self.session.headers.update({"apikey": self.api_key})
        self.cache = {}
        self.last_call_time = 0
        self.rate_limit_reached = False
        self.retry_count = 0
        self.max_retries = 1
        self.api_calls_made = 0
        # Circuit breaker: track consecutive failures per endpoint URL prefix
        self._endpoint_failures: dict[str, int] = {}
        self._disabled_endpoints: set[str] = set()

    def _rate_limited_get(
        self, url: str, params: Optional[dict] = None, quiet: bool = False
    ) -> Optional[dict]:
        if self.rate_limit_reached:
            return None

        if params is None:
            params = {}

        elapsed = time.time() - self.last_call_time
        if elapsed < self.RATE_LIMIT_DELAY:
            time.sleep(self.RATE_LIMIT_DELAY - elapsed)

        try:
            response = self.session.get(url, params=params, timeout=30)
            self.last_call_time = time.time()
            self.api_calls_made += 1

            if response.status_code == 200:
                self.retry_count = 0
                return response.json()
            elif response.status_code == 429:
                self.retry_count += 1
                if self.retry_count <= self.max_retries:
                    print("WARNING: Rate limit exceeded. Waiting 60 seconds...", file=sys.stderr)
                    time.sleep(60)
                    return self._rate_limited_get(url, params, quiet=quiet)
                else:
                    print("ERROR: Daily API rate limit reached.", file=sys.stderr)
                    self.rate_limit_reached = True
                    return None
            else:
                if not quiet:
                    print(
                        f"ERROR: API request failed: {response.status_code} - {response.text[:200]}",
                        file=sys.stderr,
                    )
                return None
        except requests.exceptions.RequestException as e:
            print(f"ERROR: Request exception: {e}", file=sys.stderr)
            return None

    def _request_with_fallback(self, endpoint_key, symbols_str, extra_params=None):
        """Try stable endpoint first, fall back to v3 for legacy users.

        Returns parsed JSON in v3-compatible shape, or None if all fail.
        Non-last endpoints use quiet=True to suppress expected 403 stderr.
        """
        params = dict(extra_params) if extra_params else {}
        endpoints = _FMP_ENDPOINTS[endpoint_key]
        is_single = "," not in symbols_str

        for i, (base_url, url_builder) in enumerate(endpoints):
            # Circuit breaker: skip endpoints with too many consecutive failures
            if base_url in self._disabled_endpoints:
                continue

            url, final_params = url_builder(base_url, symbols_str, dict(params))
            is_last = i == len(endpoints) - 1
            data = self._rate_limited_get(url, final_params, quiet=not is_last)
            if not data:  # falsy (None, [], {}) — try next endpoint
                self._record_endpoint_failure(base_url)
                continue

            # Normalize new stable EOD flat-list shape to v3-compatible dict.
            # No-op for v3 dict / historicalStockList responses.
            # `timeseries` (original request) is passed as `limit` so the
            # EOD endpoint's full-history response is truncated to the
            # legacy "most-recent N rows" contract.
            if endpoint_key == "historical":
                limit = params.get("timeseries") if isinstance(params, dict) else None
                data = _normalize_eod_flat_list(data, symbols_str, limit=limit)
                if not data:
                    self._record_endpoint_failure(base_url)
                    continue

            # Shape validation: reject truthy-but-wrong-shape responses
            valid = True
            if endpoint_key == "quote":
                if not isinstance(data, list) or len(data) == 0:
                    valid = False
                elif is_single and not any(
                    q.get("symbol", "").replace("-", ".") == symbols_str.replace("-", ".")
                    for q in data
                ):
                    valid = False

            if endpoint_key == "historical":
                if not isinstance(data, dict):
                    valid = False
                elif "historicalStockList" in data:
                    # stable batch format -> v3 single format (exact match only)
                    norm = symbols_str.replace("-", ".")
                    found = None
                    for entry in data["historicalStockList"]:
                        if entry.get("symbol", "").replace("-", ".") == norm:
                            found = {
                                "symbol": entry.get("symbol"),
                                "historical": entry.get("historical", []),
                            }
                            break
                    if found:
                        self._endpoint_failures[base_url] = 0
                        return found
                    valid = False
                elif "historical" not in data:
                    valid = False
                elif is_single and data.get("symbol"):
                    if data["symbol"].replace("-", ".") != symbols_str.replace("-", "."):
                        valid = False

            if valid:
                self._endpoint_failures[base_url] = 0
                return data
            self._record_endpoint_failure(base_url)
        return None

    def _record_endpoint_failure(self, base_url: str) -> None:
        """Track consecutive failures and disable endpoint after threshold."""
        failures = self._endpoint_failures.get(base_url, 0) + 1
        self._endpoint_failures[base_url] = failures
        if failures >= self._ENDPOINT_FAILURE_THRESHOLD:
            self._disabled_endpoints.add(base_url)

    def get_quote(self, symbols: str) -> Optional[list[dict]]:
        """Fetch real-time quote data for one or more symbols (comma-separated).

        Falls back to a yfinance-derived quote when FMP returns nothing (e.g. an
        index like ^GSPC/^VIX gated behind a paid FMP plan). Fallback is
        single-symbol only — comma lists are left to FMP.
        """
        cache_key = f"quote_{symbols}"
        if cache_key in self.cache:
            return self.cache[cache_key]

        data = self._request_with_fallback("quote", symbols)
        if not data and "," not in symbols:
            data = self._get_quote_from_yfinance(symbols)
        if data:
            self.cache[cache_key] = data
        return data

    def _get_quote_from_yfinance(self, symbol: str) -> Optional[list[dict]]:
        """Fallback quote derived from recent yfinance daily bars.

        Returns the FMP quote shape (a list with one dict carrying
        price/previousClose/change/changesPercentage/open/high/low/volume/
        yearHigh/yearLow), or ``None`` when history is unavailable. Year
        high/low are approximated from ~252 daily bars.
        """
        hist = self._get_hist_from_yfinance(symbol, 260)
        if not _has_usable_history(hist):
            return None
        bars = hist["historical"]  # most-recent-first
        latest = bars[0]
        prev_close = bars[1]["close"] if len(bars) > 1 else latest["close"]
        price = latest["close"]
        change = price - prev_close
        pct = (change / prev_close * 100) if prev_close else 0.0
        window = bars[:252]
        year_high = max(b["high"] for b in window)
        year_low = min(b["low"] for b in window)
        return [
            {
                "symbol": symbol,
                "price": price,
                "previousClose": prev_close,
                "change": round(change, 4),
                "changesPercentage": round(pct, 4),
                "open": latest["open"],
                "dayHigh": latest["high"],
                "dayLow": latest["low"],
                "high": latest["high"],
                "low": latest["low"],
                "volume": latest["volume"],
                "yearHigh": year_high,
                "yearLow": year_low,
            }
        ]

    def get_historical_prices(self, symbol: str, days: int = 365) -> Optional[dict]:
        """Fetch historical daily OHLCV data.

        Falls back to yfinance when the FMP endpoint returns no usable history
        (e.g. an index/ETF gated behind a paid FMP plan). The yfinance path
        needs no extra API key; an FMP key is still required to build the client.
        """
        cache_key = f"prices_{symbol}_{days}"
        if cache_key in self.cache:
            return self.cache[cache_key]

        data = self._request_with_fallback("historical", symbol, {"timeseries": days})
        if not _has_usable_history(data):
            data = self._get_hist_from_yfinance(symbol, days)
        if _has_usable_history(data):
            self.cache[cache_key] = data
            return data
        return None

    def _get_hist_from_yfinance(self, symbol: str, days: int) -> Optional[dict]:
        """Fallback: fetch daily history via yfinance when FMP is unavailable.

        Returns the FMP-compatible shape ``{"symbol": ..., "historical": [...]}``
        with most-recent-first bars (date/open/high/low/close/adjClose/volume),
        or ``None`` on empty/error. ``yfinance`` is imported lazily so the FMP
        success path never depends on it.
        """
        try:
            import yfinance as yf

            end = date.today()
            start = end - timedelta(days=int(days * 1.5))  # cover weekends/holidays
            df = yf.download(
                symbol,
                start=start.isoformat(),
                end=end.isoformat(),
                auto_adjust=True,
                progress=False,
            )
            if df is None or df.empty:
                return None
            # yfinance returns MultiIndex columns for a single ticker.
            if hasattr(df.columns, "levels"):
                df.columns = df.columns.droplevel(1)
            historical = []
            for idx, row in df.iterrows():
                close = float(row["Close"])
                historical.append(
                    {
                        "date": idx.strftime("%Y-%m-%d"),
                        "open": float(row["Open"]),
                        "high": float(row["High"]),
                        "low": float(row["Low"]),
                        "close": close,
                        "adjClose": close,  # auto_adjust=True -> Close already adjusted
                        "volume": int(row["Volume"]),
                    }
                )
            if not historical:
                return None
            historical.reverse()  # yfinance ascending -> FMP most-recent-first
            return {"symbol": symbol, "historical": historical[:days]}
        except Exception as e:
            print(f"WARNING: yfinance history fallback failed for {symbol}: {e}", file=sys.stderr)
            return None

    def get_batch_quotes(self, symbols: list[str]) -> dict[str, dict]:
        """Fetch quotes for a list of symbols.

        Note: The FMP `/stable/quote` endpoint only accepts a single symbol per
        call. Passing comma-separated symbols silently returns []. The
        deprecated `/api/v3/quote/A,B,C` batch syntax now returns 403 for
        non-legacy keys (retired Aug 31, 2025). So we iterate one symbol at a
        time. Rate limiting in `_rate_limited_get` keeps this within plan
        limits.
        """
        results = {}
        for symbol in symbols:
            quotes = self.get_quote(symbol)
            if quotes:
                for q in quotes:
                    if q.get("symbol"):
                        results[q["symbol"]] = q
        return results

    def get_batch_historical(self, symbols: list[str], days: int = 50) -> dict[str, list[dict]]:
        """Fetch historical prices for multiple symbols"""
        results = {}
        for symbol in symbols:
            data = self.get_historical_prices(symbol, days=days)
            if data and "historical" in data:
                results[symbol] = data["historical"]
        return results

    def calculate_ema(self, prices: list[float], period: int) -> float:
        """Calculate EMA (thin wrapper around math_utils for backward compat)."""
        from calculators.math_utils import calc_ema

        return calc_ema(prices, period)

    def calculate_sma(self, prices: list[float], period: int) -> float:
        """Calculate SMA (thin wrapper around math_utils for backward compat)."""
        from calculators.math_utils import calc_sma

        return calc_sma(prices, period)

    def get_vix_term_structure(self) -> Optional[dict]:
        """
        Auto-detect VIX term structure by comparing VIX to VIX3M.

        Returns:
            Dict with ratio, classification, or None if VIX3M unavailable.
        """
        vix_quotes = self.get_quote("^VIX")
        vix3m_quotes = self.get_quote("^VIX3M")

        if not vix_quotes or not vix3m_quotes:
            return None

        vix_price = vix_quotes[0].get("price", 0)
        vix3m_price = vix3m_quotes[0].get("price", 0)

        if vix3m_price <= 0:
            return None

        ratio = vix_price / vix3m_price

        if ratio < 0.85:
            classification = "steep_contango"
        elif ratio < 0.95:
            classification = "contango"
        elif ratio <= 1.05:
            classification = "flat"
        else:
            classification = "backwardation"

        return {
            "vix": round(vix_price, 2),
            "vix3m": round(vix3m_price, 2),
            "ratio": round(ratio, 3),
            "classification": classification,
        }

    def get_api_stats(self) -> dict:
        return {
            "cache_entries": len(self.cache),
            "api_calls_made": self.api_calls_made,
            "rate_limit_reached": self.rate_limit_reached,
        }