Twstock Heatmap

#!/usr/bin/env python3
"""
Taiwan Stock Market Heatmap AI Analysis
Uses GitHub Models API (GPT-4o Vision) to identify top losers from multiple heatmap screenshots
"""

import os
import sys
import json
import base64
import csv
import io
from datetime import datetime
from pathlib import Path
import argparse

# Fix Windows console encoding issues
if sys.platform == "win32":
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")



def load_env_file():
    """Load environment variables from .env file if it exists"""
    env_path = Path(__file__).parent.parent.parent.parent / ".env"
    
    if env_path.exists():
        with open(env_path, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                # Skip empty lines and comments
                if line and not line.startswith("#"):
                    if "=" in line:
                        key, value = line.split("=", 1)
                        key = key.strip()
                        value = value.strip().strip('"').strip("'")
                        # Only set if not already in environment
                        if key not in os.environ:
                            os.environ[key] = value


def load_stock_mapping():
    """Load stock name to ticker mapping from StockMapping.csv"""
    mapping_path = Path(__file__).parent.parent.parent.parent / "data" / "StockMapping.csv"
    
    stock_mapping = {}
    
    if mapping_path.exists():
        try:
            with open(mapping_path, "r", encoding="utf-8") as f:
                reader = csv.DictReader(f)
                for row in reader:
                    # Map stock name to ticker
                    stock_mapping[row['name']] = row['ticker']
            print(f"✓ Loaded {len(stock_mapping)} stock mappings from {mapping_path.name}")
        except Exception as e:
            print(f"Warning: Failed to load stock mapping: {e}")
    else:
        print(f"Warning: Stock mapping file not found: {mapping_path}")
    
    return stock_mapping


def verify_stock_decline_yahoo(ticker):
    """
    Verify if a stock is currently declining using Yahoo Finance API
    
    Args:
        ticker: Taiwan stock ticker (e.g., "2344")
    
    Returns:
        True if stock is declining (negative change), False otherwise
        Returns True on API errors to avoid false negatives
    """
    try:
        import yfinance as yf
        
        # Determine if TSE or OTC based on ticker
        # TSE tickers are typically 4 digits starting with 1-9
        # OTC tickers are typically 4 digits starting with specific ranges
        # For simplicity, we'll try .TW first, then .TWO if that fails
        
        ticker_tw = f"{ticker}.TW"
        ticker_two = f"{ticker}.TWO"
        
        # Try TSE first
        stock = yf.Ticker(ticker_tw)
        try:
            info = stock.info
            if info and 'regularMarketChangePercent' in info:
                change_percent = info['regularMarketChangePercent']
                is_declining = change_percent < 0
                print(f"  📊 Yahoo Finance: {ticker}.TW change = {change_percent:.2f}% → {'✓ Declining' if is_declining else '✗ Not declining'}")
                return is_declining
        except:
            pass
        
        # Try OTC if TSE failed
        stock = yf.Ticker(ticker_two)
        try:
            info = stock.info
            if info and 'regularMarketChangePercent' in info:
                change_percent = info['regularMarketChangePercent']
                is_declining = change_percent < 0
                print(f"  📊 Yahoo Finance: {ticker}.TWO change = {change_percent:.2f}% → {'✓ Declining' if is_declining else '✗ Not declining'}")
                return is_declining
        except:
            pass
        
        # If both failed, log warning and return True to avoid false negatives
        print(f"  ⚠ Yahoo Finance: Could not fetch data for {ticker} - including by default")
        return True
        
    except ImportError:
        print(f"  ⚠ yfinance not installed - skipping Yahoo verification for {ticker}")
        return True
    except Exception as e:
        print(f"  ⚠ Yahoo Finance error for {ticker}: {e} - including by default")
        return True




def encode_image(image_path):
    """Encode image to base64 string"""
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode("utf-8")


def analyze_single_image(image_path, api_token, industry_name="all", stock_mapping=None):
    """
    Analyze a single heatmap image using GitHub Models API
    """
    import requests

    print(f"Analyzing {industry_name} from {image_path}...")

    # Encode image
    base64_image = encode_image(image_path)

    # GitHub Models API endpoint
    url = "https://models.inference.ai.azure.com/chat/completions"

    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {api_token}",
    }

    prompt = f"""分析這張台股市場熱力圖截圖 ({industry_name})。

這是一個台灣股票市場熱力圖：
- 紅色/粉紅色 = 上漲 (正數百分比，例如 +2.48%, +1.14%)
- 綠色 = 下跌 (負數百分比，例如 -2.36%, -1.65%)
- 灰色 = 平盤 (0.00%)
- 每個方塊包含：公司名稱、漲跌幅百分比

任務：找出「跌幅百分比最大」（數值最負）的 5 檔股票。

⚠️ 關鍵要求 (CRITICAL):
1. 【只選綠色方塊】：只能選擇綠色的方塊！紅色/粉紅色方塊是上漲，不是下跌！
2. 【只選負數百分比】：跌幅必須是負數（例如 -2.36%），絕對不能是正數（例如 +2.48%）！
3. 【忽略方塊大小】：不要只看大方塊！跌幅大的股票通常在「小型方塊」中（例如 -5% 的小方塊比 -0.5% 的大方塊更重要）。
4. 【搜尋深綠色】：優先掃描顏色最深的綠色區塊，無論它多小。
5. 【精確排序】：必須嚴格按照百分比數值排序（例如 -5.42% 排在 -2.74% 前面）。
6. 【完整掃描】：請仔細檢查圖片右側和下方的邊緣區域，那裡常有跌幅重的小型股。
7. 【驗證顏色】：在回報之前，再次確認你選擇的方塊是綠色，不是紅色！

⚠️ 常見錯誤 (請避免):
- ❌ 錯誤：選擇深紅色方塊（那是漲停，不是跌停！）
- ❌ 錯誤：回報正數百分比（例如 +9.92%）
- ✅ 正確：只選擇綠色方塊，只回報負數百分比（例如 -2.36%）

請返回 JSON 格式：
{{
  "top_losers": [
    {{"name": "公司名稱", "change": "跌幅百分比"}},
    ... (共5筆)
  ],
  "market": "taiwan",
  "industry": "{industry_name}"
}}"""


    payload = {
        "model": "gpt-4o",
        "messages": [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": prompt},
                    {
                        "type": "image_url",
                        "image_url": {"url": f"data:image/png;base64,{base64_image}"},
                    },
                ],
            }
        ],
        "max_tokens": 1000,
        "temperature": 0.1,
    }

    try:
        response = requests.post(url, headers=headers, json=payload, timeout=60)
        response.raise_for_status()

        result = response.json()
        content = result["choices"][0]["message"]["content"]

        # Parse JSON from response
        if "```json" in content:
            content = content.split("```json")[1].split("```")[0].strip()
        elif "```" in content:
            content = content.split("```")[1].split("```")[0].strip()

        data = json.loads(content)
        
        # Add ticker symbols to each stock if mapping is provided
        # Only include stocks with valid tickers (filter out AI misidentifications)
        # AND filter stocks with decline > 3%
        # AND verify with Yahoo Finance API that stock is currently declining
        if stock_mapping and "top_losers" in data:
            reordered_losers = []
            skipped_count = 0
            filtered_by_decline = 0
            filtered_by_yahoo = 0
            
            for stock in data["top_losers"]:
                stock_name = stock.get("name", "")
                change_str = stock.get("change", "")
                
                # Look up ticker in mapping
                ticker = stock_mapping.get(stock_name, "")
                
                if not ticker:
                    print(f"  ⚠ No ticker found for: {stock_name} - SKIPPED (possible AI misidentification)")
                    skipped_count += 1
                    continue  # Skip this stock entirely
                
                # Filter by decline percentage (must be negative AND > 3%)
                try:
                    # Extract percentage value from string like "-5.23%" or "+2.5%"
                    decline_value = float(change_str.replace("%", "").replace("+", ""))
                    
                    # CRITICAL: Filter out positive values (AI color confusion - red mistaken for green)
                    if decline_value > 0:
                        print(f"  ⚠ SKIPPED {stock_name} ({change_str}) - POSITIVE value detected! AI mistook red (gain) for green (loss)")
                        skipped_count += 1
                        continue
                    
                    # Only include if decline is greater than 3% (decline_value < -3)
                    if decline_value > -3.0:
                        filtered_by_decline += 1
                        continue
                except (ValueError, AttributeError):
                    # If we can't parse the percentage, skip this stock
                    print(f"  ⚠ Invalid change format for {stock_name}: {change_str} - SKIPPED")
                    skipped_count += 1
                    continue
                
                # NEW: Verify with Yahoo Finance API that stock is currently declining
                if not verify_stock_decline_yahoo(ticker):
                    print(f"  ⚠ SKIPPED {stock_name} ({ticker}) - Yahoo Finance shows NOT declining")
                    filtered_by_yahoo += 1
                    continue
                
                # Create new ordered dict with ticker first
                reordered_stock = {
                    "ticker": ticker,
                    "name": stock_name,
                    "change": change_str
                }
                reordered_losers.append(reordered_stock)
            
            data["top_losers"] = reordered_losers
            
            if skipped_count > 0:
                print(f"  ℹ️ Filtered out {skipped_count} stock(s) without valid ticker")
            if filtered_by_decline > 0:
                print(f"  ℹ️ Filtered out {filtered_by_decline} stock(s) with decline ≤ 3%")
            if filtered_by_yahoo > 0:
                print(f"  ℹ️ Filtered out {filtered_by_yahoo} stock(s) by Yahoo Finance verification")

        
        return data

    except Exception as e:
        print(f"Error analyzing {industry_name}: {e}")
        # Return empty structure on error
        return {
            "top_losers": [],
            "market": "taiwan",
            "industry": industry_name,
            "error": str(e),
        }


def save_json_api(data, output_path):
    """Save combined JSON API response file"""

    current_time = datetime.utcnow().isoformat() + "Z"

    # Add API metadata
    api_response = {
        "status": "success",
        "data": data,
        "version": "2.0",
        "market": "taiwan",
        "source": "nstock.tw",
        "last_updated": current_time,
        "generated_at": current_time,
    }

    # Save JSON
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(api_response, f, indent=2, ensure_ascii=False)

    print(f"JSON API saved: {output_path}")


def main():
    # Load .env file if exists (for local development)
    load_env_file()
    
    # Load stock mapping from CSV
    stock_mapping = load_stock_mapping()
    
    parser = argparse.ArgumentParser(
        description="Analyze multiple Taiwan stock heatmaps using GitHub Models API"
    )
    # Allow multiple inputs in format "type:path"
    # Example: -i all:twstock.png otc-elec:twstock_otc_elec.png
    # Or use --auto to automatically scan heatmaps/ directory
    parser.add_argument(
        "-i",
        "--inputs",
        nargs="+",
        help="Input images in format 'type:path' (e.g. tse:heatmaps/twstock.png)",
        required=False,
    )
    parser.add_argument(
        "--auto",
        action="store_true",
        help="Automatically scan and analyze all PNGs in heatmaps/ directory",
    )
    parser.add_argument(
        "--batch",
        choices=["tse", "otc"],
        help="Batch mode: analyze only TSE or OTC categories (useful to avoid API rate limits)",
    )
    parser.add_argument(
        "-o", "--output", default="api/twstock_top_losers.json", help="Output JSON path"
    )
    parser.add_argument("--token", help="GitHub Models API token")

    args = parser.parse_args()
    
    # Validate arguments
    if not args.inputs and not args.auto:
        print("Error: Either --inputs or --auto must be specified")
        print("Examples:")
        print("  Auto mode:   python analyze_twstock.py --auto")
        print("  Manual mode: python analyze_twstock.py -i tse:heatmaps/twstock.png")
        sys.exit(1)

    # Get API token (priority: --token argument > environment variable)
    api_token = args.token or os.environ.get("GITHUB_TOKEN")
    if not api_token:
        print("Error: GitHub token required")
        sys.exit(1)

    script_dir = Path(__file__).parent.parent.parent.parent
    heatmaps_dir = script_dir / "heatmaps"
    results = {}
    
    # Auto-scan mode: detect all PNGs in heatmaps/ directory
    if args.auto:
        print("🔍 Auto-scanning heatmaps directory...", flush=True)
        
        if not heatmaps_dir.exists():
            print(f"Error: Heatmaps directory not found: {heatmaps_dir}")
            sys.exit(1)
        
        # Find all PNG files
        png_files = list(heatmaps_dir.glob("*.png"))
        
        if not png_files:
            print(f"Error: No PNG files found in {heatmaps_dir}")
            sys.exit(1)
        
        # Category to market mapping
        category_market = {
            'tse': 'tse', 'otc': 'otc',
            'tse-semi': 'tse', 'tse-elec': 'tse', 'tse-computer': 'tse', 'tse-plastic': 'tse', 'tse-electrical': 'tse', 'tse-construction': 'tse', 'tse-channel': 'tse', 'tse-green': 'tse',
            'otc-elec': 'otc', 'otc-semi': 'otc', 'otc-computer': 'otc', 'otc-construction': 'otc', 'otc-other': 'otc', 'otc-info': 'otc', 'otc-tourism': 'otc', 'otc-green': 'otc'
        }
        
        print(f"Found {len(png_files)} heatmap(s):", flush=True)
        
        # Auto-detect category from filename
        inputs_list = []
        for png_file in sorted(png_files):
            filename = png_file.name
            
            # Detect category from filename
            if filename == "twstock.png":
                category = "tse"
            elif filename.startswith("twstock_"):
                # Extract category from filename (e.g., twstock_otc.png -> otc)
                category = filename.replace("twstock_", "").replace(".png", "")
            else:
                category = filename.replace(".png", "")
            
            # Filter by batch if specified
            if args.batch:
                market = category_market.get(category)
                if market != args.batch:
                    continue
            
            inputs_list.append(f"{category}:{png_file}")
            print(f"  - {category}: {png_file.name}", flush=True)
        
        # Use detected inputs
        args.inputs = inputs_list
        print()

    # Process each input image
    total_inputs = len(args.inputs)
    for idx, input_arg in enumerate(args.inputs, 1):
        try:
            if ":" in input_arg:
                industry_type, filename = input_arg.split(":", 1)
            else:
                industry_type = "default"
                filename = input_arg

            image_path = script_dir / filename
            
            # If path doesn't exist, try as absolute path
            if not image_path.exists():
                image_path = Path(filename)

            if not image_path.exists():
                print(f"Warning: Image not found: {filename}")
                continue

            print(f"Analyzing {industry_type} from {image_path.name}...", flush=True)
            
            # Analyze image with stock mapping
            analysis = analyze_single_image(
                str(image_path), api_token, industry_type, stock_mapping
            )
            results[industry_type] = analysis["top_losers"]
            
            # Add delay between API calls to avoid rate limits (except for last one)
            if idx < total_inputs:
                import time
                delay_seconds = 10
                print(f"⏳ Waiting {delay_seconds}s before next analysis to avoid API rate limits...", flush=True)
                time.sleep(delay_seconds)

        except Exception as e:
            print(f"Error analyzing {industry_type}: {e}", flush=True)
            results[industry_type] = []

    # Save combined results
    output_path = script_dir / args.output
    output_path.parent.mkdir(parents=True, exist_ok=True)

    save_json_api(results, str(output_path))

    print("\nAnalysis complete!")
    print(f"Processed industries: {', '.join(results.keys())}")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Taiwan Stock Market Heatmap Screenshot - Playwright Version
Capture nStock.tw heatmap as high-quality PNG screenshot
"""

import argparse
import sys
import subprocess
import time
import os
import io
import json
from pathlib import Path
import time

# Fix Windows console encoding issues
if sys.platform == "win32":
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")


def check_dependencies():
    """Check if required packages are installed, install if not."""
    packages = []

    try:
        from playwright.sync_api import sync_playwright
    except ImportError:
        packages.append("playwright")

    try:
        from PIL import Image
    except ImportError:
        packages.append("Pillow")

    if packages:
        print(f"Installing required packages: {', '.join(packages)}...")
        install_cmd = [sys.executable, "-m", "pip", "install"]
        if sys.platform != "win32":
            install_cmd.append("--break-system-packages")
        install_cmd.extend(packages)
        subprocess.run(install_cmd, check=True)

        # Install Playwright browsers
        if "playwright" in packages:
            print("Installing Playwright browsers...")
            subprocess.run(
                [sys.executable, "-m", "playwright", "install", "chromium"], check=True
            )


def find_treemap_element(page):
    """
    Auto-detect treemap element using multiple selectors.
    Returns element if found, None for full-page fallback.
    """
    selectors = [
        # nStock specific selectors
        ".treemap-container",
        "#treemap",
        '[class*="treemap"]',
        '[class*="heatmap"]',
        ".market-heatmap",
        # Generic chart selectors
        "canvas",
        'svg[class*="chart"]',
        ".chart-container",
        # Nuxt/Vue specific
        '[data-v-][class*="map"]',
    ]

    for selector in selectors:
        try:
            element = page.wait_for_selector(selector, timeout=3000)
            if element:
                # Verify element has reasonable dimensions
                box = element.bounding_box()
                if box and box["width"] > 200 and box["height"] > 200:
                    print(f"Found treemap with selector: {selector}")
                    return element
        except Exception:
            continue

    print("No specific treemap element found, will capture main content area")
    return None



# Mapping: JSON industry name -> (iid, capture_key_suffix)
INDUSTRY_MAP = {
    "半導體": ("24", "semi"),
    "電子組件": ("28", "elec"),
    "電機": ("5", "electrical"),
    "電子通路": ("29", "channel"),
    "電腦週邊": ("25", "computer"),
    "營建": ("14", "construction"),
    "其他": ("20", "other"),
    "光電": ("26", "opto"),
    "化學": ("21", "chem"),
    "生技": ("22", "bio"),
    "汽車": ("12", "auto"),
    "電器": ("6", "cable"),
    "其他電子": ("31", "otherele"),
    "網通": ("27", "network"),
    "塑膠": ("3", "plastic"),
    "航運": ("15", "shipping"),
    "綠能環保": ("35", "green"),
    "資訊服務": ("30", "info"),
    "觀光": ("16", "tourism"),
}

# nStock heatmap URL configuration
# t1: 0=TSE (上市), 1=OTC (上櫃)
# t2=0: otc filter
# t3=0: additional filter
# t4=1: display mode
# t5=0: additional option
# iid: industry ID

# Industry map parameters: (t1_value, iid_value)
INDUSTRY_PARAMS = {
    # === TSE (上市) Categories ===
    "tse": (0, ""),  # 上市總覽 (Default)
    "tse-semi": (0, "24"),  # 上市半導體
    "tse-network": (0, "27"),  # 上市網通
    "tse-elec": (0, "28"),  # 上市電子組件
    "tse-computer": (0, "25"),  # 上市電腦週邊
    "tse-channel": (0, "29"),  # 上市電子通路
    "tse-plastic": (0, "3"),  # 上市塑膠
    "tse-electrical": (0, "5"),  # 上市電機
    "tse-construction": (0, "14"),  # 上市營建
    "tse-shipping": (0, "15"),  # 上市航運
    "tse-green": (0, "35"),  # 上市綠能環保
    "tse-opto": (0, "26"),  # 上市光電
    "tse-chem": (0, "21"),  # 上市化學
    "tse-bio": (0, "22"),  # 上市生技
    "tse-auto": (0, "12"),  # 上市汽車
    "tse-cable": (0, "6"),  # 上市電器
    "tse-otherele": (0, "31"),  # 上市其他電子
    "tse-other": (0, "20"),  # 上市其他

    # === OTC (上櫃) Categories ===
    "otc": (1, ""),  # 上櫃總覽
    "otc-semi": (1, "24"),  # 上櫃半導體
    "otc-network": (1, "27"),  # 上櫃網通
    "otc-elec": (1, "28"),  # 上櫃電子組件
    "otc-computer": (1, "25"),  # 上櫃電腦週邊
    "otc-channel": (1, "29"),  # 上櫃電子通路
    "otc-electrical": (1, "5"),  # 上櫃電機
    "otc-construction": (1, "14"),  # 上櫃營建
    "otc-other": (1, "20"),  # 上櫃其他
    "otc-info": (1, "30"),  # 上櫃資訊服務
    "otc-tourism": (1, "16"),  # 上櫃觀光
    "otc-green": (1, "35"),  # 上櫃綠能環保
    "otc-opto": (1, "26"),  # 上櫃光電
    "otc-chem": (1, "21"),  # 上櫃化學
    "otc-bio": (1, "22"),  # 上櫃生技
    "otc-cable": (1, "6"),  # 上櫃電器
    "otc-otherele": (1, "31"),  # 上櫃其他電子
}


def capture_twstock_heatmap(map_type="all", output_path="twstock.png", headless=True):
    """
    Capture nStock.tw heatmap as screenshot using Playwright.

    Args:
        map_type: Industry type (all, otc-elec, otc-semi)
        output_path: Path to save the screenshot
        headless: Run in headless mode (default: True)

    Returns:
        True if successful, False otherwise
    """
    check_dependencies()

    from playwright.sync_api import sync_playwright
    from PIL import Image
    import io as iolib

    # Construct full URL
    t1_value, iid_value = INDUSTRY_PARAMS.get(map_type, INDUSTRY_PARAMS["tse"])
    iid_param = f"iid={iid_value}" if iid_value else "iid"
    url = f"https://www.nstock.tw/market_index/heatmap?t1={t1_value}&t2=0&t3=0&t4=1&t5=0&{iid_param}&nh=0"

    print(f"Taiwan Stock Heatmap Screenshot (Playwright)")
    print(f"Type: {map_type}")
    print(f"URL: {url}")
    print(f"Output: {output_path}")
    print(f"Headless: {headless}\n")

    try:
        with sync_playwright() as p:
            # Launch browser with settings optimized for SSR pages
            print("Launching Chromium browser...")
            browser = p.chromium.launch(
                headless=headless,
                args=[
                    "--no-sandbox",
                    "--disable-blink-features=AutomationControlled",
                    "--disable-dev-shm-usage",
                    "--disable-web-security",
                    "--disable-features=IsolateOrigins,site-per-process",
                ],
            )

            # Create context with realistic settings
            context = browser.new_context(
                viewport={"width": 1920, "height": 1080},
                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
                locale="zh-TW",
                timezone_id="Asia/Taipei",
                color_scheme="light",
            )

            # Anti-detection measures
            context.add_init_script("""
                // Remove webdriver property
                Object.defineProperty(navigator, 'webdriver', {
                    get: () => undefined
                });
                
                // Mock plugins
                Object.defineProperty(navigator, 'plugins', {
                    get: () => [1, 2, 3, 4, 5]
                });
                
                // Mock languages
                Object.defineProperty(navigator, 'languages', {
                    get: () => ['zh-TW', 'zh', 'en-US', 'en']
                });
            """)

            page = context.new_page()

            # Navigate to nStock heatmap
            print("Loading nStock.tw heatmap page...")
            page.goto(url, wait_until="networkidle", timeout=60000)

            # Smart wait: detect actual treemap rendering instead of hard 20s sleep
            print("Waiting for heatmap to render...")
            treemap_selectors = [
                '[class*="treemap"] rect',
                '[class*="treemap"] canvas',
                '[class*="heatmap"] rect',
                'canvas',
                'svg rect',
            ]
            rendered = False
            for sel in treemap_selectors:
                try:
                    page.wait_for_selector(sel, state="visible", timeout=15000)
                    print(f"  Detected rendered element: {sel}")
                    rendered = True
                    break
                except Exception:
                    continue

            if not rendered:
                # Fallback: wait for network idle as a sign the page is done
                print("  No treemap element detected, waiting for network idle...")
                try:
                    page.wait_for_load_state("networkidle", timeout=15000)
                except Exception:
                    pass

            # Brief pause for animations/transitions to complete
            time.sleep(2)

            # Wait for any loading indicators to disappear
            try:
                page.wait_for_selector(".loading", state="hidden", timeout=3000)
            except Exception:
                pass

            # Check if page loaded successfully
            try:
                page_title = page.title()
                print(f"Page loaded: {page_title}")
            except Exception:
                print("Could not get page title")

            # Try to find specific treemap element
            print("Looking for heatmap element...")
            treemap = find_treemap_element(page)

            # Hide any floating elements that might interfere
            print("Cleaning up page for screenshot...")
            page.evaluate("""
                // Hide navigation, ads, and floating elements
                const selectorsToHide = [
                    'nav', 'header', '.navbar', '.nav-bar',
                    '.ad', '.ads', '[class*="advertisement"]',
                    '.popup', '.modal', '.overlay',
                    '[class*="cookie"]', '[class*="consent"]',
                    '.floating', '[style*="position: fixed"]'
                ];

                selectorsToHide.forEach(selector => {
                    document.querySelectorAll(selector).forEach(el => {
                        el.style.display = 'none';
                    });
                });

                // Also hide high z-index elements (tooltips, popups)
                document.querySelectorAll('*').forEach(el => {
                    const style = window.getComputedStyle(el);
                    const zIndex = parseInt(style.zIndex) || 0;
                    if (zIndex > 1000 && !el.closest('[class*="treemap"], [class*="heatmap"], [class*="chart"]')) {
                        el.style.display = 'none';
                    }
                });
            """)

            # Move mouse away to clear any hover effects
            page.mouse.move(10, 10)
            time.sleep(0.5)

            # Take screenshot
            print("Capturing screenshot...")

            if treemap:
                # Element-specific screenshot
                treemap.scroll_into_view_if_needed()
                time.sleep(0.5)
                screenshot_bytes = treemap.screenshot(type="png")
            else:
                # Full page with smart clipping
                # First, get the main content area dimensions
                clip_area = page.evaluate("""
                    () => {
                        // Try to find the main content container
                        const selectors = [
                            '.treemap-wrapper', '.heatmap-wrapper',
                            '.market-map', '.chart-area',
                            'main', '.main-content', '#app'
                        ];
                        
                        for (const sel of selectors) {
                            const el = document.querySelector(sel);
                            if (el) {
                                const rect = el.getBoundingClientRect();
                                if (rect.width > 400 && rect.height > 300) {
                                    return {
                                        x: rect.x,
                                        y: rect.y,
                                        width: rect.width,
                                        height: rect.height
                                    };
                                }
                            }
                        }
                        
                        // Default: capture viewport area below header
                        return {
                            x: 0,
                            y: 80,
                            width: 1920,
                            height: 900
                        };
                    }
                """)

                screenshot_bytes = page.screenshot(type="png", clip=clip_area)

            # Save screenshot
            with open(output_path, "wb") as f:
                f.write(screenshot_bytes)

            file_size = os.path.getsize(output_path)
            print(f"Screenshot saved: {output_path}")
            print(f"File size: {file_size:,} bytes")

            # Clean up
            browser.close()
            return True

    except Exception as e:
        print(f"\nError: {e}")
        import traceback

        traceback.print_exc()
        return False


def create_html(html_path, png_filename="twstock.png"):
    """Create simple HTML to display the screenshot."""

    html_content = f"""<!DOCTYPE html>
<html lang="zh-TW">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Taiwan Stock Market Heatmap</title>
    <style>
        * {{
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }}
        body {{
            background-color: #1a1a2e;
            display: flex;
            flex-direction: column;
            justify-content: center;
            align-items: center;
            min-height: 100vh;
            padding: 20px;
            font-family: 'Noto Sans TC', -apple-system, BlinkMacSystemFont, sans-serif;
        }}
        h1 {{
            color: #fff;
            margin-bottom: 20px;
            font-size: 1.5em;
        }}
        .timestamp {{
            color: #888;
            margin-bottom: 20px;
            font-size: 0.9em;
        }}
        img {{
            max-width: 100%;
            height: auto;
            display: block;
            border-radius: 8px;
            box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
        }}
        .source {{
            color: #666;
            margin-top: 20px;
            font-size: 0.8em;
        }}
        a {{
            color: #4ecdc4;
            text-decoration: none;
        }}
        a:hover {{
            text-decoration: underline;
        }}
    </style>
</head>
<body>
    <h1>Taiwan Stock Market Heatmap</h1>
    <p class="timestamp">Generated: <span id="time"></span></p>
    <img src="{png_filename}" alt="Taiwan Stock Market Heatmap">
    <p class="source">Data source: <a href="https://www.nstock.tw" target="_blank">nStock.tw</a></p>
    <script>
        document.getElementById('time').textContent = new Date().toLocaleString('zh-TW');
    </script>
</body>
</html>
"""

    with open(html_path, "w", encoding="utf-8") as f:
        f.write(html_content)

    print(f"HTML created: {html_path}")


def main():
    parser = argparse.ArgumentParser(
        description="Capture nStock.tw Taiwan stock heatmap as screenshot using Playwright"
    )
    parser.add_argument(
        "-t",
        "--type",
        default="all",
        help="Industry type (default: all - dynamically from losers JSON)",
    )
    parser.add_argument(
        "--no-html",
        action="store_true",
        help="Don't create HTML file, only save screenshot",
    )
    parser.add_argument(
        "--no-headless",
        action="store_true",
        help="Run with visible browser (for debugging)",
    )
    parser.add_argument(
        "-o",
        "--output",
        default=None,
        help="Output PNG filename (default: based on type)",
    )

    args = parser.parse_args()

    # Output paths - heatmaps directory
    script_dir = Path(__file__).parent.parent.parent.parent
    heatmaps_dir = script_dir / "heatmaps"
    
    # Create heatmaps directory if it doesn't exist
    heatmaps_dir.mkdir(exist_ok=True)

    # If 'all' is specified, dynamically determine categories from losers JSON
    if args.type == "all":
        # Clean old heatmaps before capturing new ones
        old_pngs = list(heatmaps_dir.glob("*.png"))
        if old_pngs:
            print(f"🗑️ Removing {len(old_pngs)} old heatmap(s)...", flush=True)
            for png in old_pngs:
                png.unlink()

        json_path = script_dir / "api" / "histock_top_losers.json"
        if not json_path.exists():
            print(f"❌ Losers JSON not found: {json_path}", flush=True)
            sys.exit(1)

        with open(json_path, "r", encoding="utf-8") as f:
            losers_data = json.load(f)

        # Count stocks per (market, industry) pair
        from collections import Counter
        pair_counts = Counter()
        for stock in losers_data["data"]:
            pair = (stock["market"], stock["industry"])
            pair_counts[pair] += 1

        # Map to capture keys (only include industries with >= 2 stocks)
        MIN_STOCKS = 2
        all_categories = []
        for (market, industry), count in sorted(pair_counts.items()):
            if count < MIN_STOCKS:
                continue
            if industry in INDUSTRY_MAP:
                iid, suffix = INDUSTRY_MAP[industry]
                key = f"{market}-{suffix}"
                if key in INDUSTRY_PARAMS:
                    all_categories.append(key)
                    print(f"  ✓ {key} ({industry}: {count} stocks)", flush=True)
                else:
                    print(f"⚠️ No params for key: {key}, skipping", flush=True)
            else:
                print(f"⚠️ Unknown industry: {industry} ({count} stocks), skipping", flush=True)

        all_categories = sorted(set(all_categories))

        if not all_categories:
            print("❌ No valid categories found from losers JSON", flush=True)
            sys.exit(1)

        print(f"📊 Capturing {len(all_categories)} heatmap categories (from losers JSON)...", flush=True)
        print(f"Categories: {', '.join(all_categories)}", flush=True)
        print(f"Output directory: {heatmaps_dir}\n", flush=True)
        
        start_time = time.time()
        headless = not args.no_headless
        failed_categories = []
        
        for i, category in enumerate(all_categories, 1):
            print(f"\n{'='*60}", flush=True)
            print(f"[{i}/{len(all_categories)}] Capturing: {category}", flush=True)
            print(f"{'='*60}", flush=True)
            
            # Determine filename
            png_filename = f"twstock_{category}.png" if category not in ["tse"] else "twstock.png"
            png_path = heatmaps_dir / png_filename
            
            # Capture heatmap
            success = capture_twstock_heatmap(category, str(png_path), headless=headless)
            
            if not success:
                print(f"❌ Failed to capture {category}", flush=True)
                failed_categories.append(category)
            else:
                print(f"✅ Successfully captured {category}", flush=True)
        
        # Summary
        end_time = time.time()
        elapsed_time = end_time - start_time
        minutes = int(elapsed_time // 60)
        seconds = int(elapsed_time % 60)
        
        print(f"\n{'='*60}", flush=True)
        print(f"📊 SUMMARY", flush=True)
        print(f"{'='*60}", flush=True)
        print(f"✅ Successful: {len(all_categories) - len(failed_categories)}/{len(all_categories)}", flush=True)
        if failed_categories:
            print(f"❌ Failed: {', '.join(failed_categories)}", flush=True)
        print(f"⏱️  Total time: {minutes}m {seconds}s", flush=True)
        
        # Create main HTML viewer if not disabled
        if not args.no_html:
            print(f"\n📄 Creating HTML viewer...", flush=True)
            html_path = script_dir / "index.html"
            create_html(str(html_path), "twstock.png")
            print(f"HTML: {html_path}", flush=True)
        
        if failed_categories:
            sys.exit(1)
        else:
            print(f"\n🎉 All heatmaps captured successfully!", flush=True)
            sys.exit(0)
    
    # Single category capture (original behavior)
    # Determine filename
    if args.output:
        png_filename = args.output
    else:
        png_filename = (
            f"twstock_{args.type}.png" if args.type not in ["tse", "all"] else "twstock.png"
        )

    png_path = heatmaps_dir / png_filename
    html_path = (
        script_dir / f"twstock_{args.type}.html"
        if args.type not in ["tse", "all"]
        else script_dir / "index.html"
    )

    print(f"Output directory: {heatmaps_dir}\n")

    # Capture heatmap screenshot
    headless = not args.no_headless
    success = capture_twstock_heatmap(args.type, str(png_path), headless=headless)

    if not success:
        print("\nFailed to capture screenshot")
        sys.exit(1)

    # Create HTML if requested
    if not args.no_html:
        print()
        create_html(str(html_path), png_filename)

    print(f"\nDone!")
    print(f"PNG: {png_path}")
    if not args.no_html:
        print(f"HTML: {html_path}")
        print(f"\nOpen {html_path} in your browser to view the heatmap.")

    sys.exit(0)



if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Scrape histock.tw top losers ranking page.
Outputs: api/histock_top_losers.json
Source: https://histock.tw/stock/rank.aspx?m=4&d=0&t=dt
"""

import csv
import json
import sys
import io
from pathlib import Path

# Fix Windows console encoding
if sys.platform == "win32":
    sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")
    sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace")

try:
    import requests
    from bs4 import BeautifulSoup
except ImportError:
    import subprocess
    subprocess.run([sys.executable, "-m", "pip", "install", "requests", "beautifulsoup4"], check=True)
    import requests
    from bs4 import BeautifulSoup


def load_stock_mapping(csv_path):
    """Load industry and market mapping from StockMapping.csv."""
    mapping = {}
    if not csv_path.exists():
        return mapping
    with open(csv_path, "r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            ticker = row.get("ticker", "").strip()
            if ticker:
                mapping[ticker] = {
                    "industry": row.get("industry", "").strip(),
                    "market": row.get("market", "").strip(),
                }
    return mapping


def scrape_histock_top_losers():
    """Scrape top 50 losers from histock.tw."""
    url = "https://histock.tw/stock/rank.aspx?m=4&d=0&t=dt"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "Accept-Language": "zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7",
    }

    print(f"Fetching: {url}")
    resp = requests.get(url, headers=headers, timeout=30)
    resp.raise_for_status()
    resp.encoding = "utf-8"

    soup = BeautifulSoup(resp.text, "html.parser")

    # Find the GridView table (ASP.NET control ID: CPHB1_gv)
    table = soup.find("table", id=lambda x: x and "gv" in x.lower())
    if not table:
        # Fallback: find by structure
        tables = soup.find_all("table")
        for t in tables:
            if t.find("th") and "代號" in t.get_text():
                table = t
                break

    if not table:
        print("ERROR: Could not find ranking table on page")
        sys.exit(1)

    rows = table.find_all("tr")
    print(f"Found {len(rows) - 1} data rows (excluding header)")

    # Load industry mapping
    project_root = Path(__file__).parent.parent.parent.parent
    csv_path = project_root / "data" / "StockMapping.csv"
    stock_mapping = load_stock_mapping(csv_path)
    print(f"Loaded {len(stock_mapping)} stocks from StockMapping.csv")

    results = []
    for row in rows[1:51]:  # Skip header, take first 50
        cells = row.find_all("td")
        if len(cells) < 5:
            continue

        # Column 0: ticker (may be inside <a> tag)
        ticker_el = cells[0].find("a")
        ticker = ticker_el.get_text(strip=True) if ticker_el else cells[0].get_text(strip=True)

        # Column 1: name
        name_el = cells[1].find("a")
        name = name_el.get_text(strip=True) if name_el else cells[1].get_text(strip=True)

        # Column 2: price
        price = cells[2].get_text(strip=True)

        # Column 4: change % (漲跌幅)
        change = cells[4].get_text(strip=True)

        # Lookup from StockMapping.csv
        stock_info = stock_mapping.get(ticker, {})
        industry = stock_info.get("industry", "") if stock_info else ""
        market = stock_info.get("market", "") if stock_info else ""

        results.append({
            "ticker": ticker,
            "name": name,
            "price": price,
            "change": change,
            "industry": industry,
            "market": market,
        })

    return results


def main():
    results = scrape_histock_top_losers()
    print(f"Scraped {len(results)} stocks")

    # Output JSON
    project_root = Path(__file__).parent.parent.parent.parent
    output_path = project_root / "api" / "histock_top_losers.json"
    output_path.parent.mkdir(parents=True, exist_ok=True)

    output = {
        "status": "success",
        "source": "https://histock.tw/stock/rank.aspx?m=4&d=0&t=dt",
        "count": len(results),
        "data": results,
    }

    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    print(f"Output: {output_path}")

    # Preview
    if results:
        print("\nTop 5 losers:")
        for s in results[:5]:
            print(f"  {s['ticker']} {s['name']:<6} {s['price']:>8} {s['change']:>8} {s['market']:<4} {s['industry']}")


if __name__ == "__main__":
    main()