Product Research

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Sorftime API 客户端 - 统一的数据采集接口

v2.2 - 修复大文件 JSON 解析问题

为 product-research Skill 提供简洁的 API 调用方法:
- 自动从 .mcp.json 读取 API Key
- SSE 响应解析
- Mojibake 编码修复
- 控制字符转义（在 Unicode 解码后执行）
- 返回干净的 Python dict

使用示例:
    from scripts.api_client import SorftimeClient

    client = SorftimeClient()

    # 获取类目 Top100
    top100 = client.get_category_report(site="US", node_id=12345)

    # 获取关键词详情
    keyword = client.get_keyword_detail(site="US", keyword="your keyword")

    # 获取产品详情
    product = client.get_product_detail(site="US", asin="B0XXXXXXXX")

    # 获取产品评论
    reviews = client.get_product_reviews(site="US", asin="B0XXXXXXXX", review_type="Negative")
"""

import os
import json
import re
import codecs
import subprocess
import sys
from datetime import datetime
from typing import Optional, Dict, List, Any
from pathlib import Path


# ============================================================================
# API 配置
# ============================================================================

def get_project_root():
    """获取项目根目录（.claude 的父目录）"""
    path = os.path.abspath(__file__)
    while path != os.path.dirname(path):
        if os.path.basename(path) == '.claude':
            return os.path.dirname(path)
        path = os.path.dirname(path)
    return os.getcwd()


def get_api_key():
    """
    从 .mcp.json 读取 Sorftime API Key

    Returns:
        str: API Key
    """
    project_root = get_project_root()
    mcp_config_path = os.path.join(project_root, '.mcp.json')

    if os.path.exists(mcp_config_path):
        try:
            with open(mcp_config_path, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()
            config = json.loads(content)

            # 从 URL 中提取 API key: https://mcp.sorftime.com?key=XXX
            sorftime_url = config.get('mcpServers', {}).get('sorftime', {}).get('url', '')
            if 'key=' in sorftime_url:
                api_key = sorftime_url.split('key=')[-1]
                if api_key:
                    return api_key
        except Exception as e:
            print(f"⚠ 读取 .mcp.json 失败: {e}")

    # 尝试环境变量
    api_key = os.environ.get('SORFTIME_API_KEY', '')
    if api_key:
        return api_key

    raise ValueError(
        "API Key 未找到。请确保:\n"
        "1. .mcp.json 文件存在并包含 sorftime 配置，或\n"
        "2. 设置环境变量 SORFTIME_API_KEY"
    )


# ============================================================================
# 数据处理工具函数
# ============================================================================

def safe_int(value, default=0):
    """安全转换为整数"""
    if isinstance(value, (int, float)):
        return int(value)
    if isinstance(value, str):
        cleaned = re.sub(r'[^\d.-]', '', value)
        try:
            return int(float(cleaned)) if cleaned else default
        except ValueError:
            return default
    return default


def safe_float(value, default=0.0):
    """安全转换为浮点数"""
    if isinstance(value, (int, float)):
        return float(value)
    if isinstance(value, str):
        cleaned = re.sub(r'[^\d.-]', '', value)
        try:
            return float(cleaned) if cleaned else default
        except ValueError:
            return default
    return default


def fix_mojibake(text):
    """
    修复 Mojibake 编码问题 (UTF-8/Latin-1 双重编码)

    问题: UTF-8 字节被错误解释为 Latin-1
    解决: 将错误编码的字符串重新编码为 Latin-1，然后用 UTF-8 解码
    """
    if isinstance(text, str):
        try:
            return text.encode('latin-1').decode('utf-8')
        except:
            return text
    elif isinstance(text, dict):
        return {fix_mojibake(k): fix_mojibake(v) for k, v in text.items()}
    elif isinstance(text, list):
        return [fix_mojibake(item) for item in text]
    return text


def escape_control_chars_in_json_strings(json_str):
    """
    转义 JSON 字符串值中的控制字符

    问题: API 返回的 JSON 字符串值中包含原始的换行符、制表符等控制字符
    解决: 在保持 JSON 结构不变的情况下，只转义字符串值内的控制字符
    """
    result = []
    i = 0
    in_string = False
    escape_next = False

    while i < len(json_str):
        c = json_str[i]

        if escape_next:
            result.append(c)
            escape_next = False
            i += 1
            continue

        if c == '\\':
            result.append(c)
            escape_next = True
            i += 1
            continue

        if c == '"':
            in_string = not in_string
            result.append(c)
            i += 1
            continue

        if in_string:
            if c == '\n':
                result.append('\\n')
            elif c == '\r':
                result.append('\\r')
            elif c == '\t':
                result.append('\\t')
            elif ord(c) < 32:
                result.append(' ')
            else:
                result.append(c)
        else:
            result.append(c)
        i += 1

    return ''.join(result)


def extract_json_object(text):
    """
    从文本中提取完整的 JSON 对象

    使用括号匹配算法，支持嵌套结构
    """
    stack = []
    start_idx = None

    for i, char in enumerate(text):
        if char in '{[':
            if not stack:
                start_idx = i
            stack.append(char)
        elif char in '}]':
            if stack:
                expected = '}' if char == '}' else ']'
                opening = '{' if expected == '}' else '['
                if stack[-1] == opening:
                    stack.pop()
                    if not stack:
                        json_str = text[start_idx:i+1]
                        try:
                            return json.loads(json_str)
                        except json.JSONDecodeError:
                            continue

    return None


def decode_sse_response(content):
    """
    解码 Sorftime SSE 响应

    处理流程:
    1. 清理控制字符
    2. 解析 SSE 格式 (event: message, data: {...})
    3. Unicode 解码
    4. Mojibake 修复
    5. 提取 JSON 对象

    Args:
        content: SSE 响应内容（字符串）

    Returns:
        dict: 解码后的数据
    """
    # 清理控制字符
    content = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x9f]', '', content)

    for line in content.split('\n'):
        if line.startswith('data: '):
            json_text = line[6:]  # 去掉 'data: ' 前缀
            try:
                data = json.loads(json_text)
                result_text = data.get('result', {}).get('content', [{}])[0].get('text', '')
                if result_text:
                    # Unicode 解码
                    decoded = codecs.decode(result_text, 'unicode-escape')

                    # Mojibake 修复
                    decoded = fix_mojibake(decoded)

                    # 转义 JSON 字符串值内的控制字符（关键步骤！）
                    decoded = escape_control_chars_in_json_strings(decoded)

                    # 清理剩余的控制字符
                    decoded = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f-\x9f]', '', decoded)

                    # 提取 JSON
                    json_obj = extract_json_object(decoded)
                    if json_obj:
                        return json_obj
            except Exception:
                continue

    # 如果 SSE 解析失败，尝试直接解析
    try:
        return json.loads(content)
    except:
        pass

    return None


# ============================================================================
# Sorftime API 客户端
# ============================================================================

class SorftimeClient:
    """
    Sorftime API 客户端

    提供简洁的方法调用 Sorftime MCP API
    """

    # API 工具名称映射
    TOOLS = {
        # 类目相关
        'search_categories_broadly': 'search_categories_broadly',  # 多维度广泛搜索类目
        'category_name_search': 'category_name_search',  # 按类目名称搜索（使用 searchName 参数）
        'category_report': 'category_report',
        'category_trend': 'category_trend',
        'category_keywords': 'category_keywords',

        # 关键词相关
        'keyword_detail': 'keyword_detail',
        'keyword_search_results': 'keyword_search_results',
        'keyword_extends': 'keyword_extends',
        'keyword_trend': 'keyword_trend',

        # 产品相关
        'product_detail': 'product_detail',
        'product_reviews': 'product_reviews',
        'product_traffic_terms': 'product_traffic_terms',
        'product_trend': 'product_trend',
        'product_search': 'product_search',

        # 选品相关
        'potential_product': 'potential_product',
        'competitor_product_keywords': 'competitor_product_keywords',

        # 供应链
        'ali1688': 'ali1688_similar_product',
    }

    def __init__(self, api_key: Optional[str] = None):
        """
        初始化客户端

        Args:
            api_key: Sorftime API Key，如果不提供则从 .mcp.json 读取
        """
        self.api_key = api_key or get_api_key()
        self.api_url = f'https://mcp.sorftime.com?key={self.api_key}'
        self.request_id = 0

    def _call(self, tool_name: str, arguments: Dict[str, Any]) -> tuple:
        """
        调用 Sorftime API

        Args:
            tool_name: API 工具名称
            arguments: API 参数

        Returns:
            tuple: (解析后的数据 dict, 原始响应 str)
        """
        self.request_id += 1

        payload = {
            "jsonrpc": "2.0",
            "id": self.request_id,
            "method": "tools/call",
            "params": {
                "name": tool_name,
                "arguments": arguments
            }
        }

        try:
            result = subprocess.run(
                ['curl', '-s', '-X', 'POST', self.api_url,
                 '-H', 'Content-Type: application/json',
                 '-H', 'Accept: application/json, text/event-stream',
                 '-d', json.dumps(payload)],
                capture_output=True,
                text=True,
                timeout=60,
                check=True
            )

            # 返回原始响应和解析后的数据
            raw_response = result.stdout
            data = decode_sse_response(raw_response)

            if data is None:
                # 即使解析失败，也返回原始响应供调试
                return None, raw_response

            return data, raw_response

        except subprocess.CalledProcessError as e:
            raise RuntimeError(f"API 调用失败: {e}")
        except subprocess.TimeoutExpired:
            raise RuntimeError(f"API 调用超时")

    # ========================================================================
    # 类目相关 API
    # ========================================================================

    def search_category_by_product_name(
        self,
        site: str,
        product_name: str
    ) -> Dict[str, Any]:
        """
        按产品名称搜索类目

        Args:
            site: 站点 (US, GB, DE, FR, IT, ES, CA, JP, etc.)
            product_name: 产品名称

        Returns:
            dict: 搜索结果，包含类目列表
        """
        return self._call(
            self.TOOLS['category_name_search'],
            {"amzSite": site, "searchName": product_name}  # 注意: 参数是 searchName
        )

    def search_category_by_name(
        self,
        site: str,
        category_name: str
    ) -> Dict[str, Any]:
        """
        按类目名称搜索（别名方法，与 search_category_by_product_name 相同）

        Args:
            site: 站点
            category_name: 类目名称

        Returns:
            dict: 搜索结果
        """
        return self.search_category_by_product_name(site, category_name)

    def search_categories_broadly(
        self,
        site: str,
        filters: Optional[Dict[str, Any]] = None
    ) -> Dict[str, Any]:
        """
        多维度广泛搜索类目（新增 - 用于蓝海发现）

        Args:
            site: 站点 (US, GB, DE, FR, IT, ES, CA, JP, etc.)
            filters: 筛选条件（可选）
                - top3Product_sales_share: Top3 产品销量占比上限（如 0.4 表示<40%）
                - top3Brands_sales_share: Top3 品牌销量占比上限
                - newProductSalesAmountShare: 新品销量占比下限（如 0.15 表示>15%）
                - brandCount: 品牌数量下限（如 80 表示>80 个品牌）
                - priceRange_min: 价格范围下限
                - priceRange_max: 价格范围上限
                - monthlySales_min: 月销量下限
                - monthlySales_max: 月销量上限

        Returns:
            dict: 类目列表，包含：
                - categories: 类目列表
                - total: 总数
        """
        params = {"amzSite": site}
        if filters:
            params.update(filters)
        return self._call(
            self.TOOLS['search_categories_broadly'],
            params
        )

    def get_category_report(
        self,
        site: str,
        node_id: int
    ) -> Dict[str, Any]:
        """
        获取类目 Top100 报告

        Args:
            site: 站点
            node_id: 类目 Node ID

        Returns:
            dict: Top100 产品数据
        """
        return self._call(
            self.TOOLS['category_report'],
            {"amzSite": site, "nodeId": str(node_id)}
        )

    def get_category_trend(
        self,
        site: str,
        node_id: int,
        trend_index: str = "NewProductSalesAmountShare"
    ) -> Dict[str, Any]:
        """
        获取类目趋势数据

        Args:
            site: 站点
            node_id: 类目 Node ID
            trend_index: 趋势类型
                - NewProductSalesAmountShare: 新品销量占比
                - NewProductProductShare: 新品数量占比
                - etc.

        Returns:
            dict: 结构化趋势数据
                {
                    "trend_data": [
                        {"date": "2024-03", "value": 33.35},
                        ...
                    ],
                    "metric": "新品占比",
                    "node_id": "99530371011"
                }
        """
        raw_data, raw_response = self._call(
            self.TOOLS['category_trend'],
            {"amzSite": site, "nodeId": str(node_id), "trendIndex": trend_index}
        )

        # 转换原始格式为结构化格式
        # 原始格式: ["2024年03月=33.35", "2024年04月=27.94", ...]
        # 目标格式: {"trend_data": [{"date": "2024-03", "value": 33.35}, ...]}
        if isinstance(raw_data, list):
            trend_data = []
            for item in raw_data:
                if isinstance(item, str) and '=' in item:
                    # 解析 "2024年03月=33.35" 格式
                    date_str, value_str = item.split('=', 1)
                    # 转换日期格式: "2024年03月" -> "2024-03"
                    date_match = re.search(r'(\d{4})年(\d{2})月', date_str)
                    if date_match:
                        year, month = date_match.groups()
                        formatted_date = f"{year}-{month}"
                        try:
                            value = float(value_str)
                            trend_data.append({
                                "date": formatted_date,
                                "value": value
                            })
                        except ValueError:
                            continue

            # 指标名称映射
            metric_names = {
                "NewProductSalesAmountShare": "新品销量占比",
                "NewProductProductShare": "新品数量占比",
            }

            return {
                "trend_data": trend_data,
                "metric": metric_names.get(trend_index, trend_index),
                "node_id": str(node_id),
                "site": site
            }

        return raw_data

    def get_category_keywords(
        self,
        site: str,
        node_id: int,
        page: int = 1
    ) -> Dict[str, Any]:
        """
        获取类目关键词

        Args:
            site: 站点
            node_id: 类目 Node ID
            page: 页码

        Returns:
            dict: 关键词数据
        """
        return self._call(
            self.TOOLS['category_keywords'],
            {"amzSite": site, "nodeId": str(node_id), "page": page}
        )

    # ========================================================================
    # 关键词相关 API
    # ========================================================================

    def get_keyword_detail(
        self,
        site: str,
        keyword: str
    ) -> Dict[str, Any]:
        """
        获取关键词详情

        Args:
            site: 站点
            keyword: 关键词

        Returns:
            dict: 关键词详情（搜索量、CPC、自然位产品等）
        """
        return self._call(
            self.TOOLS['keyword_detail'],
            {"amzSite": site, "keyword": keyword}
        )

    def get_keyword_search_results(
        self,
        site: str,
        keyword: str
    ) -> Dict[str, Any]:
        """
        获取关键词搜索结果（自然位产品）

        Args:
            site: 站点
            keyword: 关键词

        Returns:
            dict: 自然位产品列表
        """
        return self._call(
            self.TOOLS['keyword_search_results'],
            {"amzSite": site, "searchKeyword": keyword}
        )

    def get_keyword_extends(
        self,
        site: str,
        keyword: str
    ) -> Dict[str, Any]:
        """
        获取关键词延伸词

        Args:
            site: 站点
            keyword: 关键词

        Returns:
            dict: 延伸词列表
        """
        return self._call(
            self.TOOLS['keyword_extends'],
            {"amzSite": site, "keyword": keyword}
        )

    # ========================================================================
    # 产品相关 API
    # ========================================================================

    def get_product_detail(
        self,
        site: str,
        asin: str
    ) -> Dict[str, Any]:
        """
        获取产品详情

        Args:
            site: 站点
            asin: 产品 ASIN

        Returns:
            dict: 产品详情
        """
        return self._call(
            self.TOOLS['product_detail'],
            {"amzSite": site, "asin": asin}
        )

    def get_product_reviews(
        self,
        site: str,
        asin: str,
        review_type: str = "Both"
    ) -> Dict[str, Any]:
        """
        获取产品评论

        Args:
            site: 站点
            asin: 产品 ASIN
            review_type: 评论类型 (Both, Positive, Negative)

        Returns:
            dict: 评论列表
        """
        return self._call(
            self.TOOLS['product_reviews'],
            {"amzSite": site, "asin": asin, "reviewType": review_type}
        )

    def get_product_traffic_terms(
        self,
        site: str,
        asin: str
    ) -> Dict[str, Any]:
        """
        获取产品流量关键词（反查）

        Args:
            site: 站点
            asin: 产品 ASIN

        Returns:
            dict: 流量关键词列表
        """
        return self._call(
            self.TOOLS['product_traffic_terms'],
            {"amzSite": site, "asin": asin}
        )

    def get_product_trend(
        self,
        site: str,
        asin: str
    ) -> Dict[str, Any]:
        """
        获取产品趋势

        Args:
            site: 站点
            asin: 产品 ASIN

        Returns:
            dict: 趋势数据
        """
        return self._call(
            self.TOOLS['product_trend'],
            {"amzSite": site, "asin": asin}
        )

    def search_products(
        self,
        site: str,
        search_name: str,
        **filters
    ) -> Dict[str, Any]:
        """
        搜索产品

        Args:
            site: 站点
            search_name: 搜索关键词
            **filters: 筛选条件

        Returns:
            dict: 搜索结果
        """
        params = {"amzSite": site, "searchName": search_name}
        params.update(filters)
        return self._call(self.TOOLS['product_search'], params)

    # ========================================================================
    # 选品相关 API
    # ========================================================================

    def get_potential_products(
        self,
        site: str,
        search_name: str,
        **filters
    ) -> Dict[str, Any]:
        """
        获取潜力产品

        Args:
            site: 站点
            search_name: 搜索关键词
            **filters: 筛选条件

        Returns:
            dict: 潜力产品列表
        """
        params = {"amzSite": site, "searchName": search_name}
        params.update(filters)
        return self._call(self.TOOLS['potential_product'], params)

    def get_competitor_keywords(
        self,
        site: str,
        asin: str
    ) -> Dict[str, Any]:
        """
        获取竞品关键词布局

        Args:
            site: 站点
            asin: 产品 ASIN

        Returns:
            dict: 竞品关键词布局
        """
        return self._call(
            self.TOOLS['competitor_product_keywords'],
            {"amzSite": site, "asin": asin}
        )

    # ========================================================================
    # 供应链 API
    # ========================================================================

    def get_1688_products(
        self,
        search_name: str
    ) -> Dict[str, Any]:
        """
        获取 1688 相似产品

        Args:
            search_name: 搜索关键词

        Returns:
            dict: 1688 产品列表
        """
        return self._call(
            self.TOOLS['ali1688'],
            {"searchName": search_name}
        )


# ============================================================================
# 便捷函数
# ============================================================================

def create_client() -> SorftimeClient:
    """创建 Sorftime 客户端（便捷函数）"""
    return SorftimeClient()


# ============================================================================
# 命令行接口
# ============================================================================

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Sorftime API 客户端")
    parser.add_argument("tool", choices=[
        "category_report", "keyword_detail", "product_detail",
        "product_reviews", "category_trend"
    ], help="API 工具名称")
    parser.add_argument("--site", default="US", help="站点")
    parser.add_argument("--node-id", type=int, help="类目 Node ID")
    parser.add_argument("--keyword", help="关键词")
    parser.add_argument("--asin", help="产品 ASIN")
    parser.add_argument("--output", "-o", help="输出文件路径")

    args = parser.parse_args()

    client = SorftimeClient()

    if args.tool == "category_report":
        if not args.node_id:
            parser.error("--node-id 是必需的")
        result = client.get_category_report(args.site, args.node_id)

    elif args.tool == "keyword_detail":
        if not args.keyword:
            parser.error("--keyword 是必需的")
        result = client.get_keyword_detail(args.site, args.keyword)

    elif args.tool == "product_detail":
        if not args.asin:
            parser.error("--asin 是必需的")
        result = client.get_product_detail(args.site, args.asin)

    elif args.tool == "product_reviews":
        if not args.asin:
            parser.error("--asin 是必需的")
        result = client.get_product_reviews(args.site, args.asin)

    elif args.tool == "category_trend":
        if not args.node_id:
            parser.error("--node-id 是必需的")
        result = client.get_category_trend(args.site, args.node_id)

    # 输出结果
    if args.output:
        with open(args.output, 'w', encoding='utf-8') as f:
            json.dump(result, f, ensure_ascii=False, indent=2)
        print(f"✓ 结果已保存到: {args.output}")
    else:
        print(json.dumps(result, ensure_ascii=False, indent=2))

#!/usr/bin/env python3
"""
数据采集脚本 - product-research 技能

优化版本 v3.1 - 完全通用化（移除硬编码类别词）

使用方法:
    python collect_data.py "your keyword" US

或直接导入:
    from collect_data import collect_data
    result = collect_data("your keyword", "US")
"""

import sys
import os
import json
import re
from datetime import datetime

# 添加脚本目录到路径
script_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, script_dir)

from api_client import SorftimeClient


def create_output_dir(keyword, site):
    """创建输出目录（使用项目根目录）"""
    date_str = datetime.now().strftime('%Y%m%d')
    safe_keyword = keyword.replace(' ', '_').replace('/', '_')

    # 获取项目根目录
    # 脚本路径：.claude/skills/product-research/scripts/collect_data.py
    # 需要向上四级：scripts → product-research → skills → .claude → amazon-mcp
    current_dir = os.path.dirname(os.path.abspath(__file__))
    project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(current_dir))))

    output_dir = os.path.join(project_root, 'product-research-reports', f'{safe_keyword}_{site}_{date_str}')
    raw_dir = os.path.join(output_dir, 'raw')
    os.makedirs(raw_dir, exist_ok=True)
    return output_dir, raw_dir, date_str


def save_json(data, filepath):
    """安全保存 JSON 文件"""
    try:
        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        return True
    except Exception as e:
        print(f"  ✗ 保存失败：{e}")
        return False


def discover_blue_ocean_categories(client, site, keyword, max_categories=5):
    """
    【新增】蓝海市场发现 - 使用 search_categories_broadly

    Args:
        client: SorftimeClient 实例
        site: 站点
        keyword: 产品关键词（用于筛选相关类目）
        max_categories: 返回的类目数量

    Returns:
        list: 符合条件的类目列表
    """
    print("\n[Step 0.5] 蓝海市场发现...")

    # 筛选条件：适合新卖家的蓝海市场
    filters = {
        # 低集中度
        "top3Product_sales_share": 0.4,  # Top3 产品销量占比 < 40%
        "top3Brands_sales_share": 0.5,   # Top3 品牌销量占比 < 50%
        # 新品活跃
        "newProductSalesAmountShare": 0.15,  # 新品销量占比 > 15%
        # 市场分散
        "brandCount": 50,  # 品牌数量 > 50
        # 价格适中
        "priceRange_min": 10,
        "priceRange_max": 50,
        # 有一定规模
        "monthlySales_min": 5000,
    }

    try:
        result, _ = client.search_categories_broadly(site, filters)

        if result and isinstance(result, dict):
            categories = result.get('categories', [])

            # 过滤与关键词相关的类目
            if keyword:
                keyword_lower = keyword.lower()
                related_categories = []
                for cat in categories:
                    cat_name = cat.get('categoryName', '').lower()
                    if keyword_lower in cat_name or keyword_lower in cat.get('description', '').lower():
                        related_categories.append(cat)

                categories = related_categories[:max_categories]
            else:
                categories = categories[:max_categories]

            print(f"  ✓ 发现 {len(categories)} 个潜力类目:")
            for i, cat in enumerate(categories, 1):
                print(f"    {i}. {cat.get('categoryName', 'N/A')} "
                      f"(新品占比：{cat.get('newProductSalesAmountShare', 0)*100:.1f}%, "
                      f"Top3 占比：{cat.get('top3Product_sales_share', 0)*100:.1f}%)")

            return categories
        else:
            print(f"  ⚠ 未找到符合条件的类目")
            return []

    except Exception as e:
        print(f"  ⚠ 蓝海发现失败：{e} (非关键，继续执行)")
        return []


def find_potential_products(client, site, keyword, max_products=20):
    """
    【新增】潜力产品发现 - 使用 potential_product

    Args:
        client: SorftimeClient 实例
        site: 站点
        keyword: 产品关键词
        max_products: 返回的产品数量

    Returns:
        list: 潜力产品列表
    """
    print(f"\n[Step 1.3] 潜力产品发现：{keyword}...")

    # 筛选条件：有潜力的新品
    filters = {
        "monthlySales_min": 500,    # 月销量 > 500
        "price_min": 10,            # 价格 > $10
        "price_max": 50,            # 价格 < $50
        "rating_min": 4.0,          # 评分 > 4.0
        "daysOnMarket_max": 180,    # 上架时间 < 6 个月
    }

    try:
        result, _ = client.get_potential_products(site, keyword, **filters)

        if result and isinstance(result, dict):
            products = result.get('products', []) or result.get('productList', [])

            if not products:
                # 尝试不同的返回格式
                products = result.get('list', [])

            print(f"  ✓ 发现 {len(products)} 个潜力产品")

            # 显示 Top 5
            for i, p in enumerate(products[:5], 1):
                asin = p.get('ASIN', 'N/A')
                brand = p.get('品牌', 'N/A')
                sales = p.get('月销量', 'N/A')
                price = p.get('价格', 'N/A')
                rating = p.get('星级', 'N/A')
                days = p.get('上线天数', 'N/A')
                print(f"    {i}. {asin} | {brand} | 月销{sales} | ${price} | {rating}星 | {days}天")

            return products[:max_products]
        else:
            print(f"  ⚠ 未找到潜力产品")
            return []

    except Exception as e:
        print(f"  ⚠ 潜力产品发现失败：{e} (非关键，继续执行)")
        return []


def get_keyword_extends_data(client, site, keyword):
    """
    【新增】获取关键词延伸词 - 用于维度发现

    Args:
        client: SorftimeClient 实例
        site: 站点
        keyword: 关键词

    Returns:
        dict: 延伸词数据
    """
    print(f"\n[Step 1.4] 获取关键词延伸词：{keyword}...")

    try:
        result, _ = client.get_keyword_extends(site, keyword)

        if result:
            # 解析延伸词
            extends = result.get('extends', []) or result.get('keywords', []) or result.get('list', [])

            if isinstance(extends, list) and len(extends) > 0:
                print(f"  ✓ 获取 {len(extends)} 个延伸词")

                # 提取高频修饰词（用于维度发现）
                modifiers = []
                for item in extends:
                    if isinstance(item, dict):
                        word = item.get('keyword', item.get('word', ''))
                        search_volume = item.get('searchVolume', item.get('monthly_search', 0))
                    else:
                        word = str(item)
                        search_volume = 0

                    # 过滤掉品类通用词
                    if word and keyword.lower() not in word.lower():
                        modifiers.append({
                            'word': word,
                            'search_volume': search_volume
                        })

                # 按搜索量排序
                modifiers.sort(key=lambda x: x['search_volume'], reverse=True)

                print(f"  ✓ 提取 {len(modifiers)} 个修饰词（用于维度发现）")
                if modifiers:
                    print(f"    Top 5 修饰词：{', '.join([m['word'] for m in modifiers[:5]])}")

                return {
                    'extends': extends,
                    'modifiers': modifiers[:20]  # 保留 Top 20
                }

        print(f"  ⚠ 延伸词数据为空")
        return {}

    except Exception as e:
        print(f"  ⚠ 延伸词获取失败：{e} (非关键，继续执行)")
        return {}


def collect_data(keyword, site='US', max_keywords=3, use_blue_ocean=False):
    """
    执行完整的数据采集流程

    Args:
        keyword: 产品/类目关键词
        site: 站点代码 (US, GB, DE, etc.)
        max_keywords: 采集关键词数量
        use_blue_ocean: 是否启用蓝海发现模式

    Returns:
        dict: 采集结果摘要
    """
    print(f"🔍 选品数据采集：{keyword} ({site})")
    print("=" * 60)

    # 初始化
    client = SorftimeClient()
    output_dir, raw_dir, date_str = create_output_dir(keyword, site)

    # 结果摘要
    result = {
        'keyword': keyword,
        'site': site,
        'date': date_str,
        'category_name': None,
        'node_id': None,
        'steps_completed': [],
        'errors': [],
        'blue_ocean_categories': [],
        'potential_products': [],
        'keyword_extends': {}
    }

    # ========== Step 0.5: 蓝海市场发现（可选） ==========
    if use_blue_ocean:
        blue_ocean_cats = discover_blue_ocean_categories(client, site, keyword)
        if blue_ocean_cats:
            result['blue_ocean_categories'] = blue_ocean_cats
            save_json(blue_ocean_cats, os.path.join(raw_dir, 'blue_ocean_categories.json'))
            result['steps_completed'].append('blue_ocean_discovery')

    # ========== Step 1: 搜索类目 ==========
    print("\n[Step 1] 搜索类目...")

    category_result = None
    used_keyword = keyword

    try:
        print(f"  搜索: '{keyword}'...", end=' ')
        category_result, raw = client.search_category_by_product_name(site, keyword)

        if category_result and isinstance(category_result, list) and len(category_result) > 0:
            print(f"✓ 找到 {len(category_result)} 个类目")
        else:
            error_msg = f"类目搜索失败：未找到与 '{keyword}' 匹配的类目。请使用该类别最通用的核心名词（如使用 'camera' 而非 'digital wireless camera'）"
            print(f"  ✗ {error_msg}")
            raise Exception(error_msg)
    except Exception as e:
        print(f"  ✗ 错误: {str(e)}")
        raise

    # 使用找到的类目
    first_cat = category_result[0]
    node_id = first_cat.get('nodeId') or first_cat.get('NodeId')
    category_name = first_cat.get('categoryName') or first_cat.get('Name')

    result['category_name'] = category_name
    result['node_id'] = str(node_id)
    result['searched_keyword'] = used_keyword  # 记录实际使用的搜索词

    print(f"  ✓ 最终类目：{category_name}")
    print(f"  ✓ Node ID: {node_id}")
    if used_keyword != keyword:
        print(f"  ℹ 使用搜索词: '{used_keyword}' (原词: '{keyword}')")

    save_json(category_result, os.path.join(raw_dir, 'category_info.json'))
    result['steps_completed'].append('category_search')

    # ========== Step 2: 获取 Top100 ==========
    print(f"\n[Step 2] 获取 Top100 产品数据...")
    top100 = None
    try:
        top100, raw_response = client.get_category_report(site, result['node_id'])

        # 检查返回的数据是否有效
        if top100 is None or not isinstance(top100, dict) or len(top100) == 0:
            raise ValueError("category_report 返回无效数据")

        products = top100.get('Top100产品', []) or top100.get('Top100 产品', []) or top100.get('products', [])
        stats = top100.get('类目统计报告', {})

        print(f"  ✓ 产品数量：{len(products)}")
        if stats:
            monthly_revenue = stats.get('top100 产品月销额', 0)
            print(f"  ✓ 类目月销额：${monthly_revenue}")

        save_json(top100, os.path.join(raw_dir, 'top100.json'))
        result['steps_completed'].append('top100')

    except Exception as e:
        # category_report 不可用时，使用 product_search 作为替代
        print(f"  ⚠ category_report 不可用，尝试使用 product_search 替代...")
        try:
            # 使用 product_search 工具获取产品数据
            search_result, _ = client._call('product_search', {
                'amzSite': site,
                'searchName': keyword,
                'page': 1
            })

            if isinstance(search_result, list) and len(search_result) > 0:
                # 构造类似 top100 的数据结构
                products = search_result

                # 计算类目统计数据
                total_monthly_sales = sum(p.get('月销量', 0) for p in products)
                total_monthly_revenue = sum(p.get('月销额', 0) for p in products)
                avg_price = total_monthly_revenue / len(products) if products else 0

                top100_data = {
                    'Top100产品': products,
                    '类目统计报告': {
                        'top100 产品月销额': total_monthly_revenue,
                        'top100 产品月销量': total_monthly_sales,
                        '平均价格': avg_price,
                        '产品数量': len(products),
                        '数据来源': 'product_search (替代 category_report)'
                    }
                }

                print(f"  ✓ 产品数量：{len(products)}")
                print(f"  ✓ 类目月销额：${total_monthly_revenue:,.2f}")
                print(f"  ℹ 注意：使用 product_search 数据（非完整 Top100）")

                save_json(top100_data, os.path.join(raw_dir, 'top100.json'))
                result['steps_completed'].append('top100')
            else:
                error_msg = f"product_search 返回空数据"
                print(f"  ✗ {error_msg}")
                result['errors'].append(error_msg)

        except Exception as e2:
            error_msg = f"Top100 获取失败（category_report 和 product_search 都失败）：{e}, {e2}"
            print(f"  ✗ {error_msg}")
            result['errors'].append(error_msg)

    # ========== Step 3: 获取趋势数据 ==========
    print(f"\n[Step 3] 获取类目趋势...")
    try:
        trend = client.get_category_trend(site, result['node_id'])
        if trend:
            print(f"  ✓ 趋势数据已获取")
            save_json(trend, os.path.join(raw_dir, 'trend.json'))
            result['steps_completed'].append('trend')
        else:
            print(f"  ⚠ 趋势数据为空（非关键）")
    except Exception as e:
        error_msg = f"趋势获取失败：{e}"
        print(f"  ⚠ {error_msg} (非关键)")
        result['errors'].append(error_msg)

    # ========== Step 4: 获取关键词详情（通用关键词生成） ==========
    print(f"\n[Step 4] 获取关键词详情...")
    keywords_data = {}

    def generate_keyword_variants(base_kw, max_count=5):
        """
        通用关键词变体生成策略

        策略：
        1. 原始词
        2. 尝试生成复数形式
        3. 添加常见修饰前缀
        """
        variants = [base_kw]

        # 复数形式生成（通用规则）
        # 规则1: 添加 's'
        if not base_kw.endswith('s'):
            variants.append(base_kw + 's')
        # 规则2: 以 y 结尾，变 'ies'
        if base_kw.endswith('y') and len(base_kw) > 1:
            variants.append(base_kw[:-1] + 'ies')
        # 规则3: 以 s, x, ch, sh 结尾，添加 'es'
        if base_kw.endswith(('s', 'x', 'ch', 'sh')):
            variants.append(base_kw + 'es')

        # 添加常见修饰前缀（完全通用）
        common_prefixes = ['portable', 'wireless', 'digital', 'smart']
        for prefix in common_prefixes:
            variants.append(f"{prefix} {base_kw}")

        # 去重并限制数量
        seen = set()
        unique_variants = []
        for v in variants:
            v_lower = v.lower().strip()
            if v_lower and v_lower not in seen and len(unique_variants) < max_count:
                seen.add(v_lower)
                unique_variants.append(v)

        return unique_variants

    base_keywords = generate_keyword_variants(keyword, max_keywords)

    for kw in base_keywords:
        try:
            print(f"  - {kw}...", end=' ', flush=True)
            kw_data, _ = client.get_keyword_detail(site, kw)
            if kw_data:
                keywords_data[kw] = kw_data
                print("✓")
            else:
                print("✗ (空响应)")
        except Exception as e:
            print(f"✗ ({str(e)[:50]})")

    if keywords_data:
        save_json(keywords_data, os.path.join(raw_dir, 'keywords.json'))
        result['steps_completed'].append('keywords')
        print(f"  ✓ 成功：{len(keywords_data)}/{len(base_keywords)} 个关键词")

    # ========== Step 5: 获取关键词延伸词（新增） ==========
    extends_data = get_keyword_extends_data(client, site, keyword)
    if extends_data:
        result['keyword_extends'] = extends_data
        save_json(extends_data, os.path.join(raw_dir, 'keyword_extends.json'))
        result['steps_completed'].append('keyword_extends')

    # ========== Step 6: 发现潜力产品（新增） ==========
    potential_products = find_potential_products(client, site, keyword)
    if potential_products:
        result['potential_products'] = potential_products
        save_json(potential_products, os.path.join(raw_dir, 'potential_products.json'))
        result['steps_completed'].append('potential_products')

    # ========== Step 7: 保存汇总数据 ==========
    print(f"\n[Step 7] 保存汇总数据...")

    summary = {
        "metadata": {
            "keyword": keyword,
            "site": site,
            "date": date_str,
            "node_id": result['node_id'],
            "category_name": result['category_name'],
            "collected_at": datetime.now().isoformat()
        },
        "files": {
            "category_info": "raw/category_info.json",
            "top100": "raw/top100.json",
            "trend": "raw/trend.json",
            "keywords": "raw/keywords.json",
            "keyword_extends": "raw/keyword_extends.json" if extends_data else None,
            "potential_products": "raw/potential_products.json" if potential_products else None,
            "blue_ocean_categories": "raw/blue_ocean_categories.json" if result['blue_ocean_categories'] else None
        },
        "status": "success" if len(result['errors']) == 0 else "partial",
        "steps_completed": result['steps_completed'],
        "errors": result['errors'],
        # 预留 Dashboard 需要的数据结构（初始为空，由后续分析填充）
        "market_overview": {},
        "price_ranges": [],
        "product_types": [],
        "cross_analysis": {"price_type_matrix": []},
        "top_brands": [],
        "competitors": [],
        "voc_analysis": {"dimensions": [], "summary": ""},
        "barriers": [],
        "decision": {},
        "trend_data": [],
        "keywords": {}
    }

    save_json(summary, os.path.join(output_dir, 'data.json'))

    # ========== 完成 ==========
    print("\n" + "=" * 60)
    print(f"✓ 数据采集完成!")
    print(f"  输出目录：{output_dir}")
    print(f"  完成步骤：{', '.join(result['steps_completed'])}")

    if result['errors']:
        print(f"\n⚠ 错误 ({len(result['errors'])}):")
        for err in result['errors']:
            print(f"  - {err}")

    print("=" * 60)

    return result


# ============================================================================
# 命令行接口
# ============================================================================

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(
        description="product-research 数据采集脚本（通用版本）",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
示例:
  python collect_data.py "speaker" US
  python collect_data.py "sofa" DE --keywords 5
  python collect_data.py "mat" US --blue-ocean  # 启用蓝海发现
        """
    )

    parser.add_argument('keyword', help='产品/类目关键词')
    parser.add_argument('site', nargs='?', default='US', help='站点代码 (默认：US)')
    parser.add_argument('--keywords', '-k', type=int, default=3, help='采集关键词数量 (默认：3)')
    parser.add_argument('--blue-ocean', action='store_true', help='启用蓝海发现模式')

    args = parser.parse_args()

    collect_data(args.keyword, args.site, args.keywords, use_blue_ocean=args.blue_ocean)

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
数据验证和修复脚本 - 确保 data.json 结构正确

用法:
    python fix_data_json.py path/to/data.json
    python fix_data_json.py path/to/data.json --fix
"""

import sys
import os
import json
import argparse
from datetime import datetime
from pathlib import Path


def validate_data(data: dict) -> tuple[bool, list[str]]:
    """验证数据结构"""
    errors = []
    warnings = []

    # 必需字段检查
    required_fields = ['metadata', 'market_overview']
    for field in required_fields:
        if field not in data:
            errors.append(f"缺少必需字段: {field}")

    # metadata 检查
    if 'metadata' in data:
        metadata = data['metadata']
        required_metadata = ['category', 'site', 'date']
        for field in required_metadata:
            if field not in metadata:
                warnings.append(f"metadata 缺少字段: {field}")

    # market_overview 检查
    if 'market_overview' in data:
        mo = data['market_overview']
        required_mo = ['top100_monthly_sales', 'top100_monthly_revenue', 'avg_price']
        for field in required_mo:
            if field not in mo:
                warnings.append(f"market_overview 缺少字段: {field}")

    # go_nogo 检查
    if 'go_nogo' not in data:
        errors.append("缺少 go_nogo 字段")
    else:
        gogono = data['go_nogo']
        if 'overall_score' not in gogono and 'total_score' not in gogono:
            warnings.append("go_nogo 缺少评分字段")
        if 'decision' not in gogono and 'verdict' not in gogono:
            warnings.append("go_nogo 缺少决策字段")

    # dimensions 检查
    if 'dimensions' in data and data['dimensions']:
        # 检查每个维度是否有正确的结构
        for i, dim in enumerate(data['dimensions']):
            if 'dimension' not in dim and 'name' not in dim:
                warnings.append(f"dimensions[{i}] 缺少 'dimension' 或 'name' 字段")

    # voc_analysis 检查
    if 'voc_analysis' in data and data['voc_analysis']:
        voc = data['voc_analysis']
        if 'dimensions' not in voc:
            warnings.append("voc_analysis 缺少 'dimensions' 字段")

    is_valid = len(errors) == 0
    return is_valid, errors + warnings


def fix_data(data: dict) -> dict:
    """修复常见的数据结构问题"""
    # 修复 go_nogo 字段名称
    if 'go_nogo' in data:
        gogono = data['go_nogo']
        if 'verdict' in gogono and 'decision' not in gogono:
            gogono['decision'] = gogono['verdict']
        if 'total_score' in gogono and 'overall_score' not in gogono:
            gogono['overall_score'] = gogono['total_score']

    # 确保必需字段存在
    if 'market_overview' not in data:
        data['market_overview'] = {}

    mo = data['market_overview']
    if 'top3_product_concentration' not in mo and 'top3_concentration' in mo:
        mo['top3_product_concentration'] = mo['top3_concentration']

    return data


def main():
    parser = argparse.ArgumentParser(description="数据验证和修复脚本")
    parser.add_argument("data_file", help="data.json 文件路径")
    parser.add_argument("--fix", action="store_true", help="自动修复问题")
    parser.add_argument("--output", "-o", help="输出文件路径（默认覆盖原文件）")

    args = parser.parse_args()

    data_path = Path(args.data_file)
    if not data_path.exists():
        print(f"✗ 文件不存在: {data_path}")
        return 1

    # 读取数据
    print(f"读取数据: {data_path}")
    with open(data_path, 'r', encoding='utf-8') as f:
        data = json.load(f)

    # 验证数据
    is_valid, messages = validate_data(data)

    print("\n验证结果:")
    for msg in messages:
        prefix = "✗" if "错误" in msg or "缺少" in msg else "⚠"
        print(f"  {prefix} {msg}")

    if is_valid:
        print("\n✓ 数据结构验证通过")
    else:
        print("\n✗ 数据结构存在问题")
        if not args.fix:
            print("  提示: 使用 --fix 参数尝试自动修复")
            return 1

    # 修复数据
    if args.fix:
        print("\n修复数据...")
        data = fix_data(data)

        # 重新验证
        is_valid_after, messages_after = validate_data(data)
        if is_valid_after:
            print("✓ 数据修复成功")
        else:
            print("⚠ 部分问题无法自动修复")

        # 保存
        output_path = Path(args.output) if args.output else data_path
        with open(output_path, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)

        print(f"✓ 已保存: {output_path}")

    return 0 if is_valid else 1


if __name__ == '__main__':
    sys.exit(main())