Ultimate Search

# ===========================================
# UltimateSearchSkill 环境变量配置
# ===========================================
# 使用方法：复制此文件为 .env 并填入实际值
#   cp .env.example .env

# === grok2api 配置 ===
GROK2API_PORT=8100
# grok2api 管理面板密码（必须修改！）
GROK2API_APP_KEY=changeme-to-strong-password
# 调用 grok2api 的 API Key（必须设置，否则 API 无认证！）
GROK2API_API_KEY=

# === TavilyProxyManager 配置 ===
TAVILY_PROXY_PORT=8200
# TavilyProxyManager 的 Master Key（首次启动后从日志获取，然后填入此处）
TAVILY_MASTER_KEY=

# === 搜索脚本配置 ===
GROK_API_URL=http://127.0.0.1:8100
GROK_API_KEY=
GROK_MODEL=grok-4.1-fast
TAVILY_API_URL=http://127.0.0.1:8200
TAVILY_API_KEY=

# === FireCrawl 配置（web-fetch 的降级方案）===
# FireCrawl API 地址（默认官方 API，也支持自托管）
FIRECRAWL_API_URL=https://api.firecrawl.dev/v2
# FireCrawl API Key（可选，未配置则跳过 FireCrawl 降级）
# 获取: https://www.firecrawl.dev/
FIRECRAWL_API_KEY=

# ===========================================
# 批量 Key 导入（逗号分隔，用于 import-keys.sh 自动注册到对应服务）
# ===========================================

# Tavily API Keys（逗号分隔）
# 获取: https://www.tavily.com/ 免费 1000次/月
TAVILY_API_KEYS=

# FireCrawl API Keys（逗号分隔）
# 获取: https://www.firecrawl.dev/
FIRECRAWL_API_KEYS=

.env
data/
logs/
*.log
.DS_Store
export_sso.txt
*.txt.bak

services:
  flaresolverr:
    image: ghcr.io/flaresolverr/flaresolverr:latest
    container_name: ultimate-search-flaresolverr
    environment:
      - LOG_LEVEL=info
      - TZ=Asia/Shanghai
    restart: unless-stopped

  grok2api:
    image: ghcr.io/chenyme/grok2api:latest
    container_name: ultimate-search-grok2api
    ports:
      - "127.0.0.1:${GROK2API_PORT:-8100}:8000"
    environment:
      - DATA_DIR=/data
      - LOG_FILE_ENABLED=true
      - LOG_LEVEL=INFO
      - SERVER_STORAGE_TYPE=local
    volumes:
      - ./data/grok2api:/data
      - ./data/grok2api/logs:/app/logs
    depends_on:
      - flaresolverr
    restart: unless-stopped

  tavily-proxy:
    image: ghcr.io/xuncv/tavilyproxymanager:latest
    container_name: ultimate-search-tavily-proxy
    ports:
      - "127.0.0.1:${TAVILY_PROXY_PORT:-8200}:8080"
    environment:
      - LISTEN_ADDR=:8080
      - DATABASE_PATH=/app/data/proxy.db
      - TAVILY_BASE_URL=https://api.tavily.com
      - UPSTREAM_TIMEOUT=150s
    volumes:
      - ./data/tavily-proxy:/app/data
      # - /etc/localtime:/etc/localtime:ro  # Linux only
    restart: unless-stopped

Feature: UltimateSearch routing reliability
  Ensure the skill chooses Grok/Tavily/agent-browser consistently for different query intents.

  Scenario: X/Twitter discussion should route to Grok first
    Given a query "检索一下X上有关伊朗战争讨论"
    When the agent selects search tools
    Then the first command must be "grok-search.sh --query \"...\" --platform \"X\""
    And the follow-up verification should include Tavily or another independent source

  Scenario: Full-text extraction after search
    Given a query requiring complete page content
    When the agent has identified target URLs
    Then it should run "web-fetch.sh --url \"...\""
    And output source links for key claims

  Scenario: Dynamic page requires browser collaboration
    Given target content hidden behind JS rendering, login, or button interactions
    When direct extraction is incomplete
    Then it must use "agent-browser open/snapshot/click/wait"
    And feed the final resolved URL back into "web-fetch.sh"

  Scenario: Fact answer must be cross-verified
    Given a factual query with potential staleness
    When the answer is prepared
    Then evidence must include at least 2 independent sources
    And confidence must be labeled High/Medium/Low

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# 加载 .env（如果环境变量未设置）
if [[ -f "$SCRIPT_DIR/../.env" ]]; then
  while IFS='=' read -r key value; do
    key="$(echo "$key" | xargs)"
    [[ -z "$key" || "$key" == \#* ]] && continue
    value="$(echo "$value" | xargs | sed -e "s/^['\"]//;s/['\"]$//")"
    if [[ -z "${!key:-}" ]]; then
      export "$key=$value"
    fi
  done < "$SCRIPT_DIR/../.env"
fi

usage() {
  cat <<EOF
用法: $(basename "$0") [选项]

双引擎聚合搜索 — 并行调用 Grok AI 搜索和 Tavily 结构化搜索，合并结果

选项:
  --query "查询内容"           必需，搜索查询内容
  --tavily-depth basic|advanced  可选，Tavily 搜索深度（默认: basic）
  --help                       显示此帮助信息

示例:
  $(basename "$0") --query "FastAPI vs Flask comparison"
  $(basename "$0") --query "最新 AI 新闻" --tavily-depth advanced
EOF
  exit 0
}

error_exit() {
  echo "{\"error\": \"$1\"}"
  exit 1
}

QUERY=""
TAVILY_DEPTH="basic"

if [[ $# -eq 0 ]]; then
  usage
fi

while [[ $# -gt 0 ]]; do
  case "$1" in
    --query)
      QUERY="$2"
      shift 2
      ;;
    --tavily-depth)
      TAVILY_DEPTH="$2"
      shift 2
      ;;
    --help)
      usage
      ;;
    *)
      error_exit "未知参数: $1"
      ;;
  esac
done

[[ -z "$QUERY" ]] && error_exit "缺少必需参数 --query"

# 创建临时文件
GROK_TMP=$(mktemp)
TAVILY_TMP=$(mktemp)

# 确保退出时清理临时文件
cleanup() {
  rm -f "$GROK_TMP" "$TAVILY_TMP"
}
trap cleanup EXIT

# 并行调用两个搜索引擎
"$SCRIPT_DIR/grok-search.sh" --query "$QUERY" > "$GROK_TMP" 2>&1 &
GROK_PID=$!

"$SCRIPT_DIR/tavily-search.sh" --query "$QUERY" --depth "$TAVILY_DEPTH" > "$TAVILY_TMP" 2>&1 &
TAVILY_PID=$!

# 等待两个进程完成
GROK_EXIT=0
TAVILY_EXIT=0
wait "$GROK_PID" || GROK_EXIT=$?
wait "$TAVILY_PID" || TAVILY_EXIT=$?

# 读取结果
GROK_RESULT=$(cat "$GROK_TMP")
TAVILY_RESULT=$(cat "$TAVILY_TMP")

# 验证 JSON 有效性，无效则包装为错误
if ! echo "$GROK_RESULT" | jq empty 2>/dev/null; then
  GROK_RESULT="{\"error\": \"Grok 搜索失败: $(echo "$GROK_RESULT" | head -1 | sed 's/"/\\"/g')\"}"
fi

if ! echo "$TAVILY_RESULT" | jq empty 2>/dev/null; then
  TAVILY_RESULT="{\"error\": \"Tavily 搜索失败: $(echo "$TAVILY_RESULT" | head -1 | sed 's/"/\\"/g')\"}"
fi

# 合并结果
jq -n \
  --argjson grok "$GROK_RESULT" \
  --argjson tavily "$TAVILY_RESULT" \
  '{
    grok: $grok,
    tavily: $tavily
  }'

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# 加载 .env（如果环境变量未设置）
if [[ -f "$SCRIPT_DIR/../.env" ]]; then
  while IFS='=' read -r key value; do
    key="$(echo "$key" | xargs)"
    [[ -z "$key" || "$key" == \#* ]] && continue
    value="$(echo "$value" | xargs | sed -e "s/^['\"]//;s/['\"]$//")"
    if [[ -z "${!key:-}" ]]; then
      export "$key=$value"
    fi
  done < "$SCRIPT_DIR/../.env"
fi

GROK_API_URL="${GROK_API_URL:-}"
GROK_API_KEY="${GROK_API_KEY:-}"
GROK_MODEL="${GROK_MODEL:-grok-4.1-fast}"

usage() {
  cat <<EOF
用法: $(basename "$0") [选项]

Grok AI 智能搜索 — 通过 Grok 模型的联网能力进行 AI 驱动搜索

选项:
  --query "查询内容"     必需，搜索查询内容
  --platform "平台"      可选，聚焦平台（如 Twitter, GitHub, Reddit）
  --model "模型"         可选，覆盖默认模型 ($GROK_MODEL)
  --help                 显示此帮助信息

示例:
  $(basename "$0") --query "FastAPI 最新用法"
  $(basename "$0") --query "React 19 新特性" --platform "GitHub"
  $(basename "$0") --query "latest AI news" --model "grok-3"
EOF
  exit 0
}

error_exit() {
  echo "{\"error\": \"$1\"}"
  exit 1
}

QUERY=""
PLATFORM=""
MODEL="$GROK_MODEL"

if [[ $# -eq 0 ]]; then
  usage
fi

while [[ $# -gt 0 ]]; do
  case "$1" in
    --query)
      QUERY="$2"
      shift 2
      ;;
    --platform)
      PLATFORM="$2"
      shift 2
      ;;
    --model)
      MODEL="$2"
      shift 2
      ;;
    --help)
      usage
      ;;
    *)
      error_exit "未知参数: $1"
      ;;
  esac
done

[[ -z "$QUERY" ]] && error_exit "缺少必需参数 --query"
[[ -z "$GROK_API_URL" ]] && error_exit "未设置 GROK_API_URL"
[[ -z "$GROK_API_KEY" ]] && error_exit "未设置 GROK_API_KEY"

# system prompt（来自 GrokSearch MCP 的 search_prompt）
SYSTEM_PROMPT='# Core Instruction

1. User needs may be vague. Think divergently, infer intent from multiple angles, and leverage full conversation context to progressively clarify their true needs.
2. **Breadth-First Search**—Approach problems from multiple dimensions. Brainstorm 5+ perspectives and execute parallel searches for each. Consult as many high-quality sources as possible before responding.
3. **Depth-First Search**—After broad exploration, select ≥2 most relevant perspectives for deep investigation into specialized knowledge.
4. **Evidence-Based Reasoning & Traceable Sources**—Every claim must be followed by a citation. More credible sources strengthen arguments. If no references exist, remain silent.
5. Before responding, ensure full execution of Steps 1–4.

# Search Instruction

1. Think carefully before responding—anticipate the user'\''s true intent to ensure precision.
2. Verify every claim rigorously to avoid misinformation.
3. Follow problem logic—dig deeper until clues are exhaustively clear. Use multiple parallel tool calls per query and ensure answers are well-sourced.
4. Search in English first (prioritizing English resources for volume/quality), but switch to Chinese if context demands.
5. Prioritize authoritative sources: Wikipedia, academic databases, books, reputable media/journalism.
6. Favor sharing in-depth, specialized knowledge over generic or common-sense content.

# Output Style

1. Lead with the **most probable solution** before detailed analysis.
2. **Define every technical term** in plain language.
3. **Respect facts and search results—use statistical rigor to discern truth**.
4. **Every sentence must cite sources**. More references = stronger credibility.
5. **Strictly format outputs in polished Markdown**.'

# 构建 user message
USER_MESSAGE="$QUERY"

# 时间相关关键词检测 → 注入当前日期时间
TIME_KEYWORDS='今天|最新|当前|latest|recent|today|current|now|这几天|本周|本月|近期|最近'
if echo "$QUERY" | grep -qiE "$TIME_KEYWORDS"; then
  CURRENT_TIME="$(date '+%Y-%m-%d %H:%M:%S %Z')"
  USER_MESSAGE="[Current date and time: $CURRENT_TIME]

$USER_MESSAGE"
fi

# 平台聚焦
if [[ -n "$PLATFORM" ]]; then
  USER_MESSAGE="$USER_MESSAGE

You should focus on these platform: $PLATFORM"
fi

# 构建请求 JSON
REQUEST_JSON=$(jq -n \
  --arg model "$MODEL" \
  --arg system "$SYSTEM_PROMPT" \
  --arg user "$USER_MESSAGE" \
  '{
    model: $model,
    stream: false,
    messages: [
      { role: "system", content: $system },
      { role: "user", content: $user }
    ]
  }')

# 调用 API
RESPONSE=$(curl -s -w "\n%{http_code}" \
  -X POST "$GROK_API_URL/v1/chat/completions" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $GROK_API_KEY" \
  -d "$REQUEST_JSON")

HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | sed '$d')

if [[ "$HTTP_CODE" -ne 200 ]]; then
  error_exit "API 请求失败 (HTTP $HTTP_CODE): $BODY"
fi

# 提取结果
echo "$BODY" | jq '{
  content: .choices[0].message.content,
  model: .model,
  usage: .usage
}'

#!/usr/bin/env bash
set -eo pipefail

# =========================================================
# import-keys.sh — 从 .env 批量导入 Key 到各服务
# =========================================================

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

info()  { echo -e "${BLUE}[INFO]${NC} $*"; }
ok()    { echo -e "${GREEN}[OK]${NC} $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC} $*"; }
error() { echo -e "${RED}[ERROR]${NC} $*"; }

# 加载 .env
if [[ ! -f "$PROJECT_DIR/.env" ]]; then
  error ".env 文件不存在，请先创建"
  exit 1
fi

set -a; source "$PROJECT_DIR/.env"; set +a

GROK2API_PORT="${GROK2API_PORT:-8100}"
TAVILY_PROXY_PORT="${TAVILY_PROXY_PORT:-8200}"
GROK2API_APP_KEY="${GROK2API_APP_KEY:-grok2api}"

echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}  批量导入 Key 到服务${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# ==========================================
# 1. 导入 Grok SSO Tokens（从文件）
# ==========================================
GROK_SSO_FILE="${1:-${PROJECT_DIR}/export_sso.txt}"

if [[ -f "$GROK_SSO_FILE" ]]; then
  info "导入 Grok SSO Tokens（从 $GROK_SSO_FILE）..."

  # 读取文件中的 token，每行一个
  TOKENS_JSON="["
  FIRST=true
  while IFS= read -r line || [[ -n "$line" ]]; do
    line="$(echo "$line" | xargs)"  # trim whitespace
    [[ -z "$line" || "$line" == \#* ]] && continue
    # 去掉可能的 sso= 前缀
    line="${line#sso=}"
    if [[ "$FIRST" == "true" ]]; then
      TOKENS_JSON+="\"$line\""
      FIRST=false
    else
      TOKENS_JSON+=",\"$line\""
    fi
  done < "$GROK_SSO_FILE"
  TOKENS_JSON+="]"

  TOKEN_COUNT=$(echo "$TOKENS_JSON" | jq 'length')
  info "发现 $TOKEN_COUNT 个 Grok Token，正在导入..."

  RESULT=$(curl -s -w "\n%{http_code}" \
    -X POST "http://127.0.0.1:$GROK2API_PORT/v1/admin/tokens" \
    -H "Authorization: Bearer $GROK2API_APP_KEY" \
    -H "Content-Type: application/json" \
    -d "{\"ssoBasic\": $TOKENS_JSON}")

  HTTP_CODE=$(echo "$RESULT" | tail -1)
  BODY=$(echo "$RESULT" | sed '$d')

  if [[ "$HTTP_CODE" -eq 200 ]]; then
    ok "Grok Tokens 导入成功（$TOKEN_COUNT 个）"
  else
    error "Grok Tokens 导入失败 (HTTP $HTTP_CODE): $BODY"
  fi
else
  warn "未找到 Grok SSO 文件: $GROK_SSO_FILE，跳过"
fi

echo ""

# ==========================================
# 2. 获取 TavilyProxyManager Master Key
# ==========================================
TAVILY_MASTER_KEY="${TAVILY_MASTER_KEY:-}"

if [[ -z "$TAVILY_MASTER_KEY" ]]; then
  info "TAVILY_MASTER_KEY 未设置，尝试自动发现..."
  cd "$PROJECT_DIR"

  # 优先从日志读取（兼容旧版本输出）
  if command -v docker &>/dev/null; then
    TAVILY_MASTER_KEY=$(docker compose logs tavily-proxy 2>&1 | grep -oE 'master_key=[^ ]+' | head -1 | cut -d= -f2 || true)
  fi

  # 新版本日志可能不输出 master key，回退到本地 sqlite
  if [[ -z "$TAVILY_MASTER_KEY" && -f "$PROJECT_DIR/data/tavily-proxy/proxy.db" ]] && command -v sqlite3 &>/dev/null; then
    TAVILY_MASTER_KEY=$(sqlite3 "$PROJECT_DIR/data/tavily-proxy/proxy.db" "select value from settings where key='master_key';" 2>/dev/null || true)
  fi

  if [[ -n "$TAVILY_MASTER_KEY" ]]; then
    ok "已发现 Master Key: ${TAVILY_MASTER_KEY:0:10}..."
    warn "请将此 Key 填入 .env 的 TAVILY_MASTER_KEY 和 TAVILY_API_KEY"
  else
    error "无法自动获取 Master Key"
  fi
fi

# ==========================================
# 3. 导入 Tavily API Keys
# ==========================================
TAVILY_API_KEYS="${TAVILY_API_KEYS:-}"

if [[ -n "$TAVILY_API_KEYS" && -n "$TAVILY_MASTER_KEY" ]]; then
  info "导入 Tavily API Keys..."

  IFS=',' read -ra KEYS <<< "$TAVILY_API_KEYS"
  SUCCESS=0
  FAIL=0

  for key in "${KEYS[@]}"; do
    key="$(echo "$key" | xargs)"  # trim whitespace
    [[ -z "$key" ]] && continue

    RESULT=$(curl -s -w "\n%{http_code}" \
      -X POST "http://127.0.0.1:$TAVILY_PROXY_PORT/api/keys" \
      -H "Authorization: Bearer $TAVILY_MASTER_KEY" \
      -H "Content-Type: application/json" \
      -d "{\"key\": \"$key\", \"alias\": \"批量导入\", \"total_quota\": 1000}")

    HTTP_CODE=$(echo "$RESULT" | tail -1)
    if [[ "$HTTP_CODE" -eq 200 ]]; then
      SUCCESS=$((SUCCESS + 1))
    else
      FAIL=$((FAIL + 1))
      BODY=$(echo "$RESULT" | sed '$d')
      warn "Tavily Key ${key:0:10}... 导入失败: $BODY"
    fi
  done

  ok "Tavily Keys 导入完成：成功 $SUCCESS 个，失败 $FAIL 个"
elif [[ -z "$TAVILY_API_KEYS" ]]; then
  warn "TAVILY_API_KEYS 未设置，跳过 Tavily Key 导入"
elif [[ -z "$TAVILY_MASTER_KEY" ]]; then
  error "TAVILY_MASTER_KEY 未设置，无法导入 Tavily Keys"
fi

echo ""

# ==========================================
# 4. 导入 FireCrawl API Keys（预留）
# ==========================================
FIRECRAWL_API_KEYS="${FIRECRAWL_API_KEYS:-}"

if [[ -n "$FIRECRAWL_API_KEYS" ]]; then
  info "FireCrawl Keys 已配置，当前版本暂不支持自动导入（需要 FireCrawl 代理服务）"
  IFS=',' read -ra KEYS <<< "$FIRECRAWL_API_KEYS"
  info "  发现 ${#KEYS[@]} 个 FireCrawl Key"
fi

echo ""

# ==========================================
# 5. 更新 .env 中的 TAVILY_MASTER_KEY 和 TAVILY_API_KEY
# ==========================================
if [[ -n "$TAVILY_MASTER_KEY" ]]; then
  # 检查 .env 中是否已设置
  CURRENT_MASTER=$(grep '^TAVILY_MASTER_KEY=' "$PROJECT_DIR/.env" | cut -d= -f2- || true)
  CURRENT_API=$(grep '^TAVILY_API_KEY=' "$PROJECT_DIR/.env" | cut -d= -f2- || true)

  if [[ -z "$CURRENT_MASTER" || -z "$CURRENT_API" ]]; then
    info "自动更新 .env 中的 TAVILY_MASTER_KEY 和 TAVILY_API_KEY..."
    sed -i.bak "s|^TAVILY_MASTER_KEY=.*|TAVILY_MASTER_KEY=$TAVILY_MASTER_KEY|" "$PROJECT_DIR/.env"
    sed -i.bak "s|^TAVILY_API_KEY=.*|TAVILY_API_KEY=$TAVILY_MASTER_KEY|" "$PROJECT_DIR/.env"
    rm -f "$PROJECT_DIR/.env.bak"
    ok ".env 已更新"
  fi
fi

# ==========================================
# 验证结果
# ==========================================
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}  验证服务状态${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# 检查 grok2api Token 数量
GROK_TOKENS=$(curl -s \
  -H "Authorization: Bearer $GROK2API_APP_KEY" \
  "http://127.0.0.1:$GROK2API_PORT/v1/admin/tokens" 2>/dev/null || echo "{}")

GROK_COUNT=$(echo "$GROK_TOKENS" | jq '[.[] | length] | add // 0' 2>/dev/null || echo "?")
info "grok2api Token 数量: $GROK_COUNT"

# 检查 TavilyProxyManager Key 数量
if [[ -n "$TAVILY_MASTER_KEY" ]]; then
  TAVILY_KEYS_RESULT=$(curl -s \
    -H "Authorization: Bearer $TAVILY_MASTER_KEY" \
    "http://127.0.0.1:$TAVILY_PROXY_PORT/api/keys" 2>/dev/null || echo "{}")

  TAVILY_COUNT=$(echo "$TAVILY_KEYS_RESULT" | jq '.items | length' 2>/dev/null || echo "?")
  info "TavilyProxyManager Key 数量: $TAVILY_COUNT"
fi

echo ""
ok "导入完成！"

#!/usr/bin/env bash
set -euo pipefail

# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"

info()  { echo -e "${BLUE}[INFO]${NC} $*"; }
ok()    { echo -e "${GREEN}[OK]${NC} $*"; }
warn()  { echo -e "${YELLOW}[WARN]${NC} $*"; }
error() { echo -e "${RED}[ERROR]${NC} $*"; exit 1; }

cd "$PROJECT_DIR"

echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}  UltimateSearchSkill 部署脚本${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# 检查依赖
for cmd in docker curl jq; do
    if ! command -v "$cmd" &>/dev/null; then
        error "$cmd 未安装，请先安装"
    fi
    ok "$cmd 已安装"
done

if docker compose version &>/dev/null; then
    COMPOSE="docker compose"
elif command -v docker-compose &>/dev/null; then
    COMPOSE="docker-compose"
else
    error "docker compose 未安装"
fi
ok "docker compose 可用"

# 配置文件
if [ ! -f .env ]; then
    cp .env.example .env
    warn ".env 已从模板创建，请编辑 .env 填入实际配置"
    warn "特别注意修改 GROK2API_APP_KEY 和 GROK2API_API_KEY"
fi

# 加载环境变量
set -a; source .env; set +a

# 创建数据目录
mkdir -p data/grok2api/logs data/tavily-proxy

# 拉取镜像并启动
info "拉取 Docker 镜像..."
$COMPOSE pull

info "启动服务..."
$COMPOSE up -d

# 等待服务就绪
info "等待服务就绪..."
for i in $(seq 1 30); do
    if curl -sf "http://127.0.0.1:${GROK2API_PORT:-8100}/" >/dev/null 2>&1; then
        ok "grok2api 就绪"
        break
    fi
    [ "$i" -eq 30 ] && warn "grok2api 30秒内未就绪，请检查日志: $COMPOSE logs grok2api"
    sleep 1
done

for i in $(seq 1 30); do
    if curl -sf "http://127.0.0.1:${TAVILY_PROXY_PORT:-8200}/healthz" >/dev/null 2>&1; then
        ok "TavilyProxyManager 就绪"
        break
    fi
    [ "$i" -eq 30 ] && warn "TavilyProxyManager 30秒内未就绪，请检查日志: $COMPOSE logs tavily-proxy"
    sleep 1
done

# 获取 TavilyProxyManager Master Key
echo ""
info "TavilyProxyManager Master Key："
MASTER_KEY=$($COMPOSE logs tavily-proxy 2>&1 | grep -oP 'key=\K\S+' | head -1 || true)
if [ -n "$MASTER_KEY" ]; then
    echo -e "  ${GREEN}$MASTER_KEY${NC}"
    echo ""
    warn "请将此 Master Key 填入 .env 的 TAVILY_MASTER_KEY 和 TAVILY_API_KEY"
else
    warn "未能自动获取 Master Key，请手动查看: $COMPOSE logs tavily-proxy | grep 'master key'"
fi

# 后续步骤
echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}  部署完成！后续步骤：${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""
echo "1. 修改 .env 中的密码和 Key"
echo "2. 访问 grok2api 管理面板添加 Grok Token："
echo "   ssh -L 8100:127.0.0.1:${GROK2API_PORT:-8100} 你的服务器"
echo "   然后浏览器打开 http://localhost:8100/admin"
echo ""
echo "3. 访问 TavilyProxyManager 添加 Tavily Key："
echo "   ssh -L 8200:127.0.0.1:${TAVILY_PROXY_PORT:-8200} 你的服务器"
echo "   然后浏览器打开 http://localhost:8200"
echo ""
echo "4. 将脚本加入 PATH："
echo "   echo 'export PATH=\"$PROJECT_DIR/scripts:\$PATH\"' >> ~/.bashrc"
echo "   source ~/.bashrc"
echo ""
echo "5. 加载环境变量（每次使用前）："
echo "   source $PROJECT_DIR/.env"

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# 加载 .env（如果环境变量未设置）
if [[ -f "$SCRIPT_DIR/../.env" ]]; then
  while IFS='=' read -r key value; do
    key="$(echo "$key" | xargs)"
    [[ -z "$key" || "$key" == \#* ]] && continue
    value="$(echo "$value" | xargs | sed -e "s/^['\"]//;s/['\"]$//")"
    if [[ -z "${!key:-}" ]]; then
      export "$key=$value"
    fi
  done < "$SCRIPT_DIR/../.env"
fi

TAVILY_API_URL="${TAVILY_API_URL:-}"
TAVILY_API_KEY="${TAVILY_API_KEY:-}"

usage() {
  cat <<EOF
用法: $(basename "$0") [选项]

Tavily 结构化搜索 — 使用 Tavily Search API 进行结构化网络搜索

选项:
  --query "查询内容"           必需，搜索查询内容
  --depth basic|advanced       可选，搜索深度（默认: basic）
  --max-results N              可选，最大结果数（默认: 5）
  --topic general|news|finance 可选，搜索主题（默认: general）
  --time-range day|week|month|year  可选，时间范围过滤
  --include-answer             可选，在结果中包含 AI 生成的答案
  --include-raw                可选，包含原始内容
  --help                       显示此帮助信息

示例:
  $(basename "$0") --query "Python web frameworks comparison"
  $(basename "$0") --query "AI news" --topic news --time-range week --include-answer
  $(basename "$0") --query "stock market" --depth advanced --max-results 10
EOF
  exit 0
}

error_exit() {
  echo "{\"error\": \"$1\"}"
  exit 1
}

QUERY=""
DEPTH="basic"
MAX_RESULTS=5
TOPIC="general"
TIME_RANGE=""
INCLUDE_ANSWER=false
INCLUDE_RAW=false

if [[ $# -eq 0 ]]; then
  usage
fi

while [[ $# -gt 0 ]]; do
  case "$1" in
    --query)
      QUERY="$2"
      shift 2
      ;;
    --depth)
      DEPTH="$2"
      shift 2
      ;;
    --max-results)
      MAX_RESULTS="$2"
      shift 2
      ;;
    --topic)
      TOPIC="$2"
      shift 2
      ;;
    --time-range)
      TIME_RANGE="$2"
      shift 2
      ;;
    --include-answer)
      INCLUDE_ANSWER=true
      shift
      ;;
    --include-raw)
      INCLUDE_RAW=true
      shift
      ;;
    --help)
      usage
      ;;
    *)
      error_exit "未知参数: $1"
      ;;
  esac
done

[[ -z "$QUERY" ]] && error_exit "缺少必需参数 --query"
[[ -z "$TAVILY_API_URL" ]] && error_exit "未设置 TAVILY_API_URL"
[[ -z "$TAVILY_API_KEY" ]] && error_exit "未设置 TAVILY_API_KEY"

# 构建请求 JSON
REQUEST_JSON=$(jq -n \
  --arg query "$QUERY" \
  --arg depth "$DEPTH" \
  --argjson max_results "$MAX_RESULTS" \
  --arg topic "$TOPIC" \
  --argjson include_answer "$INCLUDE_ANSWER" \
  --argjson include_raw "$INCLUDE_RAW" \
  '{
    query: $query,
    search_depth: $depth,
    max_results: $max_results,
    topic: $topic,
    include_answer: $include_answer,
    include_raw_content: $include_raw
  }')

# 添加可选的 time_range
if [[ -n "$TIME_RANGE" ]]; then
  REQUEST_JSON=$(echo "$REQUEST_JSON" | jq --arg tr "$TIME_RANGE" '. + {time_range: $tr}')
fi

# 调用 API
RESPONSE=$(curl -s -w "\n%{http_code}" \
  -X POST "$TAVILY_API_URL/search" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $TAVILY_API_KEY" \
  -d "$REQUEST_JSON")

HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | sed '$d')

if [[ "$HTTP_CODE" -ne 200 ]]; then
  error_exit "API 请求失败 (HTTP $HTTP_CODE): $BODY"
fi

echo "$BODY" | jq '.'

#!/usr/bin/env bash
set -eo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# 加载 .env（如果环境变量未设置）
if [[ -f "$SCRIPT_DIR/../.env" ]]; then
  while IFS='=' read -r key value; do
    key="$(echo "$key" | xargs)"
    [[ -z "$key" || "$key" == \#* ]] && continue
    value="$(echo "$value" | xargs | sed -e "s/^['\"]//;s/['\"]$//")"
    if [[ -z "${!key:-}" ]]; then
      export "$key=$value"
    fi
  done < "$SCRIPT_DIR/../.env"
fi

TAVILY_API_URL="${TAVILY_API_URL:-}"
TAVILY_API_KEY="${TAVILY_API_KEY:-}"
FIRECRAWL_API_URL="${FIRECRAWL_API_URL:-https://api.firecrawl.dev/v2}"
FIRECRAWL_API_KEY="${FIRECRAWL_API_KEY:-}"
# 兼容批量 Key 配置：如果单个 Key 未设置，取 FIRECRAWL_API_KEYS 的第一个
if [[ -z "$FIRECRAWL_API_KEY" && -n "${FIRECRAWL_API_KEYS:-}" ]]; then
  FIRECRAWL_API_KEY=$(echo "$FIRECRAWL_API_KEYS" | cut -d',' -f1 | xargs)
fi

usage() {
  cat <<EOF
用法: $(basename "$0") [选项]

网页内容抓取 — 三级降级：Tavily Extract → FireCrawl Scrape → 返回错误

选项:
  --url "URL"                必需，目标网页 URL（可多次指定）
  --depth basic|advanced     可选，提取深度（默认: basic）
  --format markdown|text     可选，输出格式（默认: markdown）
  --help                     显示此帮助信息

示例:
  $(basename "$0") --url "https://example.com"
  $(basename "$0") --url "https://a.com" --url "https://b.com" --depth advanced
  $(basename "$0") --url "https://example.com" --format text
EOF
  exit 0
}

error_exit() {
  echo "{\"error\": \"$1\"}"
  exit 1
}

URLS=()
DEPTH="basic"
FORMAT="markdown"

if [[ $# -eq 0 ]]; then
  usage
fi

while [[ $# -gt 0 ]]; do
  case "$1" in
    --url)
      URLS+=("$2")
      shift 2
      ;;
    --depth)
      DEPTH="$2"
      shift 2
      ;;
    --format)
      FORMAT="$2"
      shift 2
      ;;
    --help)
      usage
      ;;
    *)
      error_exit "未知参数: $1"
      ;;
  esac
done

[[ ${#URLS[@]} -eq 0 ]] && error_exit "缺少必需参数 --url"

# ==========================================
# Tavily Extract（第一级）
# ==========================================
tavily_extract() {
  local urls_json="$1"
  [[ -z "$TAVILY_API_URL" || -z "$TAVILY_API_KEY" ]] && return 1

  local request_json
  request_json=$(jq -n \
    --argjson urls "$urls_json" \
    --arg depth "$DEPTH" \
    --arg format "$FORMAT" \
    '{
      urls: $urls,
      extract_depth: $depth,
      format: $format
    }')

  local response http_code body
  response=$(curl -s -w "\n%{http_code}" \
    --connect-timeout 6 --max-time 30 \
    -X POST "$TAVILY_API_URL/extract" \
    -H "Content-Type: application/json" \
    -H "Authorization: Bearer $TAVILY_API_KEY" \
    -d "$request_json")

  http_code=$(echo "$response" | tail -1)
  body=$(echo "$response" | sed '$d')

  if [[ "$http_code" -ne 200 ]]; then
    return 1
  fi

  # 检查是否有实际内容
  local has_content
  has_content=$(echo "$body" | jq '[.results[]? | select(.raw_content != null and .raw_content != "")] | length' 2>/dev/null || echo "0")
  if [[ "$has_content" -eq 0 ]]; then
    return 1
  fi

  echo "$body" | jq '{source: "tavily", results: .results}'
  return 0
}

# ==========================================
# FireCrawl Scrape（第二级降级）
# ==========================================
firecrawl_scrape() {
  local urls_json="$1"
  [[ -z "$FIRECRAWL_API_KEY" ]] && return 1

  local api_url="${FIRECRAWL_API_URL%/}"
  local all_results="[]"

  # FireCrawl scrape 是单 URL 接口，需要逐个调用
  local url_count
  url_count=$(echo "$urls_json" | jq 'length')

  for (( i=0; i<url_count; i++ )); do
    local url
    url=$(echo "$urls_json" | jq -r ".[$i]")

    local request_json
    request_json=$(jq -n \
      --arg url "$url" \
      '{
        url: $url,
        formats: ["markdown"]
      }')

    local response http_code body
    response=$(curl -s -w "\n%{http_code}" \
      --connect-timeout 6 --max-time 60 \
      -X POST "$api_url/scrape" \
      -H "Content-Type: application/json" \
      -H "Authorization: Bearer $FIRECRAWL_API_KEY" \
      -d "$request_json")

    http_code=$(echo "$response" | tail -1)
    body=$(echo "$response" | sed '$d')

    if [[ "$http_code" -eq 200 ]]; then
      local content
      content=$(echo "$body" | jq -r '(.data.markdown // .data.content // "") | ltrimstr(" ") | rtrimstr(" ")' 2>/dev/null || echo "")
      if [[ -n "$content" && "$content" != "null" ]]; then
        all_results=$(echo "$all_results" | jq \
          --arg url "$url" \
          --arg content "$content" \
          '. + [{url: $url, raw_content: $content}]')
      fi
    fi
  done

  local result_count
  result_count=$(echo "$all_results" | jq 'length')
  if [[ "$result_count" -eq 0 ]]; then
    return 1
  fi

  jq -n --argjson results "$all_results" '{source: "firecrawl", results: $results}'
  return 0
}

# ==========================================
# 主流程：三级降级
# ==========================================

# 构建 URL 数组 JSON
URLS_JSON=$(printf '%s\n' "${URLS[@]}" | jq -R . | jq -s .)

# 第一级：Tavily Extract
if result=$(tavily_extract "$URLS_JSON" 2>/dev/null); then
  echo "$result" | jq '.'
  exit 0
fi

# 第二级：FireCrawl Scrape
if result=$(firecrawl_scrape "$URLS_JSON" 2>/dev/null); then
  echo "$result" | jq '.'
  exit 0
fi

# 都失败
error_exit "Tavily Extract 和 FireCrawl Scrape 均失败"

#!/usr/bin/env bash
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# 加载 .env（如果环境变量未设置）
if [[ -f "$SCRIPT_DIR/../.env" ]]; then
  while IFS='=' read -r key value; do
    key="$(echo "$key" | xargs)"
    [[ -z "$key" || "$key" == \#* ]] && continue
    value="$(echo "$value" | xargs | sed -e "s/^['\"]//;s/['\"]$//")"
    if [[ -z "${!key:-}" ]]; then
      export "$key=$value"
    fi
  done < "$SCRIPT_DIR/../.env"
fi

TAVILY_API_URL="${TAVILY_API_URL:-}"
TAVILY_API_KEY="${TAVILY_API_KEY:-}"

usage() {
  cat <<EOF
用法: $(basename "$0") [选项]

站点结构映射 — 使用 Tavily Map API 发现网站 URL 结构

选项:
  --url "URL"              必需，目标站点 URL
  --depth N                可选，爬取深度，范围 1-5（默认: 1）
  --breadth N              可选，每层爬取宽度（默认: 20）
  --limit N                可选，最大 URL 数量（默认: 50）
  --instructions "说明"    可选，爬取指令说明
  --help                   显示此帮助信息

示例:
  $(basename "$0") --url "https://example.com"
  $(basename "$0") --url "https://docs.example.com" --depth 2 --limit 100
  $(basename "$0") --url "https://example.com" --instructions "只抓取文档页面"
EOF
  exit 0
}

error_exit() {
  echo "{\"error\": \"$1\"}"
  exit 1
}

URL=""
DEPTH=1
BREADTH=20
LIMIT=50
INSTRUCTIONS=""

if [[ $# -eq 0 ]]; then
  usage
fi

while [[ $# -gt 0 ]]; do
  case "$1" in
    --url)
      URL="$2"
      shift 2
      ;;
    --depth)
      DEPTH="$2"
      shift 2
      ;;
    --breadth)
      BREADTH="$2"
      shift 2
      ;;
    --limit)
      LIMIT="$2"
      shift 2
      ;;
    --instructions)
      INSTRUCTIONS="$2"
      shift 2
      ;;
    --help)
      usage
      ;;
    *)
      error_exit "未知参数: $1"
      ;;
  esac
done

[[ -z "$URL" ]] && error_exit "缺少必需参数 --url"
[[ -z "$TAVILY_API_URL" ]] && error_exit "未设置 TAVILY_API_URL"
[[ -z "$TAVILY_API_KEY" ]] && error_exit "未设置 TAVILY_API_KEY"

# 验证 depth 范围
if [[ "$DEPTH" -lt 1 || "$DEPTH" -gt 5 ]]; then
  error_exit "--depth 必须在 1-5 范围内"
fi

# 构建请求 JSON
REQUEST_JSON=$(jq -n \
  --arg url "$URL" \
  --argjson depth "$DEPTH" \
  --argjson breadth "$BREADTH" \
  --argjson limit "$LIMIT" \
  '{
    url: $url,
    depth: $depth,
    breadth: $breadth,
    limit: $limit
  }')

# 添加可选的 instructions
if [[ -n "$INSTRUCTIONS" ]]; then
  REQUEST_JSON=$(echo "$REQUEST_JSON" | jq --arg inst "$INSTRUCTIONS" '. + {instructions: $inst}')
fi

# 调用 API
RESPONSE=$(curl -s -w "\n%{http_code}" \
  -X POST "$TAVILY_API_URL/map" \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $TAVILY_API_KEY" \
  -d "$REQUEST_JSON")

HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | sed '$d')

if [[ "$HTTP_CODE" -ne 200 ]]; then
  error_exit "API 请求失败 (HTTP $HTTP_CODE): $BODY"
fi

echo "$BODY" | jq '.'