Model Fallback

#!/bin/bash
# 自动处理模型请求失败并切换的脚本
# 由 OpenClaw 在检测到模型错误时调用

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_DIR="$HOME/.openclaw/logs"
LOG_FILE="$LOG_DIR/auto-switch.log"
STATE_FILE="$LOG_DIR/switch-state.json"
FALLBACK_SCRIPT="$HOME/.openclaw/scripts/model-fallback.sh"

mkdir -p "$LOG_DIR"

# 日志函数
log() {
  local level=$1
  shift
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE"
}

# 解析命令行参数
ERROR_TYPE=""
ERROR_CODE=""
ERROR_MESSAGE=""
CURRENT_MODEL=""
RETRY_COUNT=0

while [[ $# -gt 0 ]]; do
  case $1 in
    --error-type)
      ERROR_TYPE="$2"
      shift 2
      ;;
    --error-code)
      ERROR_CODE="$2"
      shift 2
      ;;
    --error-message)
      ERROR_MESSAGE="$2"
      shift 2
      ;;
    --current-model)
      CURRENT_MODEL="$2"
      shift 2
      ;;
    --retry-count)
      RETRY_COUNT="$2"
      shift 2
      ;;
    *)
      log ERROR "未知参数: $1"
      exit 1
      ;;
  esac
done

log INFO "=== 自动模型切换处理 ==="
log INFO "错误类型: $ERROR_TYPE"
log INFO "错误代码: $ERROR_CODE"
log INFO "错误消息: $ERROR_MESSAGE"
log INFO "当前模型: $CURRENT_MODEL"
log INFO "重试次数: $RETRY_COUNT"

# 读取当前状态
if [[ -f "$STATE_FILE" ]]; then
  CONSECUTIVE_FAILURES=$(python3 -c "import json; d=json.load(open('$STATE_FILE')); print(d.get('consecutive_failures', 0))" 2>/dev/null || echo "0")
  LAST_SWITCH=$(python3 -c "import json; d=json.load(open('$STATE_FILE')); print(d.get('last_switch', 0))" 2>/dev/null || echo "0")
else
  CONSECUTIVE_FAILURES=0
  LAST_SWITCH=0
fi

log INFO "连续失败次数: $CONSECUTIVE_FAILURES"

# 决策逻辑
SHOULD_SWITCH=false
SWITCH_REASON=""

case "$ERROR_TYPE" in
  "timeout")
    # 超时错误：重试2次后切换
    if [[ $RETRY_COUNT -ge 2 ]]; then
      SHOULD_SWITCH=true
      SWITCH_REASON="超时重试次数超限"
    fi
    ;;

  "rate_limit"|"429")
    # 速率限制：立即切换
    SHOULD_SWITCH=true
    SWITCH_REASON="API 速率限制"
    ;;

  "quota_exceeded"|"insufficient_quota")
    # 配额耗尽：立即切换并标记
    SHOULD_SWITCH=true
    SWITCH_REASON="配额耗尽"
    ;;

  "authentication"|"401"|"403")
    # 认证错误：立即切换并禁用
    SHOULD_SWITCH=true
    SWITCH_REASON="认证失败"
    ;;

  "service_unavailable"|"503"|"502")
    # 服务不可用：重试3次后切换
    if [[ $RETRY_COUNT -ge 3 ]]; then
      SHOULD_SWITCH=true
      SWITCH_REASON="服务不可用"
    fi
    ;;

  "network_error"|"connection_error")
    # 网络错误：检查连续失败次数
    if [[ $CONSECUTIVE_FAILURES -ge 3 ]]; then
      SHOULD_SWITCH=true
      SWITCH_REASON="网络连续失败"
    fi
    ;;

  *)
    # 其他错误：记录但不切换
    log WARN "未知错误类型，不触发切换"
    exit 0
    ;;
esac

# 冷却期检查（避免频繁切换）
NOW=$(date +%s)
TIME_SINCE_LAST_SWITCH=$((NOW - LAST_SWITCH))
COOLDOWN_PERIOD=300  # 5分钟

if [[ $TIME_SINCE_LAST_SWITCH -lt $COOLDOWN_PERIOD ]] && [[ $SHOULD_SWITCH == true ]]; then
  log WARN "切换冷却期中，距上次切换仅 ${TIME_SINCE_LAST_SWITCH} 秒（需要 ${COOLDOWN_PERIOD} 秒）"

  # 严重错误（认证、配额）不受冷却期限制
  if [[ "$ERROR_TYPE" != "authentication" ]] && [[ "$ERROR_TYPE" != "quota_exceeded" ]]; then
    SHOULD_SWITCH=false
    SWITCH_REASON="冷却期保护"
  fi
fi

# 执行切换
if [[ "$SHOULD_SWITCH" == true ]]; then
  log WARN "触发模型切换: $SWITCH_REASON"

  # 获取当前模型（如果未提供）
  if [[ -z "$CURRENT_MODEL" ]]; then
    CURRENT_MODEL=$(openclaw status 2>/dev/null | grep -oP 'Model\s+\K\S+' | head -1 || echo "unknown")
  fi

  log INFO "切换前模型: $CURRENT_MODEL"

  # 执行切换脚本
  if [[ -x "$FALLBACK_SCRIPT" ]]; then
    log INFO "运行降级切换脚本..."
    if $FALLBACK_SCRIPT >> "$LOG_FILE" 2>&1; then
      log INFO "✓ 切换成功"

      # 获取新模型
      NEW_MODEL=$(openclaw status 2>/dev/null | grep -oP 'Model\s+\K\S+' | head -1 || echo "unknown")
      log INFO "切换后模型: $NEW_MODEL"

      # 更新状态
      cat > "$STATE_FILE" << EOF
{
  "last_switch": $(date +%s),
  "last_switch_from": "$CURRENT_MODEL",
  "last_switch_to": "$NEW_MODEL",
  "switch_reason": "$SWITCH_REASON",
  "consecutive_failures": 0
}
EOF

      # 发送通知（如果配置了）
      if command -v notify-send &> /dev/null; then
        notify-send "OpenClaw 模型切换" "已从 $CURRENT_MODEL 切换到 $NEW_MODEL" 2>/dev/null || true
      fi

      exit 0
    else
      log ERROR "✗ 切换失败"
      # 增加失败计数
      cat > "$STATE_FILE" << EOF
{
  "last_switch": $LAST_SWITCH,
  "consecutive_failures": $((CONSECUTIVE_FAILURES + 1))
}
EOF
      exit 1
    fi
  else
    log ERROR "切换脚本不存在或不可执行: $FALLBACK_SCRIPT"
    exit 1
  fi
else
  log INFO "无需切换: $SWITCH_REASON"

  # 更新失败计数
  cat > "$STATE_FILE" << EOF
{
  "last_switch": $LAST_SWITCH,
  "consecutive_failures": $((CONSECUTIVE_FAILURES + 1))
}
EOF

  exit 0
fi

#!/bin/bash
# 模型请求包装器 - 捕获错误并自动处理
# 用法: model-error-wrapper.sh --command "your-openclaw-command" --max-retries 3

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
AUTO_SWITCH_HANDLER="$SCRIPT_DIR/auto-switch-handler.sh"
LOG_DIR="$HOME/.openclaw/logs"
mkdir -p "$LOG_DIR"

# 默认参数
COMMAND=""
MAX_RETRIES=3
RETRY_DELAY=2
TIMEOUT=60
LOG_FILE="$LOG_DIR/wrapper.log"

# 解析参数
while [[ $# -gt 0 ]]; do
  case $1 in
    --command)
      COMMAND="$2"
      shift 2
      ;;
    --max-retries)
      MAX_RETRIES="$2"
      shift 2
      ;;
    --retry-delay)
      RETRY_DELAY="$2"
      shift 2
      ;;
    --timeout)
      TIMEOUT="$2"
      shift 2
      ;;
    --log-file)
      LOG_FILE="$2"
      shift 2
      ;;
    *)
      echo "未知参数: $1"
      exit 1
      ;;
  esac
done

# 日志函数
log() {
  local level=$1
  shift
  echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "$LOG_FILE"
}

# 执行命令并捕获错误
execute_with_retry() {
  local attempt=1
  local current_model=$(openclaw status 2>/dev/null | grep -oP 'Model\s+\K\S+' | head -1 || echo "unknown")

  while [[ $attempt -le $MAX_RETRIES ]]; do
    log INFO "执行命令 (尝试 $attempt/$MAX_RETRIES)..."
    log DEBUG "当前模型: $current_model"

    # 使用 timeout 命令执行
    if timeout $TIMEOUT bash -c "$COMMAND" 2>&1 | tee -a "$LOG_FILE"; then
      log INFO "✓ 命令执行成功"
      return 0
    else
      local exit_code=$?

      # 分析错误类型
      local error_type=""
      local error_code=""

      case $exit_code in
        124)  # timeout 退出码
          error_type="timeout"
          error_code="408"
          log WARN "请求超时 (超过 ${TIMEOUT}秒)"
          ;;
        28)   # curl 超时
          error_type="timeout"
          error_code="408"
          log WARN "网络超时"
          ;;
        7 | 35 | 52)  # curl 网络错误
          error_type="network_error"
          error_code="503"
          log WARN "网络连接失败"
          ;;
        *)
          # 检查错误输出
          error_output=$(tail -20 "$LOG_FILE")
          if echo "$error_output" | grep -qi "rate limit\|429\|too many requests"; then
            error_type="rate_limit"
            error_code="429"
            log WARN "API 速率限制"
          elif echo "$error_output" | grep -qi "quota\|insufficient\|credit"; then
            error_type="quota_exceeded"
            error_code="402"
            log WARN "配额耗尽"
          elif echo "$error_output" | grep -qi "auth\|401\|403\|unauthorized"; then
            error_type="authentication"
            error_code="401"
            log WARN "认证失败"
          elif echo "$error_output" | grep -qi "502\|503\|504\|service unavailable"; then
            error_type="service_unavailable"
            error_code="503"
            log WARN "服务不可用"
          else
            error_type="unknown"
            error_code="$exit_code"
            log WARN "未知错误 (退出码: $exit_code)"
          fi
          ;;
      esac

      # 调用自动切换处理脚本
      if [[ -x "$AUTO_SWITCH_HANDLER" ]]; then
        log INFO "调用自动切换处理..."
        $AUTO_SWITCH_HANDLER \
          --error-type "$error_type" \
          --error-code "$error_code" \
          --error-message "$(tail -1 "$LOG_FILE")" \
          --current-model "$current_model" \
          --retry-count "$attempt" || true
      fi

      # 如果是最后一次尝试，返回失败
      if [[ $attempt -eq $MAX_RETRIES ]]; then
        log ERROR "达到最大重试次数 ($MAX_RETRIES)，放弃"
        return 1
      fi

      # 等待后重试
      local wait_time=$((RETRY_DELAY * attempt))
      log INFO "等待 ${wait_time} 秒后重试..."
      sleep $wait_time

      # 更新当前模型（可能已经切换）
      current_model=$(openclaw status 2>/dev/null | grep -oP 'Model\s+\K\S+' | head -1 || echo "unknown")
      ((attempt++))
    fi
  done

  return 1
}

# 主函数
main() {
  if [[ -z "$COMMAND" ]]; then
    log ERROR "未指定命令，使用 --command 参数"
    exit 1
  fi

  log INFO "=== 模型请求包装器 ==="
  log INFO "命令: $COMMAND"
  log INFO "最大重试: $MAX_RETRIES"
  log INFO "超时设置: ${TIMEOUT}秒"

  execute_with_retry
  exit $?
}

# 如果直接运行
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
  main "$@"
fi

#!/bin/bash
# 模拟各种模型错误并测试自动切换

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
AUTO_SWITCH_HANDLER="$HOME/clawd/skills/model-fallback/scripts/auto-switch-handler.sh"
LOG_DIR="$HOME/.openclaw/logs"
mkdir -p "$LOG_DIR"

echo "========================================="
echo "  模型错误处理测试"
echo "========================================="
echo ""

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m'

# 测试用例
test_cases=(
  "timeout:超时错误"
  "rate_limit:速率限制"
  "quota_exceeded:配额耗尽"
  "authentication:认证失败"
  "service_unavailable:服务不可用"
  "network_error:网络错误"
)

for test_case in "${test_cases[@]}"; do
  IFS=':' read -r error_type description <<< "$test_case"

  echo -e "${YELLOW}测试: $description ($error_type)${NC}"

  # 模拟不同重试次数
  for retry_count in 1 2 3; do
    echo "  重试次数: $retry_count"

    # 调用自动切换处理脚本
    if $AUTO_SWITCH_HANDLER \
      --error-type "$error_type" \
      --error-code "500" \
      --error-message "模拟错误: $description" \
      --current-model "test-model" \
      --retry-count "$retry_count" 2>&1 | tee -a "$LOG_DIR/test-error.log"; then
      echo -e "    ${GREEN}✓${NC} 处理成功"
    else
      echo -e "    ${RED}✗${NC} 处理失败"
    fi

    sleep 1
  done

  echo ""
done

echo "========================================="
echo "  测试完成"
echo "========================================="
echo ""
echo "查看日志:"
echo "  cat $LOG_DIR/test-error.log"
echo "  cat $LOG_DIR/auto-switch.log"
echo ""