Story Long Scan

/**
 * CDP 工具函数 — 各平台采集脚本的公共依赖
 *
 * 使用方式：
 *   const { ab, sleep, evalJSON, scrollLoad, getArg, safeStr } = require("./cdp-utils");
 *
 * 前置：
 *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
 */

const { execSync } = require("child_process");

// ---------------------------------------------------------------------------
// agent-browser 工具函数
// ---------------------------------------------------------------------------

/**
 * 调用 agent-browser CLI
 * @param {number} port - CDP 端口
 * @param  {...string} args - agent-browser 参数
 * @returns {string} stdout（trim 后）
 */
function ab(port, ...args) {
  const cmd = args.map((a) => `"${a.replace(/"/g, '\\"')}"`).join(" ");
  try {
    return execSync(`agent-browser --cdp ${port} ${cmd}`, {
      encoding: "utf-8",
      timeout: 20000,
      stdio: ["pipe", "pipe", "pipe"],
    }).trim();
  } catch (e) {
    return e.stdout?.trim() || "";
  }
}

/** 等待 ms 毫秒（跨平台，不依赖系统 sleep 命令） */
function sleep(ms) {
  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
}

/** 在浏览器内执行 JS 并解析 JSON 返回值 */
function evalJSON(port, js) {
  const raw = ab(port, "eval", js);
  if (!raw || raw === "ERR") return null;
  try {
    let parsed = JSON.parse(raw);
    if (typeof parsed === "string") {
      try { parsed = JSON.parse(parsed); } catch {}
    }
    return parsed;
  } catch {
    return null;
  }
}

/**
 * 安全地将值插入浏览器 eval 字符串。
 * 使用 JSON.stringify 确保值不会因特殊字符（引号、反斜杠等）破坏 eval 字符串。
 * @param {*} val - 要插入的值
 * @returns {string} JSON 字符串表示（含引号）
 */
function safeStr(val) {
  return JSON.stringify(String(val));
}

/**
 * 滚动页面加载更多内容
 * @param {number} port - CDP 端口
 * @param {number} times - 滚动次数
 * @param {number} [interval=1000] - 每次滚动间隔（ms）
 */
function scrollLoad(port, times, interval = 1000) {
  for (let i = 0; i < times; i++) {
    ab(port, "eval", "window.scrollBy(0, window.innerHeight)");
    sleep(interval);
  }
}

/** 解析 --xxx 参数 */
function getArg(args, name) {
  const i = args.indexOf(name);
  return i >= 0 && i + 1 < args.length ? args[i + 1] : null;
}

module.exports = { ab, sleep, evalJSON, safeStr, scrollLoad, getArg };

#!/usr/bin/env node
/**
 * 刺猬猫阅读排行榜采集脚本
 *
 * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
 * 采集策略：刺猬猫 rank-index 页面单页展示所有榜单，文本解析提取结构化数据。
 * 输出 Markdown 格式匹配 scan-output-format.md 规范。
 *
 * 用法：
 *   node ciweimao-rank-scraper.js --type click       # 点击榜
 *   node ciweimao-rank-scraper.js --type monthly      # 月票榜
 *   node ciweimao-rank-scraper.js --type all           # 全部榜单
 *
 * 前置：
 *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
 */

const fs = require("fs");
const path = require("path");
const { ab, sleep, scrollLoad, getArg } = require("./cdp-utils");

const RANK_URL = "https://www.ciweimao.com/rank-index";

// eval 统一走 base64，规避复杂 JS 的 shell 转义问题（与 fanqie 一致）
function evalJSON(port, js) {
  const b64 = Buffer.from(String(js), "utf-8").toString("base64");
  const raw = ab(port, "eval", "-b", b64);
  if (!raw || raw === "ERR") return null;
  try {
    let parsed = JSON.parse(raw);
    if (typeof parsed === "string") {
      try { parsed = JSON.parse(parsed); } catch {}
    }
    return parsed;
  } catch {
    return null;
  }
}

/** 连通性 + 页面就绪自检 */
function probePage(port) {
  return evalJSON(
    port,
    "JSON.stringify({host:location.host,len:(document.body&&document.body.innerText||'').length})"
  );
}

const RANK_TYPES = [
  { id: "click", label: "点击榜", header: "点击榜" },
  { id: "favor", label: "收藏榜", header: "收藏榜" },
  { id: "recommend", label: "推荐榜", header: "推荐榜" },
  { id: "subscribe", label: "订阅榜", header: "订阅榜" },
  { id: "monthly", label: "月票榜", header: "月票榜" },
  { id: "tsukkomi", label: "吐槽榜", header: "吐槽榜" },
  { id: "newbook", label: "新书榜", header: "新书榜" },
  { id: "blade", label: "刀片榜", header: "刀片榜" },
  { id: "update", label: "更新榜", header: "更新榜" },
];

// ---------------------------------------------------------------------------
// 页面提取
// ---------------------------------------------------------------------------

/**
 * 从 rank-index 单页解析所有榜单。
 * 页面结构：每个榜单有标题行（如"点击榜"），后跟 NO.1 特殊条目 + #2-10 普通条目。
 * NO.1 格式：标题 / 作者 / 指标值（三行）
 * #2-10 格式：N[题材]书名 / 指标值（两行）
 */
function extractAllRanks(port) {
  const js =
    "JSON.stringify((()=>{" +
    "var text=document.body.innerText||'';" +
    "var lines=text.split(/\\n/).map(function(l){return l.trim()}).filter(Boolean);" +
    "var headers=['点击榜','收藏榜','推荐榜','订阅榜','月票榜','吐槽榜','新书榜','刀片榜','更新榜'];" +
    "var sections=[];var curName='';var curEntries=[];" +
    "for(var i=0;i<lines.length;i++){" +
    "  var line=lines[i];" +
    // 检测新 section
    "  var headerIdx=headers.indexOf(line);" +
    "  if(headerIdx>=0){" +
    "    if(curName&&curEntries.length)sections.push({name:curName,entries:curEntries});" +
    "    curName=headers[headerIdx];curEntries=[];continue" +
    "  }" +
    "  if(!curName)continue;" +
    // 跳过周期 tab 和 UI 文字
    "  if(/^(周榜|月榜|总榜)$/.test(line))continue;" +
    // NO.1 条目
    "  if(line==='NO.1'&&i+3<lines.length){" +
    "    var t=lines[i+1]||'';var a=lines[i+2]||'';var v=lines[i+3]||'';" +
    "    if(headers.indexOf(v)>=0)continue;" +
    "    curEntries.push({rank:1,title:t,author:a,genre:'',metric:v});" +
    "    i+=2;continue" +
    "  }" +
    // #2-10 条目：N[题材]书名
    "  var rm=line.match(/^(\\d{1,2})\\[(.+?)\\](.+)$/);" +
    "  if(rm){" +
    "    var nextVal=i+1<lines.length?lines[i+1]:'';" +
    "    var metric='';" +
    "    if(/^[\\d.]+(万)?$/.test(nextVal)){metric=nextVal;i++}" +
    "    curEntries.push({rank:parseInt(rm[1]),title:rm[3],author:'',genre:rm[2],metric:metric});" +
    "    continue" +
    "  }" +
    "}" +
    "if(curName&&curEntries.length)sections.push({name:curName,entries:curEntries});" +
    "return sections" +
    "})())";
  return evalJSON(port, js) || [];
}

/**
 * 从 DOM 获取书籍链接。每本书常有封面图 anchor（textContent 为空）和书名 anchor，
 * 按 bookId 聚合后取最长的非空文本作为书名，避免空封面 anchor 覆盖书名导致回填全失败。
 */
function extractBookUrls(port) {
  const js = `JSON.stringify((function(){
    function clean(t){return t.replace(/^[0-9]+\\[[^\\]]*\\]/,'').replace(/\\s+[0-9.]+(?:万|亿)?$/,'').trim();}
    var byId={};var order=[];
    Array.from(document.querySelectorAll('a[href*="/book/"]')).forEach(function(a){
      var h=a.getAttribute('href')||a.href||'';
      var m=h.match(/\\/book\\/([0-9]+)/);
      if(!m)return; var id=m[1];
      var t=clean((a.innerText||a.textContent||'').replace(/\\s+/g,' ').trim());
      if(!byId[id]){byId[id]='';order.push(id);}
      if(t&&t.length>byId[id].length)byId[id]=t;
    });
    return order.map(function(id){return {bookId:id,title:byId[id],url:'https://www.ciweimao.com/book/'+id};});
  })())`;
  return evalJSON(port, js) || [];
}

// ---------------------------------------------------------------------------
// 主流程
// ---------------------------------------------------------------------------

const args = process.argv.slice(2);
const PORT = parseInt(getArg(args, "--port") || "9222", 10);
const OUTDIR = getArg(args, "--outdir") || ".";
const RANKTYPE = getArg(args, "--type") || "all";

function main() {
  console.log("\n→ 采集 刺猬猫排行榜...");
  console.log(`  URL: ${RANK_URL}`);

  let sections, urls;
  try {
    ab(PORT, "open", RANK_URL);
    sleep(4000);

    // 连通性自检：CDP 未起/被重定向时给可操作报错，而非误报"结构已变"
    const probe = probePage(PORT);
    if (!probe) {
      console.error(
        `  ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome（端口 ${PORT}），且 agent-browser 可用。`
      );
      return;
    }
    if (probe.host && probe.host.indexOf("ciweimao") === -1) {
      console.error(`  ✗ 当前页面非刺猬猫（host=${probe.host}），可能被重定向，已跳过。`);
      return;
    }

    scrollLoad(PORT, 3);
    sleep(1000);

    sections = extractAllRanks(PORT);
    if (!sections.length) {
      // 懒加载可能未触发，再滚动重试一次
      scrollLoad(PORT, 2);
      sleep(1000);
      sections = extractAllRanks(PORT);
    }
    if (!sections.length) {
      console.error("[ciweimao] 采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开榜单页确认。");
      return;
    }

    urls = extractBookUrls(PORT);
  } catch (err) {
    console.error(`[ciweimao] 采集失败（页面加载或提取阶段）: ${err.message}`);
    return;
  }

  console.log(`  ✓ 提取 ${sections.length} 个榜单，${urls.length} 个书籍链接`);

  // 筛选需要的榜单类型
  const targetTypes =
    RANKTYPE === "all"
      ? RANK_TYPES
      : RANK_TYPES.filter((r) => r.id === RANKTYPE);

  for (const rt of targetTypes) {
    try {
      const section = sections.find((s) => s.name === rt.header);
      if (!section || !section.entries.length) {
        console.log(`  ⚠ ${rt.label} 无数据，跳过`);
        continue;
      }

      const now = new Date().toISOString();
      const norm = (s) => (s || "").replace(/\s+/g, "");
      const linked = section.entries.filter((e) =>
        urls.some((u) => norm(u.title) === norm(e.title))
      ).length;
      const lines = [
        `# 刺猬猫 · ${rt.label}`,
        "",
        `- 来源：${RANK_URL}`,
        `- 抓取时间：${now}`,
        `- 条目数：${section.entries.length}`,
        `- 作品页链接：${linked} / ${section.entries.length}`,
        "",
        "---",
        "",
      ];

      for (const entry of section.entries) {
        try {
          lines.push(`### #${entry.rank} ${entry.title}`);
          const meta = [
            entry.author,
            entry.genre,
            entry.metric || "",
          ].filter(Boolean).join(" · ");
          if (meta) lines.push(`*${meta}*`);

          // 按标题匹配书籍链接（归一后比对）
          const matched = urls.find((u) => norm(u.title) === norm(entry.title));
          if (matched) {
            lines.push(`[作品页](${matched.url})`);
          }

          lines.push("", "---", "");
        } catch (entryErr) {
          console.error(`[ciweimao] ${rt.label} 条目处理出错（#${entry.rank} ${entry.title}）: ${entryErr.message}`);
          lines.push("", "---", "");
        }
      }

      const date = new Date().toISOString().slice(0, 10).replace(/-/g, "");
      const filename = `刺猬猫${rt.label}_${date}.md`;
      fs.mkdirSync(OUTDIR, { recursive: true });
      const filepath = path.join(OUTDIR, filename);
      fs.writeFileSync(filepath, lines.join("\n"), "utf-8");
      console.log(`  ✓ ${rt.label}：${section.entries.length} 条 → ${filepath}`);
    } catch (rankErr) {
      console.error(`[ciweimao] ${rt.label} 处理出错，跳过: ${rankErr.message}`);
    }
  }
}

try {
  main();
} catch (e) {
  console.error(`刺猬猫采集失败: ${e && e.message ? e.message : e}`);
  process.exit(1);
}

#!/usr/bin/env node
/**
 * 番茄小说排行榜采集脚本
 *
 * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
 * 采集策略：从榜单页 __INITIAL_STATE__ 取结构化列表，再逐本请求详情页解码真实
 * 书名/作者/简介/题材/标签（番茄列表页有字体反爬，详情页 HTML 里是明文）。
 * 输出 Markdown 格式匹配 scan-output-format.md 规范。
 *
 * 用法：
 *   node fanqie-rank-scraper.js --channel 1 --type 2              # 男频阅读榜
 *   node fanqie-rank-scraper.js --channel 0 --type 1              # 女频新书榜
 *   node fanqie-rank-scraper.js --channel 1 --type 2 --outdir ./  # 指定输出目录
 *   node fanqie-rank-scraper.js --channel all                     # 全部采集
 *   node fanqie-rank-scraper.js --channel 1 --top 15              # 每题材只取前 15 本
 *
 * 前置：
 *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
 */

const fs = require("fs");
const path = require("path");
const { ab, sleep, scrollLoad, getArg } = require("./cdp-utils");

// 一次详情请求的并发批大小。番茄详情页用同步 XHR 拉取，批太大会撞上
// cdp-utils 里 ab() 的 20s 超时，导致整批返回空 → 书名全部回退成 bookId。
const DETAIL_CHUNK = 5;

// ---------------------------------------------------------------------------
// eval 封装：统一走 base64，规避复杂 JS（正则/引号/反斜杠）的 shell 转义问题
// ---------------------------------------------------------------------------

/** 在浏览器内执行 JS（base64 传参）并解析 JSON 返回值 */
function evalJSON(port, js) {
  const b64 = Buffer.from(String(js), "utf-8").toString("base64");
  const raw = ab(port, "eval", "-b", b64);
  if (!raw || raw === "ERR") return null;
  try {
    let parsed = JSON.parse(raw);
    if (typeof parsed === "string") {
      try { parsed = JSON.parse(parsed); } catch {}
    }
    return parsed;
  } catch {
    return null;
  }
}

// ---------------------------------------------------------------------------
// 页面提取
// ---------------------------------------------------------------------------

/** 连通性 + 页面就绪自检 */
function probePage(port) {
  return evalJSON(
    port,
    "JSON.stringify({host:location.host,hasState:!!window.__INITIAL_STATE__})"
  );
}

/** 构建：提取侧边菜单品类链接的浏览器 JS */
function buildCategoriesJS(prefix) {
  return `JSON.stringify((function(){
    var prefix=${JSON.stringify(prefix)};
    var out=[];var seen={};
    Array.from(document.querySelectorAll('a')).forEach(function(a){
      var href=a.getAttribute('href')||'';
      if(href.indexOf(prefix)===-1)return;
      var name=(a.innerText||a.textContent||'').trim();
      if(!name)return;
      if(seen[href])return;seen[href]=1;
      out.push({name:name,href:href});
    });
    return out;
  })())`;
}

/** 提取侧边菜单品类链接 */
function extractCategories(port, channel, type) {
  const prefix = `/rank/${channel}_${type}_`;
  return evalJSON(port, buildCategoriesJS(prefix)) || [];
}

/**
 * 从 __INITIAL_STATE__ 提取当前品类页的作品列表。
 * 多路径尝试 + 深度兜底扫描，并把字段名归一，避免站点改 state 结构就全盘失败。
 */
function buildBookListJS() {
  return `JSON.stringify((function(){
    var s=window.__INITIAL_STATE__||{};
    var cands=[
      s.rank&&s.rank.book_list, s.rank&&s.rank.bookList, s.rank&&s.rank.rankList,
      s.rankData&&s.rankData.book_list, s.page&&s.page.book_list
    ];
    var list=null;
    for(var i=0;i<cands.length;i++){ if(Array.isArray(cands[i])&&cands[i].length){list=cands[i];break;} }
    if(!list){
      var found=null;
      (function walk(o,d){
        if(found||!o||d>6)return;
        if(Array.isArray(o)){
          if(o.length&&o[0]&&typeof o[0]==='object'&&(o[0].bookId||o[0].book_id)){found=o;return;}
          for(var j=0;j<o.length&&!found;j++)walk(o[j],d+1);return;
        }
        if(typeof o==='object'){ for(var k in o){ if(found)break; try{walk(o[k],d+1)}catch(e){} } }
      })(s,0);
      list=found||[];
    }
    return list.map(function(b){return {
      bookId:String(b.bookId||b.book_id||''),
      read_count:b.read_count||b.readCount||b.read||'',
      wordNumber:b.wordNumber||b.word_number||b.wordCount||'',
      creationStatus:(b.creationStatus!=null?b.creationStatus:(b.creation_status!=null?b.creation_status:b.status)),
      lastChapterTitle:b.lastChapterTitle||b.last_chapter_title||b.lastChapter||'',
      category:b.category||b.categoryName||b.category_name||''
    };}).filter(function(b){return b.bookId;});
  })())`;
}

function extractBookList(port) {
  const list = evalJSON(port, buildBookListJS());
  return Array.isArray(list) ? list : [];
}

/**
 * 批量解码详情：逐本同步 XHR 请求 /page/{id}，多策略解析明文字段。
 * 番茄列表页书名/作者被字体反爬，详情页 HTML 内嵌 JSON 与 <title> 是明文。
 * 字段名以真实 SSR(__INITIAL_STATE__) 为准：bookName/author/abstract 明文，
 * 题材在 categoryV2(转义 JSON 数组的首个 Name)，番茄 SSR 不含数字评分。
 * 返回 { id: {title, author, desc, category, tags} }。
 */
function buildDetailJS(ids) {
  return `JSON.stringify((function(){
    var ids=${JSON.stringify(ids)};
    var map={};
    function pick(h,res){for(var i=0;i<res.length;i++){var m=h.match(res[i]);if(m&&m[1])return m[1].trim();}return '';}
    for(var k=0;k<ids.length;k++){
      var id=ids[k];
      try{
        var x=new XMLHttpRequest();
        x.open('GET','/page/'+id,false);
        x.send();
        var h=x.responseText||'';
        var title=pick(h,[
          /"bookName"\\s*:\\s*"([^"]+)"/,
          /<title>([^<]*?)(?:完整版|最新章节|在线阅读|_番茄小说|-番茄小说|_番茄|-番茄)/,
          /<meta[^>]+property="og:title"[^>]+content="([^"]+)"/,
          /<title>([^<|_]{1,40})/
        ]);
        var author=pick(h,[
          /"author"\\s*:\\s*"([^"]+)"/,
          /"authorName"\\s*:\\s*"([^"]+)"/,
          /<meta[^>]+property="og:novel:author"[^>]+content="([^"]+)"/
        ]);
        // abstract(真实简介)优先；meta description 是平台模板("番茄小说提供...")，
        // 且常带 data-rh 属性，故用宽松属性匹配兜底。
        var abs=pick(h,[/"abstract"\\s*:\\s*"([^"]{6,}?)"/]);
        var desc=abs||pick(h,[
          /<meta[^>]+name="description"[^>]+content="([^"]+)"/,
          /<meta[^>]+property="og:description"[^>]+content="([^"]+)"/
        ]);
        // 题材：category 常为空字符串，真实题材在 categoryV2(转义 JSON)首个 Name。
        var category=pick(h,[
          /"categoryV2":"\\[\\{[\\s\\S]*?\\\\"Name\\\\":\\\\"([^"\\\\]+)/,
          /"category"\\s*:\\s*"([^"]{1,20})"/,
          /<meta[^>]+property="og:novel:category"[^>]+content="([^"]+)"/
        ]);
        // 标签：番茄简介开头常带【tag+tag+...】，是题材细分的真实信号。
        var tags='';
        var bm=(abs||desc||'').match(/[【\\[]([^】\\]]{2,40})[】\\]]/);
        if(bm){tags=bm[1].split(/[+、,\\/\\s]+/).filter(Boolean).slice(0,6).join('、');}
        map[id]={title:title,author:author,desc:desc,category:category,tags:tags};
      }catch(e){
        map[id]={title:'',author:'',desc:'',category:'',tags:'',err:String(e&&e.message||e)};
      }
    }
    return map;
  })())`;
}

function fetchDetailsChunk(port, ids) {
  return evalJSON(port, buildDetailJS(ids)) || {};
}

/** 分批解码，避免单次 eval 超时；返回合并后的 map */
function fetchDetails(port, bookIds) {
  const map = {};
  for (let i = 0; i < bookIds.length; i += DETAIL_CHUNK) {
    const chunk = bookIds.slice(i, i + DETAIL_CHUNK);
    const part = fetchDetailsChunk(port, chunk);
    Object.assign(map, part);
    sleep(300);
  }
  return map;
}

// ---------------------------------------------------------------------------
// 格式化
// ---------------------------------------------------------------------------

function fmtReads(count) {
  if (!count || count === "0") return "未知";
  const n = parseInt(count, 10);
  if (isNaN(n)) return "未知";
  if (n >= 10000) return (n / 10000).toFixed(1) + "万";
  return String(n);
}

function fmtWords(count) {
  if (!count) return "未知";
  const n = parseInt(count, 10);
  if (isNaN(n)) return "未知";
  if (n >= 10000) return (n / 10000).toFixed(1) + "万";
  return String(n);
}

function fmtStatus(s) {
  const v = String(s);
  if (v === "1") return "连载中";
  if (v === "0" || v === "2") return "已完结";
  return s ? String(s) : "未知";
}

/** 清洗简介：去平台模板文本 → 折叠空白 → 句末截断 100 字 */
function cleanDesc(raw) {
  if (!raw) return "";
  let d = String(raw)
    // 简介取自 JSON 字符串原文，先还原常见转义（\n \uXXXX \" 等）
    .replace(/\\u([0-9a-fA-F]{4})/g, (_, h) => String.fromCharCode(parseInt(h, 16)))
    .replace(/\\[nrt]/g, " ")
    .replace(/\\"/g, '"')
    .replace(/番茄小说[^。！？]*?(?:免费阅读|完整版|在线阅读)[^。！？]*[。！？]/g, "")
    .replace(/番茄小说[^。！？]*?(?:免费阅读|完整版|在线阅读)[^。！？]*$/g, "")
    .replace(/\s+/g, " ")
    .trim();
  if (d.length <= 100) return d;
  const cut = d.slice(0, 100);
  const m = cut.match(/^[\s\S]*[。！？]/);
  return (m ? m[0] : cut) + "...";
}

// ---------------------------------------------------------------------------
// 主流程
// ---------------------------------------------------------------------------

const args = process.argv.slice(2);
const PORT = parseInt(getArg(args, "--port") || "9222", 10);
const OUTDIR = getArg(args, "--outdir") || ".";
const CHANNEL = getArg(args, "--channel") || "1";
const TYPE = getArg(args, "--type") || "2";
const TOP = parseInt(getArg(args, "--top") || "20", 10);

function channelLabel(ch) {
  return ch === "1" ? "男频" : "女频";
}

function typeLabel(t) {
  return t === "2" ? "阅读榜" : "新书榜";
}

function scrapeChannel(ch, type) {
  const chLabel = channelLabel(ch);
  const tyLabel = typeLabel(type);
  console.log(`\n→ 采集 ${chLabel}${tyLabel}...`);

  // 用已知品类 ID 作为入口，确保菜单只显示当前频道/类型的品类
  const initCatId = ch === "1" ? "1141" : "1139"; // 男频:西方奇幻 / 女频:古风世情
  const initUrl = `https://fanqienovel.com/rank/${ch}_${type}_${initCatId}`;
  ab(PORT, "open", initUrl);
  sleep(3000);

  // 连通性自检：把"静默写出一堆 bookId"变成可操作的报错
  const probe = probePage(PORT);
  if (!probe) {
    console.error(
      `  ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome（端口 ${PORT}），且 agent-browser 可用。`
    );
    return null;
  }
  if (probe.host && probe.host.indexOf("fanqie") === -1) {
    console.error(
      `  ✗ 当前页面非番茄（host=${probe.host}），可能被重定向到登录/验证页，已跳过。`
    );
    return null;
  }
  if (!probe.hasState) {
    console.error(`  ⚠ 页面未挂载 __INITIAL_STATE__，将尝试兜底扫描，结果可能不完整。`);
  }

  let categories = extractCategories(PORT, ch, type);
  if (!categories.length) {
    // 菜单可能懒加载，滚动后重试一次
    scrollLoad(PORT, 2);
    sleep(1000);
    categories = extractCategories(PORT, ch, type);
  }
  if (!categories.length) {
    // 仍失败：降级为只采当前入口页，至少产出数据而不是空跑
    console.log(`  ⚠ 未提取到品类菜单，降级为单题材采集（入口页）`);
    categories = [{ name: "全部（入口页）", href: `/rank/${ch}_${type}_${initCatId}` }];
  } else {
    console.log(`  发现 ${categories.length} 个品类`);
  }

  const now = new Date().toISOString();
  const lines = [
    `# 番茄 · ${chLabel}${tyLabel} · 全 ${categories.length} 题材`,
    "",
    `- 频道参数：channel=${ch}，type=${type}`,
    `- 抓取时间：${now}`,
    `- 每题材上限 ≈ ${TOP}`,
    "",
    "---",
    "",
  ];

  let totalBooks = 0;
  let resolvedTitles = 0;
  const bodyLines = [];

  for (let ci = 0; ci < categories.length; ci++) {
    const cat = categories[ci];
    console.log(`  [${ci + 1}/${categories.length}] ${cat.name}`);

    try {
      ab(PORT, "open", `https://fanqienovel.com${cat.href}`);
      sleep(2500);
      scrollLoad(PORT, 2);

      let books = extractBookList(PORT);
      if (!Array.isArray(books) || !books.length) {
        bodyLines.push(`## ${cat.name} — 0 本`, "", "---", "");
        continue;
      }
      if (books.length > TOP) books = books.slice(0, TOP);

      // 分批解码真实书名/作者/简介/题材/评分/标签
      const bookIds = books.map((b) => String(b.bookId));
      const details = fetchDetails(PORT, bookIds);

      bodyLines.push(`## ${cat.name} — ${books.length} 本`, "");

      for (let i = 0; i < books.length; i++) {
        const b = books[i];
        const info = details[String(b.bookId)] || {};
        totalBooks++;
        const resolved = !!info.title;
        if (resolved) resolvedTitles++;

        const title = info.title || "（标题待解析）";
        const author = info.author || "未知";
        const category = info.category || b.category || "";
        const catSeg = category ? ` · ${category}` : "";

        bodyLines.push(`### #${i + 1} ${title}`);
        bodyLines.push(
          `*${author}${catSeg} · ${fmtStatus(b.creationStatus)} · ${fmtReads(b.read_count)} 在读 · ${fmtWords(b.wordNumber)}字*`
        );
        if (info.tags) bodyLines.push(`**标签：** ${info.tags}`);
        bodyLines.push(`**最新更新：** ${b.lastChapterTitle || "未知"}`);
        bodyLines.push(`**bookId：** ${b.bookId}`);
        bodyLines.push(`[作品页](https://fanqienovel.com/page/${b.bookId})`);
        const desc = cleanDesc(info.desc);
        if (desc) {
          bodyLines.push("");
          bodyLines.push("**简介**");
          bodyLines.push("");
          bodyLines.push(desc);
        }
        bodyLines.push("");
      }

      bodyLines.push("---", "");
    } catch (catErr) {
      console.error(
        `  [fanqie] 品类 ${cat.name} 处理出错，跳过: ${catErr && catErr.message ? catErr.message : catErr}`
      );
      bodyLines.push(`## ${cat.name} — 采集失败`, "", "---", "");
    }
  }

  // 质量状态：标题解析比例是番茄采集成败的核心信号
  const ratio = totalBooks ? resolvedTitles / totalBooks : 0;
  const quality = totalBooks === 0
    ? "[无数据]"
    : ratio < 0.5
      ? "[标题解析异常]"
      : "[OK]";
  lines.splice(5, 0,
    `- 标题解析：成功 ${resolvedTitles} / 共 ${totalBooks}`,
    `- 数据质量：${quality}`
  );

  if (totalBooks > 0 && resolvedTitles === 0) {
    console.error(
      `  ✗ ${chLabel}${tyLabel}：${totalBooks} 本全部标题解析失败。多为详情页结构变动或登录/验证拦截，` +
      `请在 Chrome 内手动打开任一 https://fanqienovel.com/page/{bookId} 确认页面正常。`
    );
  } else if (ratio < 0.5) {
    console.error(
      `  ⚠ ${chLabel}${tyLabel}：标题解析率偏低（${resolvedTitles}/${totalBooks}），结果质量已标注。`
    );
  }

  return lines.concat(bodyLines).join("\n");
}

function main() {
  const channels = CHANNEL === "all" ? ["1", "0"] : [CHANNEL];
  const types = TYPE === "all" ? ["2", "1"] : [TYPE];

  for (const ch of channels) {
    for (const ty of types) {
      try {
        const content = scrapeChannel(ch, ty);
        if (!content) continue;

        const date = new Date().toISOString().slice(0, 10).replace(/-/g, "");
        const filename = `番茄${channelLabel(ch)}${typeLabel(ty)}_全题材_${date}.md`;
        fs.mkdirSync(OUTDIR, { recursive: true });
        const filepath = path.join(OUTDIR, filename);
        fs.writeFileSync(filepath, content, "utf-8");
        console.log(`  ✓ 已保存: ${filepath}`);
      } catch (chErr) {
        console.error(
          `[fanqie] ${channelLabel(ch)}${typeLabel(ty)} 采集失败，跳过: ${chErr && chErr.message ? chErr.message : chErr}`
        );
      }
    }
  }
}

if (require.main === module) {
  try {
    main();
  } catch (e) {
    console.error(`番茄采集失败: ${e && e.message ? e.message : e}`);
    process.exit(1);
  }
}

// 导出纯函数/JS 构建器，供测试在 sandbox 内验证解析逻辑
module.exports = {
  buildCategoriesJS,
  buildBookListJS,
  buildDetailJS,
  fmtReads,
  fmtWords,
  fmtStatus,
  cleanDesc,
};

#!/usr/bin/env node
/**
 * 晋江文学城排行榜采集脚本
 *
 * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
 * 采集策略：
 *   1) topten.php 列表页（纯文本，频道名直接出现，书名/作者交替行）解出频道分组。
 *   2) 从书名 anchor 取 novelid，逐本进 onebook.php 详情页补采核心指标
 *      （收藏数/营养液/积分/字数/状态），满足规范对晋江的硬性要求。
 * 晋江页面为 gb18030 编码：详情页用 fetch+arrayBuffer+TextDecoder('gb18030') 解码
 *   （同步 XHR 的 responseText 会按 UTF-8 解码导致中文乱码）。
 * 详情采集默认开启但有上限（每频道前 N + 总量上限），用 --list-only 可只采列表。
 *
 * 用法：
 *   node jjwxc-rank-scraper.js --type 12                  # 收入金榜（默认含详情）
 *   node jjwxc-rank-scraper.js --type 12 --top 15         # 每频道补采前 15 本
 *   node jjwxc-rank-scraper.js --type 12 --detail-limit 60 # 详情总量上限 60
 *   node jjwxc-rank-scraper.js --type 12 --list-only      # 只采列表（快，无核心指标）
 *   node jjwxc-rank-scraper.js --type all                 # 全部榜单
 *
 * 前置：
 *   node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
 */

const fs = require("fs");
const path = require("path");
const { ab, sleep, getArg } = require("./cdp-utils");

const BASE_URL = "https://www.jjwxc.net/topten.php";

const RANK_TYPES = [
  { id: "12", label: "收入金榜" },
  { id: "7", label: "月榜" },
  { id: "8", label: "季度榜" },
  { id: "14", label: "完结金榜" },
  { id: "15", label: "新手金榜" },
  { id: "17", label: "千字金榜" },
];

// 详情请求批大小（async fetch 并发，整批控制在 ab() 20s 超时内）
const DETAIL_CHUNK = 6;

// ---------------------------------------------------------------------------
// eval 封装：统一走 base64，规避复杂 JS 的 shell 转义问题（与 fanqie 一致）
// ---------------------------------------------------------------------------

function evalJSON(port, js) {
  const b64 = Buffer.from(String(js), "utf-8").toString("base64");
  const raw = ab(port, "eval", "-b", b64);
  if (!raw || raw === "ERR") return null;
  try {
    let parsed = JSON.parse(raw);
    if (typeof parsed === "string") {
      try { parsed = JSON.parse(parsed); } catch {}
    }
    return parsed;
  } catch {
    return null;
  }
}

/** 连通性 + 页面就绪自检 */
function probePage(port) {
  return evalJSON(
    port,
    "JSON.stringify({host:location.host,len:(document.body&&document.body.innerText||'').length})"
  );
}

// ---------------------------------------------------------------------------
// 列表页提取
// ---------------------------------------------------------------------------

/**
 * 提取晋江榜单数据（频道分组 + 书名/作者交替），并从书名 anchor 附上 novelid。
 */
function extractRankData(port) {
  const js =
    "JSON.stringify((function(){" +
    "var result={channels:[]};" +
    "var text=document.body.innerText||'';" +
    "var lines=text.split(/\\n/).map(function(l){return l.trim()}).filter(Boolean);" +
    // 书名 anchor → novelid（排除霸王票"X向《书名》投了Y"这类记录）
    "var idMap={};" +
    "Array.from(document.querySelectorAll('a')).forEach(function(a){" +
    "  var hm=(a.getAttribute('href')||'').match(/novelid=([0-9]+)/);if(!hm)return;" +
    "  var t=(a.innerText||a.textContent||'').trim();" +
    "  if(!t||t.indexOf('向《')>-1||t.indexOf('投')>-1||t.length>30)return;" +
    "  if(!idMap[t])idMap[t]=hm[1];" +
    "});" +
    "var channels=['古代言情','现代言情','古代穿越','现代都市纯爱','现代幻想纯爱','古代纯爱','衍生纯爱','幻想现言','奇幻言情','未来游戏悬疑','百合','无CP','二次元言情','衍生言情','衍生无cp','未来幻想纯爱','原创轻小说','多元'];" +
    "var channelSet={};channels.forEach(function(c){channelSet[c]=true});" +
    "var curChannel='';" +
    "var channelBooks={};" +
    "var expectTitle=true;" +
    "var pendingTitle='';" +
    "for(var i=0;i<lines.length;i++){" +
    "  var line=lines[i];" +
    "  if(/上榜天数记录|榜单说明/.test(line)){break}" +
    "  if(/^(免费强推|vip强推|新晋作者|月榜|季榜|半年榜|长生殿|总分榜|字数榜|收入金榜|霸王票|霸王总榜|勤奋指数|完结金榜|新手金榜|栽培月榜|驻站|完结高分|千字金榜|完结全订榜)$/.test(line)){continue}" +
    "  if(line.length>30&&line.indexOf('·')>0)continue;" +
    "  if(channelSet[line]){" +
    "    if(curChannel&&channelBooks[curChannel])channelBooks[curChannel]._finished=true;" +
    "    curChannel=line;" +
    "    if(!channelBooks[curChannel])channelBooks[curChannel]={books:[]};" +
    "    expectTitle=true;pendingTitle='';continue" +
    "  }" +
    "  if(!curChannel)continue;" +
    "  if(expectTitle){" +
    "    pendingTitle=line;expectTitle=false" +
    "  }else{" +
    "    if(pendingTitle){" +
    "      channelBooks[curChannel].books.push({title:pendingTitle,author:line,novelid:idMap[pendingTitle]||''})" +
    "    }" +
    "    expectTitle=true;pendingTitle=''" +
    "  }" +
    "}" +
    "for(var name in channelBooks){" +
    "  if(channelBooks[name].books.length>0){" +
    "    result.channels.push({name:name,books:channelBooks[name].books})" +
    "  }" +
    "}" +
    "return result" +
    "})())";
  return evalJSON(port, js);
}

// ---------------------------------------------------------------------------
// 详情页提取（gb18030 + itemprop 微数据）
// ---------------------------------------------------------------------------

/** 构建：一批 novelid 的详情解码 JS（async fetch + TextDecoder，返回 JSON 字符串） */
function buildDetailJS(ids) {
  return `Promise.all(${JSON.stringify(ids)}.map(function(id){
    return fetch('/onebook.php?novelid='+id)
      .then(function(r){return r.arrayBuffer()})
      .then(function(b){
        var h=new TextDecoder('gb18030').decode(new Uint8Array(b));
        function prop(n){var m=h.match(new RegExp('itemprop="'+n+'"[^>]*>([^<]*)<'));return m?m[1].trim():'';}
        var status=(h.match(/itemprop="updataStatus"[^>]*>\\s*([^<\\s]{1,6})/)||[,''])[1]
                 ||(h.match(/(连载中|已完结|完结)/)||[,''])[1]||'';
        return {id:id,collect:prop('collectedCount'),nutrition:prop('nutritionCount'),
                score:prop('scoreCount'),review:prop('reviewCount'),words:prop('wordCount'),status:status};
      })
      .catch(function(e){return {id:id,err:String(e&&e.message||e)}});
  })).then(function(arr){var map={};arr.forEach(function(o){map[o.id]=o});return JSON.stringify(map);})`;
}

/** 分批解码详情，合并结果 */
function fetchDetails(port, ids) {
  const map = {};
  for (let i = 0; i < ids.length; i += DETAIL_CHUNK) {
    const chunk = ids.slice(i, i + DETAIL_CHUNK);
    const part = evalJSON(port, buildDetailJS(chunk)) || {};
    Object.assign(map, part);
    sleep(400);
  }
  return map;
}

// ---------------------------------------------------------------------------
// 格式化
// ---------------------------------------------------------------------------

function fmtWan(s, unit) {
  if (s == null || s === "") return "";
  const n = parseInt(String(s).replace(/[^0-9]/g, ""), 10);
  if (isNaN(n)) return "";
  if (n >= 10000) return (n / 10000).toFixed(1) + "万" + (unit || "");
  return n + (unit || "");
}

// ---------------------------------------------------------------------------
// 主流程
// ---------------------------------------------------------------------------

const args = process.argv.slice(2);
const PORT = parseInt(getArg(args, "--port") || "9222", 10);
const OUTDIR = getArg(args, "--outdir") || ".";
const RANKTYPE = getArg(args, "--type") || "12";
const CHANNEL = getArg(args, "--channel") || "0";
const TOP = parseInt(getArg(args, "--top") || "10", 10);
const DETAIL_LIMIT = parseInt(getArg(args, "--detail-limit") || "100", 10);
const LIST_ONLY = args.includes("--list-only");

function scrapeRank(port, rankTypeId, channelId) {
  const rt = RANK_TYPES.find((r) => r.id === rankTypeId);
  if (!rt) {
    console.log(`  ⚠ 未知榜单类型: ${rankTypeId}`);
    return null;
  }

  const url = `${BASE_URL}?orderstr=${rankTypeId}&t=${channelId}`;
  const chLabel = channelId === "0" ? "全站" : `频道${channelId}`;
  console.log(`\n→ 采集 晋江${rt.label}（${chLabel}）...`);
  console.log(`  URL: ${url}`);

  let data;
  try {
    ab(port, "open", url);
    sleep(4000);

    // 连通性自检：CDP 未起/被重定向时给可操作报错，而非误报"结构已变"
    const probe = probePage(port);
    if (!probe) {
      console.error(
        `  ✗ CDP 无响应。请确认已用 browser-cdp 启动 Chrome（端口 ${port}），且 agent-browser 可用。`
      );
      return null;
    }
    if (probe.host && probe.host.indexOf("jjwxc") === -1) {
      console.error(`  ✗ 当前页面非晋江（host=${probe.host}），可能被重定向，已跳过。`);
      return null;
    }

    data = extractRankData(port);
    if (!data?.channels?.length) {
      console.error(`[jjwxc] 采集失败：未解析到榜单（页面结构可能变动或未加载）。请人工打开 ${url} 确认。`);
      return null;
    }
  } catch (err) {
    console.error(`[jjwxc] ${rt.label} 页面加载或提取出错: ${err.message}`);
    return null;
  }

  let totalBooks = 0;
  data.channels.forEach((ch) => {
    totalBooks += ch.books.length;
    const authors = new Set(ch.books.map((b) => b.author));
    if (ch.books.length >= 5 && authors.size / ch.books.length < 0.2) {
      console.log(`  ⚠ ${ch.name}：${ch.books.length} 本只有 ${authors.size} 个唯一作者，可能提取有误`);
    }
  });
  console.log(`  ✓ 列表：${data.channels.length} 个频道，共 ${totalBooks} 本`);

  // 选取每频道前 TOP 本（有 novelid 的）补采详情，受 DETAIL_LIMIT 总量约束
  let detailMap = {};
  let detailPlanned = 0;
  let detailOk = 0;
  if (!LIST_ONLY) {
    const picked = [];
    for (const ch of data.channels) {
      let n = 0;
      for (const b of ch.books) {
        if (picked.length >= DETAIL_LIMIT) break;
        if (n >= TOP) break;
        if (b.novelid) { picked.push(b.novelid); n++; }
      }
      if (picked.length >= DETAIL_LIMIT) break;
    }
    detailPlanned = picked.length;
    if (picked.length) {
      console.log(`  → 补采详情 ${picked.length} 本（每频道前 ${TOP}，上限 ${DETAIL_LIMIT}）...`);
      detailMap = fetchDetails(port, picked);
      detailOk = Object.values(detailMap).filter((d) => d && d.collect).length;
      console.log(`  ✓ 详情命中收藏数 ${detailOk}/${picked.length}`);
    }
  }

  // 质量状态：详情开启时，收藏数命中率是核心信号
  let quality = "[OK]";
  if (!LIST_ONLY && detailPlanned > 0 && detailOk === 0) {
    quality = "[详情解析异常/登录态缺失]";
    console.error(`  ⚠ 详情全部无收藏数：可能页面结构变动或需登录，已在文件头标注。`);
  } else if (LIST_ONLY) {
    quality = "[仅列表-无核心指标]";
  }

  const now = new Date().toISOString();
  const lines = [
    `# 晋江 · ${rt.label}`,
    "",
    `- 来源：${url}`,
    `- 抓取时间：${now}`,
    `- 频道数：${data.channels.length}`,
    `- 总条目数：${totalBooks}`,
    `- 详情采集：${detailOk} / ${detailPlanned}（每频道前 ${TOP}，上限 ${DETAIL_LIMIT}）`,
    `- 数据质量：${quality}`,
    "",
    "---",
    "",
  ];

  for (const ch of data.channels) {
    try {
      lines.push(`## ${ch.name} — ${ch.books.length} 本`, "");
      for (let i = 0; i < ch.books.length; i++) {
        try {
          const b = ch.books[i];
          lines.push(`### #${i + 1} ${b.title}`);
          const d = b.novelid ? detailMap[b.novelid] : null;
          const seg = [b.author || ""];
          if (d) {
            if (d.collect) seg.push("收藏 " + fmtWan(d.collect));
            if (d.nutrition) seg.push("营养液 " + fmtWan(d.nutrition));
            if (d.score) seg.push("积分 " + d.score);
            if (d.words) seg.push("字数 " + fmtWan(d.words, "字"));
            if (d.status) seg.push(d.status);
          }
          const meta = seg.filter(Boolean).join(" · ");
          if (meta) lines.push(`*${meta}*`);
          if (b.novelid) lines.push(`[作品页](https://www.jjwxc.net/onebook.php?novelid=${b.novelid})`);
          lines.push("");
        } catch (bookErr) {
          console.error(`[jjwxc] ${rt.label} ${ch.name} 第${i + 1}条处理出错: ${bookErr.message}`);
          lines.push("");
        }
      }
      lines.push("---", "");
    } catch (chErr) {
      console.error(`[jjwxc] ${rt.label} 频道「${ch.name}」处理出错，跳过: ${chErr.message}`);
    }
  }

  return lines.join("\n");
}

function main() {
  const rankTypes = RANKTYPE === "all" ? RANK_TYPES.map((r) => r.id) : [RANKTYPE];
  const channels = [CHANNEL]; // 晋江频道 ID 需从页面获取，默认全站

  for (const rt of rankTypes) {
    for (const ch of channels) {
      const content = scrapeRank(PORT, rt, ch);
      if (!content) continue;

      const rtInfo = RANK_TYPES.find((r) => r.id === rt);
      const date = new Date().toISOString().slice(0, 10).replace(/-/g, "");
      const chLabel = ch === "0" ? "全站" : `频道${ch}`;
      const filename = `晋江${rtInfo.label}_${chLabel}_${date}.md`;
      fs.mkdirSync(OUTDIR, { recursive: true });
      const filepath = path.join(OUTDIR, filename);
      fs.writeFileSync(filepath, content, "utf-8");
      console.log(`  ✓ 已保存: ${filepath}`);
    }
  }
}

if (require.main === module) {
  try {
    main();
  } catch (e) {
    console.error(`晋江采集失败: ${e && e.message ? e.message : e}`);
    process.exit(1);
  }
}

module.exports = { buildDetailJS, fmtWan };

#!/usr/bin/env node
/**
 * 起点中文网 排行榜采集脚本
 *
 * 配合 browser-cdp skill 使用。先启动 Chrome CDP 环境，再运行本脚本。
 * 采集策略：
 *   1. 默认优先读取 m.qidian.com 的 SSR pageContext JSON（不依赖 CDP，规避 PC 站风控页）。
 *   2. 移动端不可用时再回退到 Chrome CDP 采集 PC 页面。
 * 输出 Markdown 格式匹配 scan-output-format.md 规范。
 *
 * 用法：
 *   node qidian-rank-scraper.js --type hotsales               # 畅销榜
 *   node qidian-rank-scraper.js --type yuepiao                 # 月票榜
 *   node qidian-rank-scraper.js --type signnewbook             # 签约作者新书榜
 *   node qidian-rank-scraper.js --type pubnewbook              # 公众作者新书榜
 *   node qidian-rank-scraper.js --type newauthor               # 新人作者新书榜
 *   node qidian-rank-scraper.js --type newsign                 # 新人签约新书榜
 *   node qidian-rank-scraper.js --type recom                   # 原创推荐榜
 *   node qidian-rank-scraper.js --type sanjiang                 # 三江推荐（/sanjiang/，非 /rank/ 路径）
 *   node qidian-rank-scraper.js --type all                     # 全部榜单
 *   node qidian-rank-scraper.js --type hotsales --mode mobile  # 仅使用移动端 SSR
 *   node qidian-rank-scraper.js --type hotsales --mode cdp     # 仅使用旧版 CDP/PC 页面
 *
 * 前置：
 *   默认 mobile/auto 模式不需要 Chrome。
 *   cdp 模式需要：node {SKILL_DIR}/browser-cdp/scripts/setup-cdp-chrome.js 9222
 */

const fs = require("fs");
const https = require("https");
const path = require("path");
const { ab, sleep, evalJSON, scrollLoad, getArg } = require("./cdp-utils");

const PC_BASE_URL = "https://www.qidian.com/rank";
const MOBILE_BASE_URL = "https://m.qidian.com";

/** 验证码自动重试最大次数 */
const MAX_CAPTCHA_RETRIES = 3;
/** 等待用户手动解决验证码的最大秒数 */
const MAX_CAPTCHA_WAIT_SEC = 120;
/** 轮询验证码是否解除的间隔（毫秒） */
const CAPTCHA_POLL_INTERVAL = 5000;

const MOBILE_HEADERS = {
  "User-Agent":
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) " +
    "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",
  Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
  "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
  "Accept-Encoding": "identity",
};

const RANK_TYPES = [
  { id: "hotsales", label: "畅销榜", mobilePath: "/rank/hotsales/" },
  { id: "yuepiao", label: "月票榜", mobilePath: "/rank/yuepiao/" },
  {
    id: "signnewbook",
    label: "签约作者新书榜",
    mobilePath: "/rank/sign/",
    mobileLabel: "签约榜",
  },
  {
    id: "pubnewbook",
    label: "公众作者新书榜",
    mobilePath: "/rank/newbook/",
    mobileLabel: "新书榜",
  },
  { id: "newauthor", label: "新人作者新书榜", mobilePath: "/rank/newauthor/", mobileLabel: "新人榜" },
  {
    id: "newsign",
    label: "新人签约新书榜",
    mobilePath: "/rank/sign/",
    mobileLabel: "签约榜",
  },
  { id: "recom", label: "原创推荐榜", mobilePath: "/rank/rec/", mobileLabel: "推荐榜" },
  { id: "readindex", label: "阅读指数榜", mobilePath: "/rank/readindex/" },
  {
    id: "collect",
    label: "收藏榜",
    mobilePath: "/rank/newfans/",
    mobileLabel: "书友榜（移动端替代）",
  },
  {
    id: "sanjiang",
    label: "三江推荐",
    baseUrl: "https://www.qidian.com/sanjiang/",
    mobilePath: "/sanjiang/",
  },
];

// ---------------------------------------------------------------------------
// 页面提取
// ---------------------------------------------------------------------------

/**
 * 提取起点 SSR 榜单页面的书籍列表。
 * 起点页面结构：.book-img-text ul > li，每个 li 内：
 *   h2 > a          → 书名+链接
 *   p.author         → 作者 | 题材 · 子题材 | 状态
 *   p.intro          → 简介
 *   p.update > a+span → 最新更新章节+日期
 */
function extractBookList(port) {
  const js =
    "JSON.stringify((()=>{" +
    "var items=[];" +
    "var lis=document.querySelectorAll('.book-img-text ul li');" +
    "if(!lis.length){" +
    // 兜底：用 H2 链接定位
    "  var h2s=document.querySelectorAll('h2 a[href*=\"/book/\"]');" +
    "  h2s.forEach(function(a,idx){" +
    "    var c=a.parentElement;" +
    "    for(var j=0;j<3;j++){if(c.parentElement)c=c.parentElement}" +
    "    var text=c.innerText||'';" +
    "    var href=a.getAttribute('href')||a.href||'';" +
    "    var url=href?(href.indexOf('http')===0?href:'https:'+href):'';" +
    "    items.push({rank:idx+1,title:a.textContent.trim(),url:url,author:'',genre:'',status:'',descText:'',updateText:text.replace(/\\s+/g,' ').trim().substring(0,300)})" +
    "  });" +
    "  return items" +
    "}" +
    "lis.forEach(function(li,idx){" +
    "  var titleEl=li.querySelector('h2 a');" +
    "  if(!titleEl)return;" +
    "  var title=titleEl.textContent.trim();" +
    "  var href=titleEl.getAttribute('href')||titleEl.href||'';" +
    "  var url=href?(href.indexOf('http')===0?href:'https:'+href):'';" +
    // 作者：p.author > a.name
    "  var authorEl=li.querySelector('p.author a.name');" +
    "  var author=authorEl?authorEl.textContent.trim():'';" +
    // 题材：p.author > a (非 .name 非 .go-sub-type)
    "  var genreEls=li.querySelectorAll('p.author a');" +
    "  var genre='';var subGenre='';" +
    "  genreEls.forEach(function(a){" +
    "    if(a.classList.contains('name'))return;" +
    "    if(!genre){genre=a.textContent.trim()}else if(!subGenre){subGenre=a.textContent.trim()}" +
    "  });" +
    // 状态：p.author > span:last-child
    "  var statusEl=li.querySelector('p.author span');" +
    "  var status=statusEl?statusEl.textContent.trim():'';" +
    // 简介：p.intro
    "  var introEl=li.querySelector('p.intro');" +
    "  var descText=introEl?introEl.textContent.trim():'';" +
    // 更新：p.update
    "  var updateEl=li.querySelector('p.update');" +
    "  var updateText=updateEl?updateEl.textContent.replace(/\\s+/g,' ').trim():'';" +
    "  if(title){" +
    "    items.push({rank:idx+1,title:title,url:url,author:author,genre:genre+(subGenre?'·'+subGenre:''),status:status,descText:descText,updateText:updateText})" +
    "  }" +
    "});" +
    "return items" +
    "})())";
  return evalJSON(port, js) || [];
}

/** 从详情页提取标签和简介 */
function extractDetail(port) {
  const js =
    "JSON.stringify((()=>{" +
    "var tags=Array.from(document.querySelectorAll('[class*=\"tag\"] a,[class*=\"label\"] a')).map(function(a){return a.textContent.trim()});" +
    "var intro=document.querySelector('[class*=\"intro\"],[class*=\"summary\"],[class*=\"desc\"]');" +
    "var introText=intro?intro.textContent.trim():'';" +
    "var update=document.querySelector('[class*=\"update\"],[class*=\"latest\"]');" +
    "var updateText=update?update.textContent.trim():'';" +
    "return {tags:tags,intro:introText,update:updateText}" +
    "})())";
  return evalJSON(port, js);
}

/**
 * 检测当前页面是否被验证码/安全验证拦截。
 * 起点常见拦截页面特征：页面中出现验证码关键词，或页面缺少榜单 DOM 元素。
 * @returns {{ blocked: boolean, reason: string } | null} 若被拦截返回原因对象，否则 null
 */
function isCaptchaPage(port) {
  const js =
    "JSON.stringify((()=>{" +
    "var bodyText=document.body?(document.body.innerText||'').substring(0,3000):'';" +
    "var lower=bodyText.toLowerCase();" +
    "var keywords=['验证','captcha','verify','安全验证','滑块','拖动','请完成验证'," +
    "'混元','人机验证','异常请求','访问验证','操作频繁','请求过于频繁','waf','请稍后再试'];" +
    "for(var i=0;i<keywords.length;i++){" +
    "  if(lower.indexOf(keywords[i])>-1){" +
    "    return {blocked:true,reason:keywords[i]};" +
    "  }" +
    "}" +
    "var hasContent=document.querySelector('.book-img-text ul li,.rank-body,.rank-list,.book-img-text');" +
    "if(!hasContent){" +
    "  return {blocked:true,reason:'页面无榜单内容(可能被拦截)'};" +
    "}" +
    "return {blocked:false,reason:''};" +
    "})())";
  const result = evalJSON(port, js);
  return result && result.blocked === true ? result : null;
}

/**
 * 打开 URL 并等待页面加载，自动处理验证码拦截。
 * 重试策略：
 *   1. 正常加载页面
 *   2. 检测到验证码 → 等待递增延时后刷新重试（最多 MAX_CAPTCHA_RETRIES 次）
 *   3. 仍被拦截 → 提示用户在 Chrome CDP 窗口手动完成验证，轮询等待直到解除或超时
 *
 * @returns {boolean} true=页面已就绪，false=无法通过验证码
 */
function openWithCaptchaHandling(port, url) {
  for (let attempt = 1; attempt <= MAX_CAPTCHA_RETRIES; attempt++) {
    ab(port, "open", url);
    // 首次 3 秒，后续每次多等 2 秒
    sleep(3000 + (attempt - 1) * 2000);

    const captcha = isCaptchaPage(port);
    if (!captcha) {
      return true;
    }
    console.log(`  ⚠ 检测到安全拦截 (${captcha.reason})，第 ${attempt}/${MAX_CAPTCHA_RETRIES} 次重试...`);
    // 递增等待后再次尝试
    sleep(attempt * 5000);
  }

  // 自动重试全部失败 → 等待用户手动处理
  console.log(`  ⚠ 自动重试未通过验证码，请在 Chrome CDP 窗口手动完成验证`);
  console.log(`  ⏳ 等待手动验证（最长 ${MAX_CAPTCHA_WAIT_SEC} 秒）...`);

  const startTime = Date.now();
  while (Date.now() - startTime < MAX_CAPTCHA_WAIT_SEC * 1000) {
    sleep(CAPTCHA_POLL_INTERVAL);
    // 刷新页面检查验证码是否已解除
    ab(port, "open", url);
    sleep(3000);
    const captcha = isCaptchaPage(port);
    if (!captcha) {
      console.log(`  ✓ 验证码已解除，继续采集`);
      return true;
    }
    const elapsed = Math.round((Date.now() - startTime) / 1000);
    process.stdout.write(`  等待中... (${elapsed}s)\r`);
  }
  console.log(`  ✗ 等待超时，验证码仍未解除`);
  return false;
}

// ---------------------------------------------------------------------------
// 移动端 SSR 提取（默认路径）
// ---------------------------------------------------------------------------

function mobileUrl(pathname) {
  if (!pathname) return "";
  return pathname.startsWith("http") ? pathname : `${MOBILE_BASE_URL}${pathname}`;
}

function fetchText(url, redirects = 3) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, { headers: MOBILE_HEADERS, timeout: 15000 }, (res) => {
      if (
        redirects > 0 &&
        res.statusCode >= 300 &&
        res.statusCode < 400 &&
        res.headers.location
      ) {
        res.resume();
        const nextUrl = new URL(res.headers.location, url).toString();
        fetchText(nextUrl, redirects - 1).then(resolve, reject);
        return;
      }

      let body = "";
      res.setEncoding("utf8");
      res.on("data", (chunk) => {
        body += chunk;
      });
      res.on("end", () => {
        if (res.statusCode < 200 || res.statusCode >= 300) {
          reject(new Error(`HTTP ${res.statusCode}`));
          return;
        }
        resolve(body);
      });
    });

    req.on("timeout", () => {
      req.destroy(new Error("request timeout"));
    });
    req.on("error", reject);
  });
}

function extractMobilePageContext(html) {
  const m = html.match(
    /<script[^>]+id=["']vite-plugin-ssr_pageContext["'][^>]*>([\s\S]*?)<\/script>/i
  );
  if (!m) return null;
  try {
    return JSON.parse(m[1]);
  } catch (e) {
    console.log(`  ⚠ 移动端 pageContext JSON 解析失败: ${e.message}`);
    return null;
  }
}

function normalizeMobileBook(record, idx) {
  const title = record.bName || record.bookName || "";
  const bid = record.bid || record.bookId || "";
  const genre = [record.cat, record.subCat].filter(Boolean).join("·");
  const stats = [];
  if (record.cnt) stats.push(record.cnt);
  if (record.rankCnt) stats.push(`榜单值 ${record.rankCnt}`);

  return {
    rank: record.rankNum || idx + 1,
    title,
    url: bid ? `${MOBILE_BASE_URL}/book/${bid}/` : "",
    author: record.bAuth || record.author || "",
    genre,
    status: stats.join(" · "),
    descText: record.desc || "",
    updateText: "",
  };
}

function renderMarkdown(rt, books, url, sourceMode, extraLines = []) {
  const now = new Date().toISOString();
  const lines = [
    `# 起点 · ${rt.label}`,
    "",
    `- 来源：${url}`,
    `- 抓取方式：${sourceMode}`,
    `- 抓取时间：${now}`,
    `- 条目数：${books.length}`,
    ...extraLines,
    "",
    "---",
    "",
  ];

  for (let i = 0; i < books.length; i++) {
    const b = books[i];
    lines.push(`## #${b.rank || i + 1} ${b.title}`);
    const meta = [b.author, b.genre, b.status].filter(Boolean).join(" · ");
    if (meta) lines.push(`*${meta}*`);
    if (b.updateText) lines.push(`**最新更新：** ${b.updateText}`);
    if (b.tags?.length) lines.push(`**标签：** ${b.tags.join("、")}`);
    if (b.url) lines.push(`[作品页](${b.url})`);
    if (b.descText) {
      lines.push("");
      lines.push("**简介**");
      lines.push("");
      lines.push(b.descText);
    }
    lines.push("", "---", "");
  }

  return lines.join("\n");
}

async function scrapeRankMobile(rankTypeId) {
  const rt = RANK_TYPES.find((r) => r.id === rankTypeId);
  if (!rt) {
    console.log(`  ⚠ 未知榜单类型: ${rankTypeId}`);
    return null;
  }
  if (!rt.mobilePath) {
    console.log(`  ⚠ 榜单 ${rankTypeId} 暂无移动端 SSR 路径`);
    return null;
  }

  const url = mobileUrl(rt.mobilePath);
  console.log(`\n→ 采集 起点${rt.label}（移动端 SSR）...`);
  console.log(`  URL: ${url}`);

  const html = await fetchText(url);
  const pageContext = extractMobilePageContext(html);
  const pageData = pageContext?.pageContext?.pageProps?.pageData;
  const records = pageData?.records || [];
  const books = records.map(normalizeMobileBook).filter((b) => b.title);

  if (!books.length) {
    console.log("  ⚠ 移动端 SSR 未提取到书籍");
    return null;
  }

  console.log(`  ✓ 提取 ${books.length} 本`);

  const extraLines = [];
  if (rt.mobileLabel && rt.mobileLabel !== rt.label) {
    extraLines.push(`- 移动端实际榜单：${rt.mobileLabel}`);
  }
  if (FETCH_DETAIL) {
    extraLines.push("- 说明：移动端 SSR 已包含简介；--detail 在 mobile/auto 模式下不会额外打开详情页。");
  }

  return renderMarkdown(rt, books, url, "mobile-ssr", extraLines);
}

// ---------------------------------------------------------------------------
// 主流程
// ---------------------------------------------------------------------------

const args = process.argv.slice(2);
const PORT = parseInt(getArg(args, "--port") || "9222", 10);
const OUTDIR = getArg(args, "--outdir") || ".";
const RANKTYPE = getArg(args, "--type") || "hotsales";
const SCRAPE_MODE = getArg(args, "--mode") || "auto"; // auto | mobile | cdp
const FETCH_DETAIL = (getArg(args, "--detail") || "no") === "yes";

function scrapeRankCDP(port, rankTypeId) {
  const rt = RANK_TYPES.find((r) => r.id === rankTypeId);
  if (!rt) {
    console.log(`  ⚠ 未知榜单类型: ${rankTypeId}`);
    return null;
  }

  const url = rt.baseUrl || `${PC_BASE_URL}/${rankTypeId}/`;
  console.log(`\n→ 采集 起点${rt.label}（CDP/PC）...`);
  console.log(`  URL: ${url}`);

  const pageReady = openWithCaptchaHandling(port, url);
  if (!pageReady) {
    console.log("  ✗ 起点采集失败：页面无法通过验证码拦截");
    return null;
  }

  scrollLoad(port, 3);
  sleep(1000);

  const books = extractBookList(port);
  if (!books.length) {
    console.log("  ⚠ 未提取到书籍");
    return null;
  }
  console.log(`  ✓ 提取 ${books.length} 本`);

  // 可选：逐条获取详情页补充数据
  if (FETCH_DETAIL) {
    console.log("  正在获取详情页补充数据...");
    for (let i = 0; i < Math.min(books.length, 20); i++) {
      const b = books[i];
      if (!b.url) continue;
      ab(port, "open", b.url);
      sleep(1500);
      const detail = extractDetail(port);
      if (detail) {
        if (detail.tags?.length) b.tags = detail.tags;
        if (detail.intro) b.descText = detail.intro;
        if (detail.update) b.updateText = detail.update;
      }
      console.log(`    [${i + 1}/${books.length}] ${b.title}`);
    }
    // 返回榜单页
    ab(port, "open", url);
    sleep(2000);
  }

  return renderMarkdown(rt, books, url, "cdp-pc");
}

async function scrapeRank(rankTypeId) {
  if (!["auto", "mobile", "cdp"].includes(SCRAPE_MODE)) {
    throw new Error(`未知 --mode: ${SCRAPE_MODE}（可选 auto/mobile/cdp）`);
  }

  if (SCRAPE_MODE !== "cdp") {
    try {
      const content = await scrapeRankMobile(rankTypeId);
      if (content || SCRAPE_MODE === "mobile") return content;
    } catch (e) {
      console.log(`  ⚠ 移动端 SSR 采集失败: ${e.message}`);
      if (SCRAPE_MODE === "mobile") return null;
    }
  }

  if (SCRAPE_MODE !== "mobile") {
    console.log("  → 回退到 CDP/PC 页面采集");
    return scrapeRankCDP(PORT, rankTypeId);
  }

  return null;
}

async function main() {
  const rankTypes = RANKTYPE === "all" ? RANK_TYPES.map((r) => r.id) : [RANKTYPE];

  for (const rt of rankTypes) {
    const content = await scrapeRank(rt);
    if (!content) continue;

    const rtInfo = RANK_TYPES.find((r) => r.id === rt);
    const date = new Date().toISOString().slice(0, 10).replace(/-/g, "");
    const filename = `起点${rtInfo.label}_${date}.md`;
    fs.mkdirSync(OUTDIR, { recursive: true });
    const filepath = path.join(OUTDIR, filename);
    fs.writeFileSync(filepath, content, "utf-8");
    console.log(`  ✓ 已保存: ${filepath}`);
  }
}

main().catch((e) => {
  console.error(`起点采集失败: ${e.message}`);
  process.exit(1);
});