Web To Markdown

interface:
  display_name: "Web to Markdown"
  short_description: "Convert URLs to clean Markdown via smart routing."
  default_prompt: "Convert the provided URL into clean markdown using web-to-markdown routing rules."

{
  "name": "web-to-markdown-skill",
  "version": "1.0.0",
  "private": true,
  "type": "module",
  "description": "Routing and extraction scripts for web-to-markdown skill.",
  "scripts": {
    "convert": "node scripts/url_to_markdown.mjs",
    "convert:special": "node scripts/fetch_special_sites.mjs",
    "convert:special:http": "node scripts/fetch_special_sites_http.mjs",
    "convert:generic": "node scripts/fetch_generic_fallback.mjs"
  },
  "dependencies": {
    "@mozilla/readability": "^0.5.0",
    "cuimp": "^1.10.0",
    "jsdom": "^24.1.3",
    "puppeteer": "^24.6.1",
    "puppeteer-extra": "^3.3.6",
    "puppeteer-extra-plugin-anonymize-ua": "^2.4.6",
    "puppeteer-extra-plugin-stealth": "^2.11.2",
    "turndown": "^7.2.0"
  }
}

#!/usr/bin/env node

import fs from 'node:fs';
import process from 'node:process';
import { Readability } from '@mozilla/readability';
import { JSDOM } from 'jsdom';
import TurndownService from 'turndown';
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import AnonymizeUAPlugin from 'puppeteer-extra-plugin-anonymize-ua';

const CHROME_UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36';

puppeteer.use(StealthPlugin());
puppeteer.use(
  AnonymizeUAPlugin({
    customFn: () => CHROME_UA,
    stripHeadless: true
  })
);

function resolveChromeExecutablePath() {
  const candidates = [
    process.env.CHROME_PATH,
    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
    '/Applications/Chromium.app/Contents/MacOS/Chromium',
    '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'
  ].filter(Boolean);

  for (const candidate of candidates) {
    if (fs.existsSync(candidate)) {
      return candidate;
    }
  }

  return undefined;
}

function fallbackExtract(doc) {
  const selectors = [
    'article',
    'main',
    '[role="main"]',
    '[class*="article"]',
    '[class*="content"]',
    '[class*="post"]',
    '[class*="entry"]'
  ];

  let bestNode = null;
  let bestScore = 0;

  for (const selector of selectors) {
    const nodes = doc.querySelectorAll(selector);
    for (const node of nodes) {
      const text = (node.textContent || '').replace(/\s+/g, ' ').trim();
      const score = text.length;
      if (score > bestScore) {
        bestNode = node;
        bestScore = score;
      }
    }
  }

  if (!bestNode || bestScore < 180) {
    return null;
  }

  const title =
    doc.querySelector('h1')?.textContent?.trim() ||
    doc.querySelector('title')?.textContent?.trim() ||
    '';

  return {
    title,
    byline: '',
    excerpt: (bestNode.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 180),
    content: bestNode.innerHTML
  };
}

function sanitizeHtml(inputHtml, pageUrl) {
  const dom = new JSDOM(inputHtml, { url: pageUrl });
  const doc = dom.window.document;

  const noiseSelectors = [
    'script',
    'style',
    'noscript',
    'iframe',
    'header',
    'footer',
    'nav',
    '.advertisement',
    '.ad',
    '[class*="ad-"]',
    '[id*="ad-"]'
  ];

  for (const selector of noiseSelectors) {
    doc.querySelectorAll(selector).forEach((node) => node.remove());
  }

  const reader = new Readability(doc, {
    keepClasses: false,
    charThreshold: 160,
    nbTopCandidates: 10
  });

  const parsed = reader.parse();
  if (parsed && parsed.content) {
    return parsed;
  }

  const fallback = fallbackExtract(doc);
  if (fallback) {
    return fallback;
  }

  throw new Error('Failed to extract readable content from HTML');
}

function htmlToMarkdown(title, byline, excerpt, htmlContent) {
  const turndown = new TurndownService({
    headingStyle: 'atx',
    codeBlockStyle: 'fenced',
    bulletListMarker: '-',
    emDelimiter: '_'
  });

  const bodyMarkdown = turndown.turndown(htmlContent).trim();
  const lines = [];

  if (title) {
    lines.push(`# ${title.trim()}`);
    lines.push('');
  }

  if (byline) {
    lines.push(`> Author: ${byline.trim()}`);
    lines.push('');
  }

  if (excerpt) {
    lines.push(`> ${excerpt.trim()}`);
    lines.push('');
  }

  lines.push(bodyMarkdown);
  return lines.join('\n').trim();
}

function isLikelyShell(markdown) {
  const text = (markdown || '').replace(/\s+/g, ' ').trim().toLowerCase();
  if (text.length < 250) {
    return true;
  }

  const signals = [
    'enable javascript',
    'access denied',
    'robot check',
    'please verify you are human',
    'continue to read',
    'log in to continue'
  ];

  return signals.some((signal) => text.includes(signal));
}

async function fetchHtmlDirect(url, timeoutMs) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);

  try {
    const response = await fetch(url, {
      method: 'GET',
      headers: {
        'user-agent': CHROME_UA,
        accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8'
      },
      redirect: 'follow',
      signal: controller.signal
    });

    const text = await response.text();
    if (!response.ok) {
      throw new Error(`Direct fetch failed (${response.status} ${response.statusText})`);
    }
    if (!text.trim()) {
      throw new Error('Direct fetch returned empty html');
    }

    return {
      resolvedUrl: response.url || url,
      html: text
    };
  } catch (error) {
    if (error instanceof Error && error.name === 'AbortError') {
      throw new Error(`Direct fetch timed out after ${timeoutMs}ms`);
    }
    throw error;
  } finally {
    clearTimeout(timer);
  }
}

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve) => {
      let total = 0;
      const distance = 900;
      const timer = setInterval(() => {
        const max = Math.max(
          document.body?.scrollHeight || 0,
          document.documentElement?.scrollHeight || 0
        );
        window.scrollBy(0, distance);
        total += distance;
        if (total >= max) {
          clearInterval(timer);
          resolve();
        }
      }, 120);
    });
  });
}

async function fetchByBrowser(url, timeoutMs) {
  const browser = await puppeteer.launch({
    headless: true,
    executablePath: resolveChromeExecutablePath(),
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage',
      '--disable-blink-features=AutomationControlled'
    ]
  });

  try {
    const page = await browser.newPage();
    await page.setUserAgent(CHROME_UA);
    await page.setViewport({ width: 1440, height: 2200, deviceScaleFactor: 2 });
    await page.setExtraHTTPHeaders({
      'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8'
    });

    await page.goto(url, {
      waitUntil: 'domcontentloaded',
      timeout: timeoutMs
    });

    await sleep(900);
    await autoScroll(page);
    await sleep(500);

    const html = await page.content();
    return {
      resolvedUrl: page.url(),
      html
    };
  } finally {
    await browser.close();
  }
}

export async function fetchGenericToMarkdown(rawUrl, options = {}) {
  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 30000;
  const normalized = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;

  let directError = null;
  try {
    const directResult = await fetchHtmlDirect(normalized, timeoutMs);
    const readable = sanitizeHtml(directResult.html, directResult.resolvedUrl);
    const markdown = htmlToMarkdown(
      readable.title,
      readable.byline,
      readable.excerpt,
      readable.content
    );

    if (isLikelyShell(markdown)) {
      throw new Error('Direct fetch returned likely shell/teaser content');
    }

    return {
      source: 'direct-readability',
      strategy: 'generic-http-fallback',
      requestedUrl: rawUrl,
      resolvedUrl: directResult.resolvedUrl,
      title: readable.title || '',
      markdown
    };
  } catch (error) {
    directError = error instanceof Error ? error.message : String(error);
  }

  const browserResult = await fetchByBrowser(normalized, timeoutMs);
  const readable = sanitizeHtml(browserResult.html, browserResult.resolvedUrl);
  const markdown = htmlToMarkdown(
    readable.title,
    readable.byline,
    readable.excerpt,
    readable.content
  );

  if (isLikelyShell(markdown)) {
    throw new Error(`Generic browser fallback returned likely shell content. HTTP error: ${directError || ''}`);
  }

  return {
    source: 'browser-readability',
    strategy: 'generic-browser-fallback',
    requestedUrl: rawUrl,
    resolvedUrl: browserResult.resolvedUrl,
    title: readable.title || '',
    markdown,
    fallbackReason: directError || ''
  };
}

function parseArgs(argv) {
  const args = {
    url: '',
    timeoutMs: 30000,
    json: false
  };

  for (let i = 2; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token) continue;

    if (token === '--json') {
      args.json = true;
      continue;
    }

    if (token === '--timeout-ms') {
      const value = Number(argv[i + 1]);
      if (!Number.isFinite(value) || value <= 0) {
        throw new Error('--timeout-ms must be a positive number');
      }
      args.timeoutMs = value;
      i += 1;
      continue;
    }

    if (!args.url) {
      args.url = token;
      continue;
    }

    throw new Error(`Unexpected argument: ${token}`);
  }

  if (!args.url) {
    throw new Error('Usage: node scripts/fetch_generic_fallback.mjs <url> [--timeout-ms 30000] [--json]');
  }

  return args;
}

async function main() {
  try {
    const args = parseArgs(process.argv);
    const result = await fetchGenericToMarkdown(args.url, {
      timeoutMs: args.timeoutMs
    });

    if (args.json) {
      process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
      return;
    }

    process.stdout.write(`${result.markdown}\n`);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    process.stderr.write(`fetch_generic_fallback failed: ${message}\n`);
    process.exitCode = 1;
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await main();
}

#!/usr/bin/env node

import process from 'node:process';
import { createCuimpHttp } from 'cuimp';

const SPECIAL_HOSTS = [
  'mp.weixin.qq.com',
  'weixin.qq.com',
  'zhuanlan.zhihu.com',
  'www.zhihu.com',
  'zhihu.com',
  'feishu.cn',
  'www.feishu.cn',
  'larkoffice.com',
  'www.larkoffice.com'
];

const CHROME_UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36';

const cuimpClient = createCuimpHttp({
  descriptor: { browser: 'chrome', version: '136' },
  autoDownload: true,
  logger: {
    info: () => {},
    warn: () => {},
    error: () => {},
    debug: () => {}
  }
});

function isSpecialHost(hostname) {
  return SPECIAL_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`));
}

function normalizeUrl(url) {
  if (!url || typeof url !== 'string') {
    throw new Error('URL is required');
  }

  const candidate = /^https?:\/\//i.test(url.trim()) ? url.trim() : `https://${url.trim()}`;
  const parsed = new URL(candidate);
  if (!isSpecialHost(parsed.hostname)) {
    throw new Error(`Unsupported host for special HTTP fetch: ${parsed.hostname}`);
  }

  return parsed.toString();
}

function parseArgs(argv) {
  const args = {
    url: '',
    timeoutMs: 30000,
    json: false
  };

  for (let i = 2; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token) continue;

    if (token === '--json') {
      args.json = true;
      continue;
    }

    if (token === '--timeout-ms') {
      const value = Number(argv[i + 1]);
      if (!Number.isFinite(value) || value <= 0) {
        throw new Error('--timeout-ms must be a positive number');
      }
      args.timeoutMs = value;
      i += 1;
      continue;
    }

    if (!args.url) {
      args.url = token;
      continue;
    }

    throw new Error(`Unexpected argument: ${token}`);
  }

  if (!args.url) {
    throw new Error('Usage: node scripts/fetch_special_sites_http.mjs <url> [--timeout-ms 30000] [--json]');
  }

  return args;
}

function buildHeaders(url) {
  const parsed = new URL(url);
  const origin = `${parsed.protocol}//${parsed.host}`;
  return {
    'user-agent': CHROME_UA,
    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'cache-control': 'no-cache',
    pragma: 'no-cache',
    'upgrade-insecure-requests': '1',
    referer: origin
  };
}

export async function fetchSpecialSiteHtml(rawUrl, options = {}) {
  const url = normalizeUrl(rawUrl);
  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 30000;

  const response = await cuimpClient.request({
    url,
    method: 'GET',
    timeout: timeoutMs,
    maxRedirects: 20,
    headers: buildHeaders(url)
  });

  const resolvedUrl = response.request?.url || url;
  const statusCode = Number(response.status || 0);
  const contentType = response.headers?.['content-type'] || response.headers?.['Content-Type'] || '';
  const html =
    typeof response.data === 'string'
      ? response.data
      : response.rawBody
        ? response.rawBody.toString('utf-8')
        : '';

  return {
    source: 'cuimp',
    strategy: 'special-http-fetch',
    requestedUrl: rawUrl,
    resolvedUrl,
    statusCode,
    contentType,
    html
  };
}

async function main() {
  try {
    const args = parseArgs(process.argv);
    const result = await fetchSpecialSiteHtml(args.url, {
      timeoutMs: args.timeoutMs
    });

    if (args.json) {
      process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
      return;
    }

    process.stdout.write(`${result.html}\n`);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    process.stderr.write(`fetch_special_sites_http failed: ${message}\n`);
    process.exitCode = 1;
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await main();
}

#!/usr/bin/env node

import process from 'node:process';
import fs from 'node:fs';
import { fetchSpecialSiteHtml } from './fetch_special_sites_http.mjs';
import { Readability } from '@mozilla/readability';
import { JSDOM } from 'jsdom';
import TurndownService from 'turndown';
import puppeteer from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import AnonymizeUAPlugin from 'puppeteer-extra-plugin-anonymize-ua';

const CHROME_UA =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36';

const SPECIAL_HOSTS = [
  'mp.weixin.qq.com',
  'weixin.qq.com',
  'zhuanlan.zhihu.com',
  'www.zhihu.com',
  'zhihu.com',
  'feishu.cn',
  'www.feishu.cn',
  'feishu.cn/docs',
  'larkoffice.com',
  'www.larkoffice.com'
];

puppeteer.use(StealthPlugin());
puppeteer.use(
  AnonymizeUAPlugin({
    customFn: () => CHROME_UA,
    stripHeadless: true
  })
);

function isSpecialHost(hostname) {
  return SPECIAL_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`));
}

function normalizeUrl(url) {
  if (!url || typeof url !== 'string') {
    throw new Error('URL is required');
  }

  const candidate = /^https?:\/\//i.test(url.trim()) ? url.trim() : `https://${url.trim()}`;
  const parsed = new URL(candidate);
  if (!isSpecialHost(parsed.hostname)) {
    throw new Error(`Unsupported host for special fetch script: ${parsed.hostname}`);
  }

  return parsed.toString();
}

function parseArgs(argv) {
  const args = {
    url: '',
    timeoutMs: 30000,
    json: false
  };

  for (let i = 2; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token) continue;

    if (token === '--json') {
      args.json = true;
      continue;
    }

    if (token === '--timeout-ms') {
      const value = Number(argv[i + 1]);
      if (!Number.isFinite(value) || value <= 0) {
        throw new Error('--timeout-ms must be a positive number');
      }
      args.timeoutMs = value;
      i += 1;
      continue;
    }

    if (!args.url) {
      args.url = token;
      continue;
    }

    throw new Error(`Unexpected argument: ${token}`);
  }

  if (!args.url) {
    throw new Error('Usage: node scripts/fetch_special_sites.mjs <url> [--timeout-ms 30000] [--json]');
  }

  return args;
}

async function dismissKnownOverlays(page) {
  const selectors = [
    // Zhihu login modal and overlays.
    '.signFlowModal',
    '.Modal-wrapper',
    '.Modal-backdrop',
    '.signFlowDialog',
    '.css-1ynzxqw',
    // Wechat QR prompts / overlays (best effort).
    '.js_wechat_qrcode',
    '.wx_tips',
    '.rich_media_global_msg',
    // Feishu modals.
    '[data-testid="modal-mask"]',
    '[class*="modal"]'
  ];

  await page.evaluate((overlaySelectors) => {
    for (const selector of overlaySelectors) {
      document.querySelectorAll(selector).forEach((node) => {
        try {
          node.remove();
        } catch (_) {
          // Ignore detached nodes.
        }
      });
    }

    // Restore page scroll when it is blocked by login walls.
    const html = document.documentElement;
    const body = document.body;
    if (html) {
      html.style.overflow = 'auto';
      html.style.position = 'static';
    }
    if (body) {
      body.style.overflow = 'auto';
      body.style.position = 'static';
      body.style.height = 'auto';
    }
  }, selectors);
}

async function autoScroll(page) {
  await page.evaluate(async () => {
    await new Promise((resolve) => {
      let total = 0;
      const distance = 800;
      const timer = setInterval(() => {
        const max = Math.max(
          document.body?.scrollHeight || 0,
          document.documentElement?.scrollHeight || 0
        );
        window.scrollBy(0, distance);
        total += distance;
        if (total >= max) {
          clearInterval(timer);
          resolve();
        }
      }, 120);
    });
  });
}

function sleep(ms) {
  return new Promise((resolve) => {
    setTimeout(resolve, ms);
  });
}

function resolveChromeExecutablePath() {
  const candidates = [
    process.env.CHROME_PATH,
    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
    '/Applications/Chromium.app/Contents/MacOS/Chromium',
    '/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'
  ].filter(Boolean);

  for (const candidate of candidates) {
    if (fs.existsSync(candidate)) {
      return candidate;
    }
  }

  return undefined;
}

function sanitizeHtml(inputHtml, pageUrl) {
  const dom = new JSDOM(inputHtml, { url: pageUrl });
  const doc = dom.window.document;

  // Remove common noisy nodes before Readability.
  const noiseSelectors = [
    'script',
    'style',
    'noscript',
    'iframe',
    'header',
    'footer',
    'nav',
    '.advertisement',
    '.ad',
    '[class*="ad-"]',
    '[id*="ad-"]',
    '.Modal-wrapper',
    '.Modal-backdrop',
    '.signFlowModal'
  ];

  for (const selector of noiseSelectors) {
    doc.querySelectorAll(selector).forEach((node) => node.remove());
  }

  const reader = new Readability(doc, {
    keepClasses: false,
    charThreshold: 120,
    nbTopCandidates: 10
  });

  const parsed = reader.parse();
  if (parsed && parsed.content) {
    return parsed;
  }

  const fallback = fallbackExtract(doc);
  if (fallback) {
    return fallback;
  }

  throw new Error('Failed to extract readable content with Mozilla Readability');
}

function fallbackExtract(doc) {
  const selectors = [
    'article',
    'main',
    '[role="main"]',
    '.ql-editor',
    '.docx-editor',
    '[class*="doc-content"]',
    '[class*="article-content"]',
    '[class*="Post-RichText"]',
    '[class*="content"]'
  ];

  let bestNode = null;
  let bestScore = 0;

  for (const selector of selectors) {
    const nodes = doc.querySelectorAll(selector);
    for (const node of nodes) {
      const text = (node.textContent || '').replace(/\s+/g, ' ').trim();
      const score = text.length;
      if (score > bestScore) {
        bestNode = node;
        bestScore = score;
      }
    }
  }

  if (!bestNode || bestScore < 120) {
    return null;
  }

  const title =
    doc.querySelector('h1')?.textContent?.trim() ||
    doc.querySelector('title')?.textContent?.trim() ||
    '';
  const excerpt = (bestNode.textContent || '').replace(/\s+/g, ' ').trim().slice(0, 200);

  return {
    title,
    byline: '',
    excerpt,
    content: bestNode.innerHTML
  };
}

function htmlToMarkdown(title, byline, excerpt, htmlContent) {
  const turndown = new TurndownService({
    headingStyle: 'atx',
    codeBlockStyle: 'fenced',
    bulletListMarker: '-',
    emDelimiter: '_'
  });

  turndown.addRule('removeEmptyLinks', {
    filter: 'a',
    replacement(content, node) {
      const href = node.getAttribute('href');
      if (!href) {
        return content;
      }
      const text = content.trim() || href;
      return `[${text}](${href})`;
    }
  });

  const bodyMarkdown = turndown.turndown(htmlContent).trim();
  const lines = [];

  if (title) {
    lines.push(`# ${title.trim()}`);
    lines.push('');
  }

  if (byline) {
    lines.push(`> Author: ${byline.trim()}`);
    lines.push('');
  }

  if (excerpt) {
    lines.push(`> ${excerpt.trim()}`);
    lines.push('');
  }

  lines.push(bodyMarkdown);
  return lines.join('\n').trim();
}

function detectBlockedContent(pageUrl, title, markdown) {
  const hostname = new URL(pageUrl).hostname.toLowerCase();
  const normalizedTitle = (title || '').toLowerCase();
  const normalizedBody = (markdown || '').toLowerCase();

  if (hostname.endsWith('zhihu.com')) {
    const blockedSignals = [
      '你似乎来到了没有知识存在的荒原',
      'go to home',
      '去往首页',
      'page error'
    ];

    const matched = blockedSignals.some(
      (signal) => normalizedTitle.includes(signal.toLowerCase()) || normalizedBody.includes(signal.toLowerCase())
    );

    if (matched) {
      throw new Error('Zhihu returned an error/blocked page instead of article content');
    }
  }

  if (hostname.endsWith('feishu.cn') || hostname.endsWith('larkoffice.com')) {
    if (normalizedTitle.includes('飞书云文档') && normalizedBody.length < 320) {
      throw new Error('Feishu returned shell/limited page instead of full document content');
    }
  }
}

function detectShellContent(pageUrl, title, markdown) {
  const hostname = new URL(pageUrl).hostname.toLowerCase();
  const normalizedTitle = (title || '').toLowerCase().trim();
  const normalizedBody = (markdown || '').toLowerCase();
  const contentLength = normalizedBody.replace(/\s+/g, '').length;

  if (hostname.endsWith('zhihu.com')) {
    const shellSignals = [
      '知乎，让每一次点击都充满意义',
      '欢迎来到知乎',
      '下载知乎app',
      '打开知乎'
    ];
    if (shellSignals.some((signal) => normalizedBody.includes(signal.toLowerCase()))) {
      return 'Zhihu shell page';
    }
    if ((normalizedTitle === '' || normalizedTitle === '知乎') && contentLength < 600) {
      return 'Zhihu content too short';
    }
  }

  if (hostname.endsWith('feishu.cn') || hostname.endsWith('larkoffice.com')) {
    if (normalizedTitle === 'docs' && contentLength < 900) {
      return 'Feishu docs shell page';
    }
    if (normalizedBody.includes('最近修改') && contentLength < 1000) {
      return 'Feishu metadata-only page';
    }
  }

  if (hostname.endsWith('weixin.qq.com')) {
    if (normalizedBody.includes('继续滑动看下一个') && contentLength < 900) {
      return 'WeChat teaser/shell page';
    }
  }

  return '';
}

async function waitForLikelyContent(page, targetUrl) {
  const hostname = new URL(targetUrl).hostname.toLowerCase();
  let selectors = [];

  if (hostname.endsWith('zhihu.com')) {
    selectors = ['article', '.Post-RichTextContainer', '[class*="RichText"]'];
  } else if (hostname.endsWith('feishu.cn') || hostname.endsWith('larkoffice.com')) {
    selectors = ['article', 'main', '.ql-editor', '[class*="doc-content"]'];
  } else if (hostname.endsWith('weixin.qq.com')) {
    selectors = ['#js_content', '.rich_media_content'];
  }

  for (const selector of selectors) {
    try {
      await page.waitForSelector(selector, { timeout: 4500 });
      return;
    } catch {
      // Try next selector.
    }
  }
}

export async function fetchSpecialSiteToMarkdown(rawUrl, options = {}) {
  const url = normalizeUrl(rawUrl);
  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : 30000;

  let httpFetcherError = null;
  try {
    const httpResult = await fetchSpecialSiteHtml(url, { timeoutMs });
    const readable = sanitizeHtml(httpResult.html, httpResult.resolvedUrl || url);
    const markdown = htmlToMarkdown(
      readable.title,
      readable.byline,
      readable.excerpt,
      readable.content
    );
    detectBlockedContent(httpResult.resolvedUrl || url, readable.title, markdown);
    const shellReason = detectShellContent(httpResult.resolvedUrl || url, readable.title, markdown);
    if (shellReason) {
      throw new Error(shellReason);
    }

    return {
      source: 'curl-cffi+readability',
      strategy: 'special-http-fetch',
      requestedUrl: rawUrl,
      resolvedUrl: httpResult.resolvedUrl || url,
      title: readable.title || '',
      byline: readable.byline || '',
      excerpt: readable.excerpt || '',
      markdown
    };
  } catch (error) {
    httpFetcherError = error instanceof Error ? error.message : String(error);
  }

  const browser = await puppeteer.launch({
    headless: true,
    executablePath: resolveChromeExecutablePath(),
    args: [
      '--no-sandbox',
      '--disable-setuid-sandbox',
      '--disable-dev-shm-usage',
      '--disable-blink-features=AutomationControlled'
    ]
  });

  try {
    const page = await browser.newPage();
    await page.setUserAgent(CHROME_UA);
    await page.setViewport({ width: 1440, height: 2200, deviceScaleFactor: 2 });
    await page.setExtraHTTPHeaders({
      'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
      'sec-ch-ua': '"Chromium";v="123", "Google Chrome";v="123", "Not:A-Brand";v="8"',
      'sec-ch-ua-mobile': '?0',
      'sec-ch-ua-platform': '"macOS"'
    });

    await page.goto(url, {
      waitUntil: 'domcontentloaded',
      timeout: timeoutMs
    });

    await waitForLikelyContent(page, url);
    await sleep(1200);
    await dismissKnownOverlays(page);
    await autoScroll(page);
    await sleep(500);

    const html = await page.content();
    const readable = sanitizeHtml(html, url);
    const markdown = htmlToMarkdown(
      readable.title,
      readable.byline,
      readable.excerpt,
      readable.content
    );
    detectBlockedContent(page.url(), readable.title, markdown);
    const shellReason = detectShellContent(page.url(), readable.title, markdown);
    if (shellReason) {
      throw new Error(`Browser fallback still returned shell content: ${shellReason}`);
    }

    return {
      source: 'browser-readability',
      strategy: 'special-browser-fetch-fallback',
      requestedUrl: rawUrl,
      resolvedUrl: page.url(),
      title: readable.title || '',
      byline: readable.byline || '',
      excerpt: readable.excerpt || '',
      markdown,
      fallbackReason: httpFetcherError || ''
    };
  } finally {
    await browser.close();
  }
}

async function main() {
  try {
    const args = parseArgs(process.argv);
    const result = await fetchSpecialSiteToMarkdown(args.url, {
      timeoutMs: args.timeoutMs
    });

    if (args.json) {
      process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
      return;
    }

    process.stdout.write(`${result.markdown}\n`);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    process.stderr.write(`fetch_special_sites failed: ${message}\n`);
    process.exitCode = 1;
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await main();
}

#!/usr/bin/env node

import process from 'node:process';

const R_JINA_BASE = 'https://r.jina.ai/';
const DEFUDDLE_BASE = 'https://defuddle.md/';

const X_HOSTS = ['x.com', 'www.x.com', 'twitter.com', 'www.twitter.com'];
const YOUTUBE_HOSTS = ['youtube.com', 'www.youtube.com', 'm.youtube.com', 'youtu.be'];
const SPECIAL_HOSTS = [
  'mp.weixin.qq.com',
  'weixin.qq.com',
  'zhuanlan.zhihu.com',
  'www.zhihu.com',
  'zhihu.com',
  'feishu.cn',
  'www.feishu.cn',
  'larkoffice.com',
  'www.larkoffice.com'
];

const DEFAULT_TIMEOUT_MS = 30000;
const DEFAULT_USER_AGENT =
  'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36';

function parseArgs(argv) {
  const args = {
    url: '',
    timeoutMs: DEFAULT_TIMEOUT_MS,
    json: false
  };

  for (let i = 2; i < argv.length; i += 1) {
    const token = argv[i];
    if (!token) continue;

    if (token === '--json') {
      args.json = true;
      continue;
    }

    if (token === '--timeout-ms') {
      const value = Number(argv[i + 1]);
      if (!Number.isFinite(value) || value <= 0) {
        throw new Error('--timeout-ms must be a positive number');
      }
      args.timeoutMs = value;
      i += 1;
      continue;
    }

    if (!args.url) {
      args.url = token;
      continue;
    }

    throw new Error(`Unexpected argument: ${token}`);
  }

  if (!args.url) {
    throw new Error('Usage: node scripts/url_to_markdown.mjs <url> [--json] [--timeout-ms 30000]');
  }

  return args;
}

function normalizeUrl(rawUrl) {
  if (!rawUrl || typeof rawUrl !== 'string') {
    throw new Error('URL is required');
  }

  const trimmed = rawUrl.trim();
  const defuddleDirectPrefix = `${DEFUDDLE_BASE}http`;
  if (trimmed.toLowerCase().startsWith(defuddleDirectPrefix)) {
    const embedded = trimmed.slice(DEFUDDLE_BASE.length);
    try {
      return new URL(embedded);
    } catch {
      throw new Error(`Invalid defuddle-embedded URL: ${rawUrl}`);
    }
  }

  const rJinaDirectPrefix = `${R_JINA_BASE}http`;
  if (trimmed.toLowerCase().startsWith(rJinaDirectPrefix)) {
    const embedded = trimmed.slice(R_JINA_BASE.length);
    try {
      return new URL(embedded);
    } catch {
      throw new Error(`Invalid r.jina.ai-embedded URL: ${rawUrl}`);
    }
  }

  const prefixed = /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;

  try {
    return new URL(prefixed);
  } catch {
    throw new Error(`Invalid URL: ${rawUrl}`);
  }
}

function isInHosts(hostname, hostList) {
  return hostList.some((host) => hostname === host || hostname.endsWith(`.${host}`));
}

function pickStrategy(urlObj) {
  const host = urlObj.hostname.toLowerCase();

  if (isInHosts(host, SPECIAL_HOSTS)) {
    return 'special-browser-fetch';
  }

  if (isInHosts(host, YOUTUBE_HOSTS)) {
    return 'defuddle';
  }

  if (isInHosts(host, X_HOSTS)) {
    return 'r-jina';
  }

  return 'r-jina';
}

function buildProxyUrl(base, targetUrl) {
  return `${base}${targetUrl}`;
}

async function fetchMarkdownFromProxy(proxyUrl, timeoutMs) {
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);

  try {
    const response = await fetch(proxyUrl, {
      method: 'GET',
      headers: {
        'user-agent': DEFAULT_USER_AGENT,
        accept: 'text/markdown,text/plain;q=0.9,*/*;q=0.8'
      },
      signal: controller.signal
    });

    if (!response.ok) {
      throw new Error(`Proxy request failed (${response.status} ${response.statusText})`);
    }

    const text = (await response.text()).trim();
    if (!text) {
      throw new Error('Proxy returned empty content');
    }

    return {
      markdown: text,
      contentType: response.headers.get('content-type') || ''
    };
  } catch (error) {
    if (error instanceof Error && error.name === 'AbortError') {
      throw new Error(`Request timed out after ${timeoutMs}ms`);
    }
    throw error;
  } finally {
    clearTimeout(timer);
  }
}

export async function urlToMarkdown(rawUrl, options = {}) {
  const timeoutMs = Number(options.timeoutMs) > 0 ? Number(options.timeoutMs) : DEFAULT_TIMEOUT_MS;
  const normalized = normalizeUrl(rawUrl);
  const normalizedUrl = normalized.toString();
  const strategy = pickStrategy(normalized);

  if (strategy === 'special-browser-fetch') {
    let fetchSpecialSiteToMarkdown;
    try {
      ({ fetchSpecialSiteToMarkdown } = await import('./fetch_special_sites.mjs'));
    } catch (error) {
      const reason = error instanceof Error ? error.message : String(error);
      throw new Error(
        `Special-site dependencies are missing. Run 'npm install' in the skill folder first. Details: ${reason}`
      );
    }

    const result = await fetchSpecialSiteToMarkdown(normalizedUrl, { timeoutMs });
    return {
      source: result.source,
      strategy: result.strategy || strategy,
      proxyUrl: '',
      requestedUrl: rawUrl,
      resolvedUrl: result.resolvedUrl,
      title: result.title || '',
      markdown: result.markdown,
      ...(result.fallbackReason ? { fallbackReason: result.fallbackReason } : {})
    };
  }

  if (strategy === 'defuddle') {
    const proxyUrl = buildProxyUrl(DEFUDDLE_BASE, normalizedUrl);
    try {
      const proxyResult = await fetchMarkdownFromProxy(proxyUrl, timeoutMs);
      return {
        source: 'defuddle',
        strategy,
        proxyUrl,
        requestedUrl: rawUrl,
        resolvedUrl: normalizedUrl,
        title: '',
        contentType: proxyResult.contentType,
        markdown: proxyResult.markdown
      };
    } catch (defuddleError) {
      const reason = defuddleError instanceof Error ? defuddleError.message : String(defuddleError);
      const { fetchGenericToMarkdown } = await import('./fetch_generic_fallback.mjs');
      const fallback = await fetchGenericToMarkdown(normalizedUrl, { timeoutMs });
      return {
        source: fallback.source,
        strategy: fallback.strategy,
        proxyUrl: proxyUrl,
        requestedUrl: rawUrl,
        resolvedUrl: fallback.resolvedUrl,
        title: fallback.title || '',
        markdown: fallback.markdown,
        fallbackReason: reason
      };
    }
  }

  const proxyUrl = buildProxyUrl(R_JINA_BASE, normalizedUrl);
  try {
    const proxyResult = await fetchMarkdownFromProxy(proxyUrl, timeoutMs);
    return {
      source: 'r.jina.ai',
      strategy,
      proxyUrl,
      requestedUrl: rawUrl,
      resolvedUrl: normalizedUrl,
      title: '',
      contentType: proxyResult.contentType,
      markdown: proxyResult.markdown
    };
  } catch (jinaError) {
    const reason = jinaError instanceof Error ? jinaError.message : String(jinaError);
    const { fetchGenericToMarkdown } = await import('./fetch_generic_fallback.mjs');
    const fallback = await fetchGenericToMarkdown(normalizedUrl, { timeoutMs });
    return {
      source: fallback.source,
      strategy: fallback.strategy,
      proxyUrl,
      requestedUrl: rawUrl,
      resolvedUrl: fallback.resolvedUrl,
      title: fallback.title || '',
      markdown: fallback.markdown,
      fallbackReason: reason
    };
  }
}

async function main() {
  try {
    const args = parseArgs(process.argv);
    const result = await urlToMarkdown(args.url, { timeoutMs: args.timeoutMs });

    if (args.json) {
      process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
      return;
    }

    process.stdout.write(`${result.markdown}\n`);
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    process.stderr.write(`url_to_markdown failed: ${message}\n`);
    process.exitCode = 1;
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  await main();
}