Baoyu Url To Markdown

import { spawn, type ChildProcess } from "node:child_process";
import fs from "node:fs";
import { mkdir } from "node:fs/promises";
import net from "node:net";
import process from "node:process";

import { resolveUrlToMarkdownChromeProfileDir } from "./paths.js";
import { CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS } from "./constants.js";

type CdpSendOptions = { sessionId?: string; timeoutMs?: number };

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function fetchWithTimeout(url: string, init: RequestInit & { timeoutMs?: number } = {}): Promise<Response> {
  const { timeoutMs, ...rest } = init;
  if (!timeoutMs || timeoutMs <= 0) return fetch(url, rest);
  const ctl = new AbortController();
  const t = setTimeout(() => ctl.abort(), timeoutMs);
  try {
    return await fetch(url, { ...rest, signal: ctl.signal });
  } finally {
    clearTimeout(t);
  }
}

export class CdpConnection {
  private ws: WebSocket;
  private nextId = 0;
  private pending = new Map<number, { resolve: (v: unknown) => void; reject: (e: Error) => void; timer: ReturnType<typeof setTimeout> | null }>();
  private eventHandlers = new Map<string, Set<(params: unknown) => void>>();

  private constructor(ws: WebSocket) {
    this.ws = ws;
    this.ws.addEventListener("message", (event) => {
      try {
        const data = typeof event.data === "string" ? event.data : new TextDecoder().decode(event.data as ArrayBuffer);
        const msg = JSON.parse(data) as { id?: number; method?: string; params?: unknown; result?: unknown; error?: { message?: string } };
        if (msg.id) {
          const p = this.pending.get(msg.id);
          if (p) {
            this.pending.delete(msg.id);
            if (p.timer) clearTimeout(p.timer);
            if (msg.error?.message) p.reject(new Error(msg.error.message));
            else p.resolve(msg.result);
          }
        } else if (msg.method) {
          const handlers = this.eventHandlers.get(msg.method);
          if (handlers) {
            for (const h of handlers) h(msg.params);
          }
        }
      } catch {}
    });
    this.ws.addEventListener("close", () => {
      for (const [id, p] of this.pending.entries()) {
        this.pending.delete(id);
        if (p.timer) clearTimeout(p.timer);
        p.reject(new Error("CDP connection closed."));
      }
    });
  }

  static async connect(url: string, timeoutMs: number): Promise<CdpConnection> {
    const ws = new WebSocket(url);
    await new Promise<void>((resolve, reject) => {
      const t = setTimeout(() => reject(new Error("CDP connection timeout.")), timeoutMs);
      ws.addEventListener("open", () => { clearTimeout(t); resolve(); });
      ws.addEventListener("error", () => { clearTimeout(t); reject(new Error("CDP connection failed.")); });
    });
    return new CdpConnection(ws);
  }

  on(event: string, handler: (params: unknown) => void): void {
    let handlers = this.eventHandlers.get(event);
    if (!handlers) {
      handlers = new Set();
      this.eventHandlers.set(event, handlers);
    }
    handlers.add(handler);
  }

  off(event: string, handler: (params: unknown) => void): void {
    this.eventHandlers.get(event)?.delete(handler);
  }

  async send<T = unknown>(method: string, params?: Record<string, unknown>, opts?: CdpSendOptions): Promise<T> {
    const id = ++this.nextId;
    const msg: Record<string, unknown> = { id, method };
    if (params) msg.params = params;
    if (opts?.sessionId) msg.sessionId = opts.sessionId;
    const timeoutMs = opts?.timeoutMs ?? 15_000;
    const out = await new Promise<unknown>((resolve, reject) => {
      const t = timeoutMs > 0 ? setTimeout(() => { this.pending.delete(id); reject(new Error(`CDP timeout: ${method}`)); }, timeoutMs) : null;
      this.pending.set(id, { resolve, reject, timer: t });
      this.ws.send(JSON.stringify(msg));
    });
    return out as T;
  }

  close(): void {
    try { this.ws.close(); } catch {}
  }
}

export async function getFreePort(): Promise<number> {
  return await new Promise((resolve, reject) => {
    const srv = net.createServer();
    srv.unref();
    srv.on("error", reject);
    srv.listen(0, "127.0.0.1", () => {
      const addr = srv.address();
      if (!addr || typeof addr === "string") {
        srv.close(() => reject(new Error("Unable to allocate a free TCP port.")));
        return;
      }
      const port = addr.port;
      srv.close((err) => (err ? reject(err) : resolve(port)));
    });
  });
}

export function findChromeExecutable(): string | null {
  const override = process.env.URL_CHROME_PATH?.trim();
  if (override && fs.existsSync(override)) return override;

  const candidates: string[] = [];
  switch (process.platform) {
    case "darwin":
      candidates.push(
        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
        "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
        "/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
        "/Applications/Chromium.app/Contents/MacOS/Chromium",
        "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"
      );
      break;
    case "win32":
      candidates.push(
        "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
        "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
        "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
        "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe"
      );
      break;
    default:
      candidates.push(
        "/usr/bin/google-chrome",
        "/usr/bin/google-chrome-stable",
        "/usr/bin/chromium",
        "/usr/bin/chromium-browser",
        "/snap/bin/chromium",
        "/usr/bin/microsoft-edge"
      );
      break;
  }

  for (const p of candidates) {
    if (fs.existsSync(p)) return p;
  }
  return null;
}

export async function waitForChromeDebugPort(port: number, timeoutMs: number): Promise<string> {
  const start = Date.now();
  while (Date.now() - start < timeoutMs) {
    try {
      const res = await fetchWithTimeout(`http://127.0.0.1:${port}/json/version`, { timeoutMs: 5_000 });
      if (!res.ok) throw new Error(`status=${res.status}`);
      const j = (await res.json()) as { webSocketDebuggerUrl?: string };
      if (j.webSocketDebuggerUrl) return j.webSocketDebuggerUrl;
    } catch {}
    await sleep(200);
  }
  throw new Error("Chrome debug port not ready");
}

export async function launchChrome(url: string, port: number, headless: boolean = false): Promise<ChildProcess> {
  const chrome = findChromeExecutable();
  if (!chrome) throw new Error("Chrome executable not found. Install Chrome or set URL_CHROME_PATH env.");
  const profileDir = resolveUrlToMarkdownChromeProfileDir();
  await mkdir(profileDir, { recursive: true });

  const args = [
    `--remote-debugging-port=${port}`,
    `--user-data-dir=${profileDir}`,
    "--no-first-run",
    "--no-default-browser-check",
    "--disable-popup-blocking",
  ];
  if (headless) args.push("--headless=new");
  args.push(url);

  return spawn(chrome, args, { stdio: "ignore" });
}

export async function waitForNetworkIdle(cdp: CdpConnection, sessionId: string, timeoutMs: number = NETWORK_IDLE_TIMEOUT_MS): Promise<void> {
  return new Promise((resolve) => {
    let timer: ReturnType<typeof setTimeout> | null = null;
    let pending = 0;
    const cleanup = () => {
      if (timer) clearTimeout(timer);
      cdp.off("Network.requestWillBeSent", onRequest);
      cdp.off("Network.loadingFinished", onFinish);
      cdp.off("Network.loadingFailed", onFinish);
    };
    const done = () => { cleanup(); resolve(); };
    const resetTimer = () => {
      if (timer) clearTimeout(timer);
      timer = setTimeout(done, timeoutMs);
    };
    const onRequest = () => { pending++; resetTimer(); };
    const onFinish = () => { pending = Math.max(0, pending - 1); if (pending <= 2) resetTimer(); };
    cdp.on("Network.requestWillBeSent", onRequest);
    cdp.on("Network.loadingFinished", onFinish);
    cdp.on("Network.loadingFailed", onFinish);
    resetTimer();
  });
}

export async function waitForPageLoad(cdp: CdpConnection, sessionId: string, timeoutMs: number = 30_000): Promise<void> {
  return new Promise((resolve, reject) => {
    const timer = setTimeout(() => {
      cdp.off("Page.loadEventFired", handler);
      resolve();
    }, timeoutMs);
    const handler = () => {
      clearTimeout(timer);
      cdp.off("Page.loadEventFired", handler);
      resolve();
    };
    cdp.on("Page.loadEventFired", handler);
  });
}

export async function createTargetAndAttach(cdp: CdpConnection, url: string): Promise<{ targetId: string; sessionId: string }> {
  const { targetId } = await cdp.send<{ targetId: string }>("Target.createTarget", { url });
  const { sessionId } = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId, flatten: true });
  await cdp.send("Network.enable", {}, { sessionId });
  await cdp.send("Page.enable", {}, { sessionId });
  return { targetId, sessionId };
}

export async function navigateAndWait(cdp: CdpConnection, sessionId: string, url: string, timeoutMs: number): Promise<void> {
  const loadPromise = new Promise<void>((resolve, reject) => {
    const timer = setTimeout(() => reject(new Error("Page load timeout")), timeoutMs);
    const handler = (params: unknown) => {
      const p = params as { name?: string };
      if (p.name === "load" || p.name === "DOMContentLoaded") {
        clearTimeout(timer);
        cdp.off("Page.lifecycleEvent", handler);
        resolve();
      }
    };
    cdp.on("Page.lifecycleEvent", handler);
  });
  await cdp.send("Page.navigate", { url }, { sessionId });
  await loadPromise;
}

export async function evaluateScript<T>(cdp: CdpConnection, sessionId: string, expression: string, timeoutMs: number = 30_000): Promise<T> {
  const result = await cdp.send<{ result: { value?: T; type?: string; description?: string } }>(
    "Runtime.evaluate",
    { expression, returnByValue: true, awaitPromise: true },
    { sessionId, timeoutMs }
  );
  return result.result.value as T;
}

export async function autoScroll(cdp: CdpConnection, sessionId: string, steps: number = 8, waitMs: number = 600): Promise<void> {
  let lastHeight = await evaluateScript<number>(cdp, sessionId, "document.body.scrollHeight");
  for (let i = 0; i < steps; i++) {
    await evaluateScript<void>(cdp, sessionId, "window.scrollTo(0, document.body.scrollHeight)");
    await sleep(waitMs);
    const newHeight = await evaluateScript<number>(cdp, sessionId, "document.body.scrollHeight");
    if (newHeight === lastHeight) break;
    lastHeight = newHeight;
  }
  await evaluateScript<void>(cdp, sessionId, "window.scrollTo(0, 0)");
}

export function killChrome(chrome: ChildProcess): void {
  try { chrome.kill("SIGTERM"); } catch {}
  setTimeout(() => {
    if (!chrome.killed) {
      try { chrome.kill("SIGKILL"); } catch {}
    }
  }, 2_000).unref?.();
}

import { resolveUrlToMarkdownChromeProfileDir } from "./paths.js";

export const DEFAULT_USER_AGENT =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";

export const USER_DATA_DIR = resolveUrlToMarkdownChromeProfileDir();

export const DEFAULT_TIMEOUT_MS = 30_000;
export const CDP_CONNECT_TIMEOUT_MS = 15_000;
export const NETWORK_IDLE_TIMEOUT_MS = 1_500;
export const POST_LOAD_DELAY_MS = 800;
export const SCROLL_STEP_WAIT_MS = 600;
export const SCROLL_MAX_STEPS = 8;

import { parseHTML } from "linkedom";
import { Readability } from "@mozilla/readability";
import TurndownService from "turndown";
import { gfm } from "turndown-plugin-gfm";

export interface PageMetadata {
  url: string;
  title: string;
  description?: string;
  author?: string;
  published?: string;
  captured_at: string;
}

export interface ConversionResult {
  metadata: PageMetadata;
  markdown: string;
}

interface ExtractionCandidate {
  title: string | null;
  byline: string | null;
  excerpt: string | null;
  published: string | null;
  html: string | null;
  textContent: string;
  method: string;
}

type AnyRecord = Record<string, unknown>;

const MIN_CONTENT_LENGTH = 120;
const GOOD_CONTENT_LENGTH = 900;

const CONTENT_SELECTORS = [
  "article",
  "main article",
  "[role='main'] article",
  "[itemprop='articleBody']",
  ".article-content",
  ".article-body",
  ".post-content",
  ".entry-content",
  ".story-body",
  "main",
  "[role='main']",
  "#content",
  ".content",
];

const REMOVE_SELECTORS = [
  "script",
  "style",
  "noscript",
  "template",
  "iframe",
  "svg",
  "path",
  "nav",
  "aside",
  "footer",
  "header",
  "form",
  ".advertisement",
  ".ads",
  ".social-share",
  ".related-articles",
  ".comments",
  ".newsletter",
  ".cookie-banner",
  ".cookie-consent",
  "[role='navigation']",
  "[aria-label*='cookie' i]",
];

const PUBLISHED_TIME_SELECTORS = [
  "meta[property='article:published_time']",
  "meta[name='pubdate']",
  "meta[name='publishdate']",
  "meta[name='date']",
  "time[datetime]",
];

const ARTICLE_TYPES = new Set([
  "Article",
  "NewsArticle",
  "BlogPosting",
  "WebPage",
  "ReportageNewsArticle",
]);

const NEXT_DATA_CONTENT_PATHS = [
  "props.pageProps.content.body",
  "props.pageProps.article.body",
  "props.pageProps.article.content",
  "props.pageProps.post.body",
  "props.pageProps.post.content",
  "props.pageProps.data.body",
  "props.pageProps.story.body.content",
];

const cleanupAndExtractScriptBody = String.raw`
(function () {
  const baseUrl = document.baseURI || location.href;

  function toAbsolute(url) {
    if (!url) return url;
    try {
      return new URL(url, baseUrl).href;
    } catch {
      return url;
    }
  }

  function absolutizeAttr(selector, attr) {
    document.querySelectorAll(selector).forEach((el) => {
      const value = el.getAttribute(attr);
      if (!value) return;
      const abs = toAbsolute(value);
      if (abs) el.setAttribute(attr, abs);
    });
  }

  function absolutizeSrcset(selector) {
    document.querySelectorAll(selector).forEach((el) => {
      const srcset = el.getAttribute("srcset");
      if (!srcset) return;

      const normalized = srcset
        .split(",")
        .map((part) => {
          const trimmed = part.trim();
          if (!trimmed) return "";
          const pieces = trimmed.split(/\s+/);
          const url = pieces[0];
          const descriptor = pieces.slice(1).join(" ");
          const absoluteUrl = toAbsolute(url);
          return descriptor ? absoluteUrl + " " + descriptor : absoluteUrl;
        })
        .filter(Boolean)
        .join(", ");

      if (normalized) {
        el.setAttribute("srcset", normalized);
      }
    });
  }

  absolutizeAttr("a[href]", "href");
  absolutizeAttr("img[src], video[src], audio[src], source[src]", "src");
  absolutizeSrcset("img[srcset], source[srcset]");

  const removeSelectors = [
    "noscript",
    "template",
    ".cookie-banner",
    ".cookie-consent",
    ".consent-banner",
    "[aria-label*='cookie' i]",
    ".advertisement",
    ".ads"
  ];

  for (const sel of removeSelectors) {
    try {
      document.querySelectorAll(sel).forEach((el) => el.remove());
    } catch {}
  }

  function getMeta(names) {
    for (const name of names) {
      const el = document.querySelector('meta[name="' + name + '"]') || document.querySelector('meta[property="' + name + '"]');
      const content = el && el.getAttribute("content");
      if (content && content.trim()) return content.trim();
    }
    return undefined;
  }

  function flattenJsonLdItems(value) {
    if (!value || typeof value !== "object") return [];
    if (Array.isArray(value)) {
      return value.flatMap(flattenJsonLdItems);
    }

    const obj = value;
    if (Array.isArray(obj["@graph"])) {
      return obj["@graph"].flatMap(flattenJsonLdItems);
    }

    return [obj];
  }

  function extractJsonLdMeta() {
    const scripts = document.querySelectorAll('script[type="application/ld+json"]');
    for (const script of scripts) {
      try {
        const parsed = JSON.parse(script.textContent || "");
        const items = flattenJsonLdItems(parsed);
        for (const item of items) {
          const rawType = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
          if (typeof rawType !== "string") continue;
          if (!["Article", "NewsArticle", "BlogPosting", "WebPage"].includes(rawType)) continue;

          const author = (() => {
            if (typeof item.author === "string") return item.author;
            if (Array.isArray(item.author) && item.author.length > 0) {
              const first = item.author[0];
              return first && typeof first === "object" ? first.name : undefined;
            }
            if (item.author && typeof item.author === "object") {
              return item.author.name;
            }
            return undefined;
          })();

          return {
            title: item.headline || item.name,
            description: item.description,
            author: typeof author === "string" ? author : undefined,
            published: item.datePublished || item.dateCreated,
          };
        }
      } catch {}
    }
    return {};
  }

  const jsonLd = extractJsonLdMeta();

  const timeEl = document.querySelector("time[datetime]");
  const title =
    getMeta(["og:title", "twitter:title"]) ||
    (typeof jsonLd.title === "string" ? jsonLd.title : undefined) ||
    document.querySelector("h1")?.textContent?.trim() ||
    document.title?.trim() ||
    "";

  const description =
    getMeta(["description", "og:description", "twitter:description"]) ||
    (typeof jsonLd.description === "string" ? jsonLd.description : undefined);

  const author =
    getMeta(["author", "article:author", "twitter:creator"]) ||
    (typeof jsonLd.author === "string" ? jsonLd.author : undefined);

  const published =
    timeEl?.getAttribute("datetime") ||
    getMeta(["article:published_time", "datePublished", "publishdate", "date"]) ||
    (typeof jsonLd.published === "string" ? jsonLd.published : undefined);

  return {
    title,
    description,
    author,
    published,
    html: document.documentElement.outerHTML,
  };
})()
`;

export const cleanupAndExtractScript = cleanupAndExtractScriptBody;

function pickString(...values: unknown[]): string | null {
  for (const value of values) {
    if (typeof value === "string") {
      const trimmed = value.trim();
      if (trimmed) return trimmed;
    }
  }
  return null;
}

function generateExcerpt(excerpt: string | null, textContent: string | null): string | null {
  if (excerpt) return excerpt;
  if (!textContent) return null;
  const trimmed = textContent.trim();
  if (!trimmed) return null;
  return trimmed.length > 200 ? `${trimmed.slice(0, 200)}...` : trimmed;
}

function extractPublishedTime(document: any): string | null {
  for (const selector of PUBLISHED_TIME_SELECTORS) {
    const el = document.querySelector(selector);
    if (!el) continue;
    const value = el.getAttribute("content") ?? el.getAttribute("datetime");
    if (value && value.trim()) return value.trim();
  }
  return null;
}

function extractTitle(document: any): string | null {
  const ogTitle = document.querySelector("meta[property='og:title']")?.getAttribute("content");
  if (ogTitle && ogTitle.trim()) return ogTitle.trim();

  const twitterTitle = document
    .querySelector("meta[name='twitter:title']")
    ?.getAttribute("content");
  if (twitterTitle && twitterTitle.trim()) return twitterTitle.trim();

  const title = document.querySelector("title")?.textContent?.trim();
  if (title) {
    const cleaned = title.split(/\s*[-|–—]\s*/)[0]?.trim();
    if (cleaned) return cleaned;
  }

  const h1 = document.querySelector("h1")?.textContent?.trim();
  return h1 || null;
}

function extractTextFromHtml(html: string): string {
  const { document } = parseHTML(`<!doctype html><html><body>${html}</body></html>`);
  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
    for (const el of document.querySelectorAll(selector)) {
      el.remove();
    }
  }
  return document.body?.textContent?.replace(/\s+/g, " ").trim() ?? "";
}

function parseDocument(html: string): any {
  const normalized = /<\s*html[\s>]/i.test(html)
    ? html
    : `<!doctype html><html><body>${html}</body></html>`;
  return parseHTML(normalized).document;
}

function sanitizeHtml(html: string): string {
  const { document } = parseHTML(`<div id="__root">${html}</div>`);
  const root = document.querySelector("#__root");
  if (!root) return html;

  for (const selector of ["script", "style", "iframe", "noscript", "template", "svg", "path"]) {
    for (const el of root.querySelectorAll(selector)) {
      el.remove();
    }
  }

  return root.innerHTML;
}

function flattenJsonLdItems(data: unknown): AnyRecord[] {
  if (!data || typeof data !== "object") return [];
  if (Array.isArray(data)) return data.flatMap(flattenJsonLdItems);

  const item = data as AnyRecord;
  if (Array.isArray(item["@graph"])) {
    return (item["@graph"] as unknown[]).flatMap(flattenJsonLdItems);
  }

  return [item];
}

function parseJsonLdScripts(document: any): AnyRecord[] {
  const results: AnyRecord[] = [];
  const scripts = document.querySelectorAll("script[type='application/ld+json']");

  for (const script of scripts) {
    try {
      const data = JSON.parse(script.textContent ?? "");
      results.push(...flattenJsonLdItems(data));
    } catch {
      // ignore malformed blocks
    }
  }

  return results;
}

function isArticleType(item: AnyRecord): boolean {
  const value = Array.isArray(item["@type"]) ? item["@type"][0] : item["@type"];
  return typeof value === "string" && ARTICLE_TYPES.has(value);
}

function extractAuthorFromJsonLd(authorData: unknown): string | null {
  if (typeof authorData === "string") return authorData;
  if (!authorData || typeof authorData !== "object") return null;

  if (Array.isArray(authorData)) {
    const names = authorData
      .map((author) => extractAuthorFromJsonLd(author))
      .filter((name): name is string => Boolean(name));
    return names.length > 0 ? names.join(", ") : null;
  }

  const author = authorData as AnyRecord;
  return typeof author.name === "string" ? author.name : null;
}

function parseJsonLdItem(item: AnyRecord): ExtractionCandidate | null {
  if (!isArticleType(item)) return null;

  const rawContent =
    (typeof item.articleBody === "string" && item.articleBody) ||
    (typeof item.text === "string" && item.text) ||
    (typeof item.description === "string" && item.description) ||
    null;

  if (!rawContent) return null;

  const content = rawContent.trim();
  const htmlLike = /<\/?[a-z][\s\S]*>/i.test(content);
  const textContent = htmlLike ? extractTextFromHtml(content) : content;

  if (textContent.length < MIN_CONTENT_LENGTH) return null;

  return {
    title: pickString(item.headline, item.name),
    byline: extractAuthorFromJsonLd(item.author),
    excerpt: pickString(item.description),
    published: pickString(item.datePublished, item.dateCreated),
    html: htmlLike ? content : null,
    textContent,
    method: "json-ld",
  };
}

function tryJsonLdExtraction(document: any): ExtractionCandidate | null {
  for (const item of parseJsonLdScripts(document)) {
    const extracted = parseJsonLdItem(item);
    if (extracted) return extracted;
  }
  return null;
}

function getByPath(value: unknown, path: string): unknown {
  let current = value;
  for (const part of path.split(".")) {
    if (!current || typeof current !== "object") return undefined;
    current = (current as AnyRecord)[part];
  }
  return current;
}

function isContentBlockArray(value: unknown): value is AnyRecord[] {
  if (!Array.isArray(value) || value.length === 0) return false;
  return value.slice(0, 5).some((item) => {
    if (!item || typeof item !== "object") return false;
    const obj = item as AnyRecord;
    return "type" in obj || "text" in obj || "textHtml" in obj || "content" in obj;
  });
}

function extractTextFromContentBlocks(blocks: AnyRecord[]): string {
  const parts: string[] = [];

  function pushParagraph(text: string): void {
    const trimmed = text.trim();
    if (!trimmed) return;
    parts.push(trimmed, "\n\n");
  }

  function walk(node: unknown): void {
    if (!node || typeof node !== "object") return;
    const block = node as AnyRecord;

    if (typeof block.text === "string") {
      pushParagraph(block.text);
      return;
    }

    if (typeof block.textHtml === "string") {
      pushParagraph(extractTextFromHtml(block.textHtml));
      return;
    }

    if (Array.isArray(block.items)) {
      for (const item of block.items) {
        if (item && typeof item === "object") {
          const text = pickString((item as AnyRecord).text);
          if (text) parts.push(`- ${text}\n`);
        }
      }
      parts.push("\n");
    }

    if (Array.isArray(block.components)) {
      for (const component of block.components) {
        walk(component);
      }
    }

    if (Array.isArray(block.content)) {
      for (const child of block.content) {
        walk(child);
      }
    }
  }

  for (const block of blocks) {
    walk(block);
  }

  return parts.join("").replace(/\n{3,}/g, "\n\n").trim();
}

function tryStringBodyExtraction(
  content: string,
  meta: AnyRecord,
  document: any,
  method: string
): ExtractionCandidate | null {
  if (!content || content.length < MIN_CONTENT_LENGTH) return null;

  const isHtml = /<\/?[a-z][\s\S]*>/i.test(content);
  const html = isHtml ? sanitizeHtml(content) : null;
  const textContent = isHtml ? extractTextFromHtml(html) : content.trim();

  if (textContent.length < MIN_CONTENT_LENGTH) return null;

  return {
    title: pickString(meta.headline, meta.title, extractTitle(document)),
    byline: pickString(meta.byline, meta.author),
    excerpt: pickString(meta.description, meta.excerpt, generateExcerpt(null, textContent)),
    published: pickString(meta.datePublished, meta.publishedAt, extractPublishedTime(document)),
    html,
    textContent,
    method,
  };
}

function tryNextDataExtraction(document: any): ExtractionCandidate | null {
  try {
    const script = document.querySelector("script#__NEXT_DATA__");
    if (!script?.textContent) return null;

    const data = JSON.parse(script.textContent) as AnyRecord;
    const pageProps = (getByPath(data, "props.pageProps") ?? {}) as AnyRecord;

    for (const path of NEXT_DATA_CONTENT_PATHS) {
      const value = getByPath(data, path);

      if (typeof value === "string") {
        const parentPath = path.split(".").slice(0, -1).join(".");
        const parent = (getByPath(data, parentPath) ?? {}) as AnyRecord;
        const meta = {
          ...pageProps,
          ...parent,
          title: parent.title ?? (pageProps.title as string | undefined),
        };

        const candidate = tryStringBodyExtraction(value, meta, document, "next-data");
        if (candidate) return candidate;
      }

      if (isContentBlockArray(value)) {
        const textContent = extractTextFromContentBlocks(value);
        if (textContent.length < MIN_CONTENT_LENGTH) continue;

        return {
          title: pickString(
            getByPath(data, "props.pageProps.content.headline"),
            getByPath(data, "props.pageProps.article.headline"),
            getByPath(data, "props.pageProps.article.title"),
            getByPath(data, "props.pageProps.post.title"),
            pageProps.title,
            extractTitle(document)
          ),
          byline: pickString(
            getByPath(data, "props.pageProps.author.name"),
            getByPath(data, "props.pageProps.article.author.name")
          ),
          excerpt: pickString(
            getByPath(data, "props.pageProps.content.description"),
            getByPath(data, "props.pageProps.article.description"),
            pageProps.description,
            generateExcerpt(null, textContent)
          ),
          published: pickString(
            getByPath(data, "props.pageProps.content.datePublished"),
            getByPath(data, "props.pageProps.article.datePublished"),
            getByPath(data, "props.pageProps.publishedAt"),
            extractPublishedTime(document)
          ),
          html: null,
          textContent,
          method: "next-data",
        };
      }
    }
  } catch {
    return null;
  }

  return null;
}

function buildReadabilityCandidate(
  article: ReturnType<Readability["parse"]>,
  document: any,
  method: string
): ExtractionCandidate | null {
  const textContent = article?.textContent?.trim() ?? "";
  if (textContent.length < MIN_CONTENT_LENGTH) return null;

  return {
    title: pickString(article?.title, extractTitle(document)),
    byline: pickString(article?.byline),
    excerpt: pickString(article?.excerpt, generateExcerpt(null, textContent)),
    published: pickString(article?.publishedTime, extractPublishedTime(document)),
    html: article?.content ? sanitizeHtml(article.content) : null,
    textContent,
    method,
  };
}

function tryReadability(document: any): ExtractionCandidate | null {
  try {
    const strictClone = document.cloneNode(true) as Document;
    const strictResult = buildReadabilityCandidate(
      new Readability(strictClone).parse(),
      document,
      "readability"
    );
    if (strictResult) return strictResult;

    const relaxedClone = document.cloneNode(true) as Document;
    return buildReadabilityCandidate(
      new Readability(relaxedClone, { charThreshold: 120 }).parse(),
      document,
      "readability-relaxed"
    );
  } catch {
    return null;
  }
}

function trySelectorExtraction(document: any): ExtractionCandidate | null {
  for (const selector of CONTENT_SELECTORS) {
    const element = document.querySelector(selector);
    if (!element) continue;

    const clone = element.cloneNode(true) as Element;
    for (const removeSelector of REMOVE_SELECTORS) {
      for (const node of clone.querySelectorAll(removeSelector)) {
        node.remove();
      }
    }

    const html = sanitizeHtml(clone.innerHTML);
    const textContent = extractTextFromHtml(html);
    if (textContent.length < MIN_CONTENT_LENGTH) continue;

    return {
      title: extractTitle(document),
      byline: null,
      excerpt: generateExcerpt(null, textContent),
      published: extractPublishedTime(document),
      html,
      textContent,
      method: `selector:${selector}`,
    };
  }

  return null;
}

function tryBodyExtraction(document: any): ExtractionCandidate | null {
  const body = document.body;
  if (!body) return null;

  const clone = body.cloneNode(true) as Element;
  for (const removeSelector of REMOVE_SELECTORS) {
    for (const node of clone.querySelectorAll(removeSelector)) {
      node.remove();
    }
  }

  const html = sanitizeHtml(clone.innerHTML);
  const textContent = extractTextFromHtml(html);
  if (!textContent) return null;

  return {
    title: extractTitle(document),
    byline: null,
    excerpt: generateExcerpt(null, textContent),
    published: extractPublishedTime(document),
    html,
    textContent,
    method: "body-fallback",
  };
}

function pickBestCandidate(candidates: ExtractionCandidate[]): ExtractionCandidate | null {
  if (candidates.length === 0) return null;

  const methodOrder = [
    "readability",
    "readability-relaxed",
    "next-data",
    "json-ld",
    "selector:",
    "body-fallback",
  ];

  function methodRank(method: string): number {
    const idx = methodOrder.findIndex((entry) =>
      entry.endsWith(":") ? method.startsWith(entry) : method === entry
    );
    return idx === -1 ? methodOrder.length : idx;
  }

  const ranked = [...candidates].sort((a, b) => {
    const rankA = methodRank(a.method);
    const rankB = methodRank(b.method);
    if (rankA !== rankB) return rankA - rankB;
    return (b.textContent.length ?? 0) - (a.textContent.length ?? 0);
  });

  for (const candidate of ranked) {
    if (candidate.textContent.length >= GOOD_CONTENT_LENGTH) {
      return candidate;
    }
  }

  for (const candidate of ranked) {
    if (candidate.textContent.length >= MIN_CONTENT_LENGTH) {
      return candidate;
    }
  }

  return ranked[0];
}

function extractFromHtml(html: string): ExtractionCandidate | null {
  const document = parseDocument(html);

  const readabilityCandidate = tryReadability(document);
  const nextDataCandidate = tryNextDataExtraction(document);
  const jsonLdCandidate = tryJsonLdExtraction(document);
  const selectorCandidate = trySelectorExtraction(document);
  const bodyCandidate = tryBodyExtraction(document);

  const candidates = [
    readabilityCandidate,
    nextDataCandidate,
    jsonLdCandidate,
    selectorCandidate,
    bodyCandidate,
  ].filter((candidate): candidate is ExtractionCandidate => Boolean(candidate));

  const winner = pickBestCandidate(candidates);
  if (!winner) return null;

  return {
    ...winner,
    title: winner.title ?? extractTitle(document),
    published: winner.published ?? extractPublishedTime(document),
    excerpt: winner.excerpt ?? generateExcerpt(null, winner.textContent),
  };
}

const turndown = new TurndownService({
  headingStyle: "atx",
  hr: "---",
  bulletListMarker: "-",
  codeBlockStyle: "fenced",
  emDelimiter: "*",
  strongDelimiter: "**",
  linkStyle: "inlined",
});

turndown.use(gfm);

turndown.remove(["script", "style", "iframe", "noscript", "template", "svg", "path"]);

turndown.addRule("collapseFigure", {
  filter: "figure",
  replacement(content) {
    return `\n\n${content.trim()}\n\n`;
  },
});

turndown.addRule("dropInvisibleAnchors", {
  filter(node) {
    return node.nodeName === "A" && !(node as Element).textContent?.trim();
  },
  replacement() {
    return "";
  },
});

function normalizeMarkdown(markdown: string): string {
  return markdown
    .replace(/\r\n/g, "\n")
    .replace(/[ \t]+\n/g, "\n")
    .replace(/\n{3,}/g, "\n\n")
    .trim();
}

function convertHtmlToMarkdown(html: string): string {
  if (!html || !html.trim()) return "";

  try {
    const sanitized = sanitizeHtml(html);
    return turndown.turndown(sanitized);
  } catch {
    return "";
  }
}

function fallbackPlainText(html: string): string {
  const document = parseDocument(html);
  for (const selector of ["script", "style", "noscript", "template", "iframe", "svg", "path"]) {
    for (const el of document.querySelectorAll(selector)) {
      el.remove();
    }
  }
  const text = document.body?.textContent ?? document.documentElement?.textContent ?? "";
  return normalizeMarkdown(text.replace(/\s+/g, " "));
}

export function htmlToMarkdown(html: string): string {
  if (!html || !html.trim()) return "";

  const extracted = extractFromHtml(html);
  if (!extracted) {
    return fallbackPlainText(html);
  }

  let markdown = extracted.html ? convertHtmlToMarkdown(extracted.html) : "";
  if (!markdown.trim()) {
    markdown = extracted.textContent;
  }

  return normalizeMarkdown(markdown);
}

function escapeYamlValue(value: string): string {
  return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\r?\n/g, "\\n");
}

export function formatMetadataYaml(meta: PageMetadata): string {
  const lines = ["---"];
  lines.push(`url: ${meta.url}`);
  lines.push(`title: "${escapeYamlValue(meta.title)}"`);
  if (meta.description) lines.push(`description: "${escapeYamlValue(meta.description)}"`);
  if (meta.author) lines.push(`author: "${escapeYamlValue(meta.author)}"`);
  if (meta.published) lines.push(`published: "${escapeYamlValue(meta.published)}"`);
  lines.push(`captured_at: "${escapeYamlValue(meta.captured_at)}"`);
  lines.push("---");
  return lines.join("\n");
}

export function createMarkdownDocument(result: ConversionResult): string {
  const yaml = formatMetadataYaml(result.metadata);
  const escapedTitle = result.metadata.title.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const titleRegex = new RegExp(`^#\\s+${escapedTitle}\\s*(\\n|$)`, "i");
  const hasTitle = titleRegex.test(result.markdown.trimStart());
  const title = result.metadata.title && !hasTitle ? `\n\n# ${result.metadata.title}\n\n` : "\n\n";
  return yaml + title + result.markdown;
}

import { createInterface } from "node:readline";
import { writeFile, mkdir, access } from "node:fs/promises";
import path from "node:path";
import process from "node:process";

import { CdpConnection, getFreePort, launchChrome, waitForChromeDebugPort, waitForNetworkIdle, waitForPageLoad, autoScroll, evaluateScript, killChrome } from "./cdp.js";
import { cleanupAndExtractScript, htmlToMarkdown, createMarkdownDocument, type PageMetadata, type ConversionResult } from "./html-to-markdown.js";
import { resolveUrlToMarkdownDataDir } from "./paths.js";
import { DEFAULT_TIMEOUT_MS, CDP_CONNECT_TIMEOUT_MS, NETWORK_IDLE_TIMEOUT_MS, POST_LOAD_DELAY_MS, SCROLL_STEP_WAIT_MS, SCROLL_MAX_STEPS } from "./constants.js";

function sleep(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

async function fileExists(filePath: string): Promise<boolean> {
  try {
    await access(filePath);
    return true;
  } catch {
    return false;
  }
}

interface Args {
  url: string;
  output?: string;
  wait: boolean;
  timeout: number;
}

function parseArgs(argv: string[]): Args {
  const args: Args = { url: "", wait: false, timeout: DEFAULT_TIMEOUT_MS };
  for (let i = 2; i < argv.length; i++) {
    const arg = argv[i];
    if (arg === "--wait" || arg === "-w") {
      args.wait = true;
    } else if (arg === "-o" || arg === "--output") {
      args.output = argv[++i];
    } else if (arg === "--timeout" || arg === "-t") {
      args.timeout = parseInt(argv[++i], 10) || DEFAULT_TIMEOUT_MS;
    } else if (!arg.startsWith("-") && !args.url) {
      args.url = arg;
    }
  }
  return args;
}

function generateSlug(title: string, url: string): string {
  const text = title || new URL(url).pathname.replace(/\//g, "-");
  return text
    .toLowerCase()
    .replace(/[^\w\s-]/g, "")
    .replace(/\s+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^-|-$/g, "")
    .slice(0, 50) || "page";
}

function formatTimestamp(): string {
  const now = new Date();
  const pad = (n: number) => n.toString().padStart(2, "0");
  return `${now.getFullYear()}${pad(now.getMonth() + 1)}${pad(now.getDate())}-${pad(now.getHours())}${pad(now.getMinutes())}${pad(now.getSeconds())}`;
}

async function generateOutputPath(url: string, title: string): Promise<string> {
  const domain = new URL(url).hostname.replace(/^www\./, "");
  const slug = generateSlug(title, url);
  const dataDir = resolveUrlToMarkdownDataDir();
  const basePath = path.join(dataDir, domain, `${slug}.md`);

  if (!(await fileExists(basePath))) {
    return basePath;
  }

  const timestampSlug = `${slug}-${formatTimestamp()}`;
  return path.join(dataDir, domain, `${timestampSlug}.md`);
}

async function waitForUserSignal(): Promise<void> {
  console.log("Page opened. Press Enter when ready to capture...");
  const rl = createInterface({ input: process.stdin, output: process.stdout });
  await new Promise<void>((resolve) => {
    rl.once("line", () => { rl.close(); resolve(); });
  });
}

async function captureUrl(args: Args): Promise<ConversionResult> {
  const port = await getFreePort();
  const chrome = await launchChrome(args.url, port, false);

  let cdp: CdpConnection | null = null;
  try {
    const wsUrl = await waitForChromeDebugPort(port, 30_000);
    cdp = await CdpConnection.connect(wsUrl, CDP_CONNECT_TIMEOUT_MS);

    const targets = await cdp.send<{ targetInfos: Array<{ targetId: string; type: string; url: string }> }>("Target.getTargets");
    const pageTarget = targets.targetInfos.find(t => t.type === "page" && t.url.startsWith("http"));
    if (!pageTarget) throw new Error("No page target found");

    const { sessionId } = await cdp.send<{ sessionId: string }>("Target.attachToTarget", { targetId: pageTarget.targetId, flatten: true });
    await cdp.send("Network.enable", {}, { sessionId });
    await cdp.send("Page.enable", {}, { sessionId });

    if (args.wait) {
      await waitForUserSignal();
    } else {
      console.log("Waiting for page to load...");
      await Promise.race([
        waitForPageLoad(cdp, sessionId, 15_000),
        sleep(8_000)
      ]);
      await waitForNetworkIdle(cdp, sessionId, NETWORK_IDLE_TIMEOUT_MS);
      await sleep(POST_LOAD_DELAY_MS);
      console.log("Scrolling to trigger lazy load...");
      await autoScroll(cdp, sessionId, SCROLL_MAX_STEPS, SCROLL_STEP_WAIT_MS);
      await sleep(POST_LOAD_DELAY_MS);
    }

    console.log("Capturing page content...");
    const extracted = await evaluateScript<{ title: string; description?: string; author?: string; published?: string; html: string }>(
      cdp, sessionId, cleanupAndExtractScript, args.timeout
    );

    const metadata: PageMetadata = {
      url: args.url,
      title: extracted.title || "",
      description: extracted.description,
      author: extracted.author,
      published: extracted.published,
      captured_at: new Date().toISOString()
    };

    const markdown = htmlToMarkdown(extracted.html);
    return { metadata, markdown };
  } finally {
    if (cdp) {
      try { await cdp.send("Browser.close", {}, { timeoutMs: 5_000 }); } catch {}
      cdp.close();
    }
    killChrome(chrome);
  }
}

async function main(): Promise<void> {
  const args = parseArgs(process.argv);
  if (!args.url) {
    console.error("Usage: bun main.ts <url> [-o output.md] [--wait] [--timeout ms]");
    process.exit(1);
  }

  try {
    new URL(args.url);
  } catch {
    console.error(`Invalid URL: ${args.url}`);
    process.exit(1);
  }

  console.log(`Fetching: ${args.url}`);
  console.log(`Mode: ${args.wait ? "wait" : "auto"}`);

  const result = await captureUrl(args);
  const outputPath = args.output || await generateOutputPath(args.url, result.metadata.title);
  const outputDir = path.dirname(outputPath);
  await mkdir(outputDir, { recursive: true });

  const document = createMarkdownDocument(result);
  await writeFile(outputPath, document, "utf-8");

  console.log(`Saved: ${outputPath}`);
  console.log(`Title: ${result.metadata.title || "(no title)"}`);
}

main().catch((err) => {
  console.error("Error:", err instanceof Error ? err.message : String(err));
  process.exit(1);
});

import os from "node:os";
import path from "node:path";
import process from "node:process";

const APP_DATA_DIR = "baoyu-skills";
const URL_TO_MARKDOWN_DATA_DIR = "url-to-markdown";
const PROFILE_DIR_NAME = "chrome-profile";

export function resolveUserDataRoot(): string {
  if (process.platform === "win32") {
    return process.env.APPDATA ?? path.join(os.homedir(), "AppData", "Roaming");
  }
  if (process.platform === "darwin") {
    return path.join(os.homedir(), "Library", "Application Support");
  }
  return process.env.XDG_DATA_HOME ?? path.join(os.homedir(), ".local", "share");
}

export function resolveUrlToMarkdownDataDir(): string {
  const override = process.env.URL_DATA_DIR?.trim();
  if (override) return path.resolve(override);
  return path.join(process.cwd(), URL_TO_MARKDOWN_DATA_DIR);
}

export function resolveUrlToMarkdownChromeProfileDir(): string {
  const override = process.env.URL_CHROME_PROFILE_DIR?.trim();
  if (override) return path.resolve(override);
  return path.join(resolveUserDataRoot(), APP_DATA_DIR, URL_TO_MARKDOWN_DATA_DIR, PROFILE_DIR_NAME);
}