Baoyu Danger X To Markdown

{
  "lockfileVersion": 1,
  "configVersion": 1,
  "workspaces": {
    "": {
      "name": "baoyu-danger-x-to-markdown-scripts",
      "dependencies": {
        "baoyu-chrome-cdp": "^0.1.1",
      },
    },
  },
  "packages": {
    "baoyu-chrome-cdp": ["baoyu-chrome-cdp@0.1.1", "", {}, "sha512-OR3PQ7NzJDykCXl20TnkZuwvNQ0hsVC2czje93P72xQaA3vKOyPN/Q1CwEgKuYzP7Rka4Fdh4HvURj6AoNR7Tg=="],
  }
}

import { resolveXToMarkdownChromeProfileDir } from "./paths.js";

export const DEFAULT_BEARER_TOKEN =
  "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
export const DEFAULT_USER_AGENT =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36";
export const X_LOGIN_URL = "https://x.com/home";
export const X_USER_DATA_DIR = resolveXToMarkdownChromeProfileDir();

export const X_COOKIE_NAMES = ["auth_token", "ct0", "gt", "twid"] as const;
export const X_REQUIRED_COOKIES = ["auth_token", "ct0"] as const;

export const FALLBACK_QUERY_ID = "id8pHQbQi7eZ6P9mA1th1Q";
export const FALLBACK_FEATURE_SWITCHES = [
  "profile_label_improvements_pcf_label_in_post_enabled",
  "responsive_web_profile_redirect_enabled",
  "rweb_tipjar_consumption_enabled",
  "verified_phone_label_enabled",
  "responsive_web_graphql_skip_user_profile_image_extensions_enabled",
  "responsive_web_graphql_timeline_navigation_enabled",
];
export const FALLBACK_FIELD_TOGGLES = ["withPayments", "withAuxiliaryUserLabels"];

export const FALLBACK_TWEET_QUERY_ID = "HJ9lpOL-ZlOk5CkCw0JW6Q";
export const FALLBACK_TWEET_FEATURE_SWITCHES = [
  "creator_subscriptions_tweet_preview_api_enabled",
  "premium_content_api_read_enabled",
  "communities_web_enable_tweet_community_results_fetch",
  "c9s_tweet_anatomy_moderator_badge_enabled",
  "responsive_web_grok_analyze_button_fetch_trends_enabled",
  "responsive_web_grok_analyze_post_followups_enabled",
  "responsive_web_jetfuel_frame",
  "responsive_web_grok_share_attachment_enabled",
  "responsive_web_grok_annotations_enabled",
  "articles_preview_enabled",
  "responsive_web_edit_tweet_api_enabled",
  "graphql_is_translatable_rweb_tweet_is_translatable_enabled",
  "view_counts_everywhere_api_enabled",
  "longform_notetweets_consumption_enabled",
  "responsive_web_twitter_article_tweet_consumption_enabled",
  "tweet_awards_web_tipping_enabled",
  "responsive_web_grok_show_grok_translated_post",
  "responsive_web_grok_analysis_button_from_backend",
  "post_ctas_fetch_enabled",
  "creator_subscriptions_quote_tweet_preview_enabled",
  "freedom_of_speech_not_reach_fetch_enabled",
  "standardized_nudges_misinfo",
  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled",
  "longform_notetweets_rich_text_read_enabled",
  "longform_notetweets_inline_media_enabled",
  "profile_label_improvements_pcf_label_in_post_enabled",
  "responsive_web_profile_redirect_enabled",
  "rweb_tipjar_consumption_enabled",
  "verified_phone_label_enabled",
  "responsive_web_grok_image_annotation_enabled",
  "responsive_web_grok_imagine_annotation_enabled",
  "responsive_web_grok_community_note_auto_translation_is_enabled",
  "responsive_web_graphql_skip_user_profile_image_extensions_enabled",
  "responsive_web_graphql_timeline_navigation_enabled",
  "responsive_web_enhance_cards_enabled",
];
export const FALLBACK_TWEET_FIELD_TOGGLES = [
  "withArticleRichContentState",
  "withArticlePlainText",
  "withGrokAnalyze",
  "withDisallowedReplyControls",
  "withPayments",
  "withAuxiliaryUserLabels",
];

export const FALLBACK_TWEET_DETAIL_QUERY_ID = "_8aYOgEDz35BrBcBal1-_w";
export const FALLBACK_TWEET_DETAIL_FEATURE_SWITCHES = [
  "rweb_video_screen_enabled",
  "profile_label_improvements_pcf_label_in_post_enabled",
  "rweb_tipjar_consumption_enabled",
  "verified_phone_label_enabled",
  "creator_subscriptions_tweet_preview_api_enabled",
  "responsive_web_graphql_timeline_navigation_enabled",
  "responsive_web_graphql_skip_user_profile_image_extensions_enabled",
  "premium_content_api_read_enabled",
  "communities_web_enable_tweet_community_results_fetch",
  "c9s_tweet_anatomy_moderator_badge_enabled",
  "responsive_web_grok_analyze_button_fetch_trends_enabled",
  "responsive_web_grok_analyze_post_followups_enabled",
  "responsive_web_jetfuel_frame",
  "responsive_web_grok_share_attachment_enabled",
  "articles_preview_enabled",
  "responsive_web_edit_tweet_api_enabled",
  "graphql_is_translatable_rweb_tweet_is_translatable_enabled",
  "view_counts_everywhere_api_enabled",
  "longform_notetweets_consumption_enabled",
  "responsive_web_twitter_article_tweet_consumption_enabled",
  "tweet_awards_web_tipping_enabled",
  "responsive_web_grok_show_grok_translated_post",
  "responsive_web_grok_analysis_button_from_backend",
  "creator_subscriptions_quote_tweet_preview_enabled",
  "freedom_of_speech_not_reach_fetch_enabled",
  "standardized_nudges_misinfo",
  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled",
  "longform_notetweets_rich_text_read_enabled",
  "longform_notetweets_inline_media_enabled",
  "responsive_web_grok_image_annotation_enabled",
  "responsive_web_enhance_cards_enabled",
];
export const FALLBACK_TWEET_DETAIL_FEATURE_DEFAULTS: Record<string, boolean> = {
  rweb_video_screen_enabled: false,
  profile_label_improvements_pcf_label_in_post_enabled: true,
  rweb_tipjar_consumption_enabled: true,
  verified_phone_label_enabled: false,
  creator_subscriptions_tweet_preview_api_enabled: true,
  responsive_web_graphql_timeline_navigation_enabled: true,
  responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
  premium_content_api_read_enabled: false,
  communities_web_enable_tweet_community_results_fetch: true,
  c9s_tweet_anatomy_moderator_badge_enabled: true,
  responsive_web_grok_analyze_button_fetch_trends_enabled: false,
  responsive_web_grok_analyze_post_followups_enabled: true,
  responsive_web_jetfuel_frame: false,
  responsive_web_grok_share_attachment_enabled: true,
  articles_preview_enabled: true,
  responsive_web_edit_tweet_api_enabled: true,
  graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
  view_counts_everywhere_api_enabled: true,
  longform_notetweets_consumption_enabled: true,
  responsive_web_twitter_article_tweet_consumption_enabled: true,
  tweet_awards_web_tipping_enabled: false,
  responsive_web_grok_show_grok_translated_post: false,
  responsive_web_grok_analysis_button_from_backend: true,
  creator_subscriptions_quote_tweet_preview_enabled: false,
  freedom_of_speech_not_reach_fetch_enabled: true,
  standardized_nudges_misinfo: true,
  tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
  longform_notetweets_rich_text_read_enabled: true,
  longform_notetweets_inline_media_enabled: true,
  responsive_web_grok_image_annotation_enabled: true,
  responsive_web_enhance_cards_enabled: false,
};
export const FALLBACK_TWEET_DETAIL_FIELD_TOGGLES = [
  "withArticleRichContentState",
  "withArticlePlainText",
  "withGrokAnalyze",
  "withDisallowedReplyControls",
];

import fs from "node:fs";
import path from "node:path";
import { mkdir, readFile, writeFile } from "node:fs/promises";

import { resolveXToMarkdownCookiePath } from "./paths.js";

export type CookieMap = Record<string, string>;

export type CookieFileData =
  | {
      cookies: CookieMap;
      updated_at: number;
      source?: string;
    }
  | {
      version: number;
      updatedAt: string;
      cookieMap: CookieMap;
      source?: string;
    };

export async function read_cookie_file(
  p: string = resolveXToMarkdownCookiePath()
): Promise<CookieMap | null> {
  try {
    if (!fs.existsSync(p) || !fs.statSync(p).isFile()) return null;
    const raw = await readFile(p, "utf8");
    const data = JSON.parse(raw) as unknown;

    if (data && typeof data === "object" && "cookies" in (data as any)) {
      const cookies = (data as any).cookies as unknown;
      if (cookies && typeof cookies === "object") {
        const out: CookieMap = {};
        for (const [k, v] of Object.entries(cookies as Record<string, unknown>)) {
          if (typeof v === "string") out[k] = v;
        }
        return Object.keys(out).length > 0 ? out : null;
      }
    }

    if (data && typeof data === "object" && "cookieMap" in (data as any)) {
      const cookies = (data as any).cookieMap as unknown;
      if (cookies && typeof cookies === "object") {
        const out: CookieMap = {};
        for (const [k, v] of Object.entries(cookies as Record<string, unknown>)) {
          if (typeof v === "string") out[k] = v;
        }
        return Object.keys(out).length > 0 ? out : null;
      }
    }

    if (data && typeof data === "object") {
      const out: CookieMap = {};
      for (const [k, v] of Object.entries(data as Record<string, unknown>)) {
        if (typeof v === "string") out[k] = v;
      }
      return Object.keys(out).length > 0 ? out : null;
    }

    return null;
  } catch {
    return null;
  }
}

export async function write_cookie_file(
  cookies: CookieMap,
  p: string = resolveXToMarkdownCookiePath(),
  source?: string
): Promise<void> {
  const dir = path.dirname(p);
  await mkdir(dir, { recursive: true });

  const payload: CookieFileData = {
    version: 1,
    updatedAt: new Date().toISOString(),
    cookieMap: cookies,
    source,
  };
  await writeFile(p, JSON.stringify(payload, null, 2), "utf8");
}

export const readCookieFile = read_cookie_file;
export const writeCookieFile = write_cookie_file;

import {
  CdpConnection,
  findChromeExecutable as findChromeExecutableBase,
  findExistingChromeDebugPort,
  gracefulKillChrome,
  getFreePort,
  launchChrome as launchChromeBase,
  openPageSession,
  sleep,
  waitForChromeDebugPort,
  type PlatformCandidates,
} from "baoyu-chrome-cdp";

import process from "node:process";

import { read_cookie_file, write_cookie_file } from "./cookie-file.js";
import { resolveXToMarkdownCookiePath } from "./paths.js";
import { X_COOKIE_NAMES, X_REQUIRED_COOKIES, X_LOGIN_URL, X_USER_DATA_DIR } from "./constants.js";
import type { CookieLike } from "./types.js";

const CHROME_CANDIDATES_FULL: PlatformCandidates = {
  darwin: [
    "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
    "/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
    "/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
    "/Applications/Chromium.app/Contents/MacOS/Chromium",
    "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
  ],
  win32: [
    "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
    "C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe",
    "C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe",
    "C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
  ],
  default: [
    "/usr/bin/google-chrome",
    "/usr/bin/google-chrome-stable",
    "/usr/bin/chromium",
    "/usr/bin/chromium-browser",
    "/snap/bin/chromium",
    "/usr/bin/microsoft-edge",
  ],
};

function findChromeExecutable(): string | null {
  return findChromeExecutableBase({
    candidates: CHROME_CANDIDATES_FULL,
    envNames: ["X_CHROME_PATH"],
  }) ?? null;
}

async function launchChrome(profileDir: string, port: number) {
  const chromePath = findChromeExecutable();
  if (!chromePath) throw new Error("Chrome executable not found.");
  return await launchChromeBase({
    chromePath,
    profileDir,
    port,
    url: X_LOGIN_URL,
    extraArgs: ["--disable-popup-blocking"],
  });
}

async function fetchXCookiesViaCdp(
  profileDir: string,
  timeoutMs: number,
  verbose: boolean,
  log?: (message: string) => void
): Promise<Record<string, string>> {
  const existingPort = await findExistingChromeDebugPort({ profileDir });
  const reusing = existingPort !== null;
  const port = existingPort ?? await getFreePort("X_DEBUG_PORT");
  const chrome = reusing ? null : await launchChrome(profileDir, port);

  let cdp: CdpConnection | null = null;
  let targetId: string | null = null;
  try {
    const wsUrl = await waitForChromeDebugPort(port, 30_000, { includeLastError: true });
    cdp = await CdpConnection.connect(wsUrl, 15_000);

    const page = await openPageSession({
      cdp,
      reusing,
      url: X_LOGIN_URL,
      matchTarget: (target) => target.type === "page" && (
        target.url.includes("x.com") || target.url.includes("twitter.com")
      ),
      enableNetwork: true,
    });
    const { sessionId } = page;
    targetId = page.targetId;

    if (verbose) {
      log?.(reusing
        ? `[x-cookies] Reusing existing Chrome on port ${port}. Waiting for cookies...`
        : "[x-cookies] Chrome opened. If needed, complete X login in the window. Waiting for cookies...");
    }

    const start = Date.now();
    let last: Record<string, string> = {};

    while (Date.now() - start < timeoutMs) {
      const { cookies } = await cdp.send<{ cookies: CookieLike[] }>(
        "Network.getCookies",
        { urls: ["https://x.com/", "https://twitter.com/"] },
        { sessionId, timeoutMs: 10_000 }
      );

      const m = buildXCookieMap((cookies ?? []).filter(Boolean));
      last = m;
      if (hasRequiredXCookies(m)) {
        return m;
      }

      await sleep(1000);
    }

    throw new Error(`Timed out waiting for X cookies. Last keys: ${Object.keys(last).join(", ")}`);
  } finally {
    if (cdp) {
      if (reusing && targetId) {
        try {
          await cdp.send("Target.closeTarget", { targetId }, { timeoutMs: 5_000 });
        } catch {}
      }
      cdp.close();
    }

    if (chrome) await gracefulKillChrome(chrome, port);
  }
}

function resolveCookieDomain(cookie: CookieLike): string | null {
  const rawDomain = cookie.domain?.trim();
  if (rawDomain) {
    return rawDomain.startsWith(".") ? rawDomain.slice(1) : rawDomain;
  }
  const rawUrl = cookie.url?.trim();
  if (rawUrl) {
    try {
      return new URL(rawUrl).hostname;
    } catch {
      return null;
    }
  }
  return null;
}

function pickCookieValue<T extends CookieLike>(cookies: T[], name: string): string | undefined {
  const matches = cookies.filter((cookie) => cookie.name === name && typeof cookie.value === "string");
  if (matches.length === 0) return undefined;

  const preferred = matches.find((cookie) => {
    const domain = resolveCookieDomain(cookie);
    return domain === "x.com" && (cookie.path ?? "/") === "/";
  });
  const xDomain = matches.find((cookie) => (resolveCookieDomain(cookie) ?? "").endsWith("x.com"));
  const twitterDomain = matches.find((cookie) => (resolveCookieDomain(cookie) ?? "").endsWith("twitter.com"));
  return (preferred ?? xDomain ?? twitterDomain ?? matches[0])?.value;
}

function buildXCookieMap<T extends CookieLike>(cookies: T[]): Record<string, string> {
  const cookieMap: Record<string, string> = {};
  for (const name of X_COOKIE_NAMES) {
    const value = pickCookieValue(cookies, name);
    if (value) cookieMap[name] = value;
  }
  return cookieMap;
}

export function hasRequiredXCookies(cookieMap: Record<string, string>): boolean {
  return X_REQUIRED_COOKIES.every((name) => Boolean(cookieMap[name]));
}

function filterXCookieMap(cookieMap: Record<string, string>): Record<string, string> {
  const filtered: Record<string, string> = {};
  for (const name of X_COOKIE_NAMES) {
    const value = cookieMap[name];
    if (value) filtered[name] = value;
  }
  return filtered;
}

function buildInlineCookiesFromEnv(): CookieLike[] {
  const cookies: CookieLike[] = [];
  const authToken = process.env.X_AUTH_TOKEN?.trim();
  const ct0 = process.env.X_CT0?.trim();
  const gt = process.env.X_GUEST_TOKEN?.trim();
  const twid = process.env.X_TWID?.trim();

  if (authToken) {
    cookies.push({ name: "auth_token", value: authToken, domain: "x.com", path: "/" });
  }
  if (ct0) {
    cookies.push({ name: "ct0", value: ct0, domain: "x.com", path: "/" });
  }
  if (gt) {
    cookies.push({ name: "gt", value: gt, domain: "x.com", path: "/" });
  }
  if (twid) {
    cookies.push({ name: "twid", value: twid, domain: "x.com", path: "/" });
  }

  return cookies;
}

async function loadXCookiesFromInline(log?: (message: string) => void): Promise<Record<string, string>> {
  const inline = buildInlineCookiesFromEnv();
  if (inline.length === 0) return {};

  const cookieMap = buildXCookieMap(
    inline.filter((cookie): cookie is CookieLike => Boolean(cookie?.name && typeof cookie.value === "string"))
  );

  if (Object.keys(cookieMap).length > 0) {
    log?.(`[x-cookies] Loaded X cookies from env: ${Object.keys(cookieMap).length} cookie(s).`);
  } else {
    log?.("[x-cookies] Env cookies provided but no X cookies matched.");
  }

  return cookieMap;
}

async function loadXCookiesFromFile(log?: (message: string) => void): Promise<Record<string, string>> {
  const cookiePath = resolveXToMarkdownCookiePath();
  const fileMap = filterXCookieMap((await read_cookie_file(cookiePath)) ?? {});
  if (Object.keys(fileMap).length > 0) {
    log?.(`[x-cookies] Loaded X cookies from file: ${cookiePath} (${Object.keys(fileMap).length} cookie(s))`);
  }
  return fileMap;
}

async function loadXCookiesFromCdp(log?: (message: string) => void): Promise<Record<string, string>> {
  try {
    const cookieMap = await fetchXCookiesViaCdp(X_USER_DATA_DIR, 5 * 60 * 1000, true, log);
    if (!hasRequiredXCookies(cookieMap)) return cookieMap;

    const cookiePath = resolveXToMarkdownCookiePath();
    try {
      await write_cookie_file(cookieMap, cookiePath, "cdp");
      log?.(`[x-cookies] Cookies saved to ${cookiePath}`);
    } catch (error) {
      log?.(
        `[x-cookies] Failed to write cookie file (${cookiePath}): ${
          error instanceof Error ? error.message : String(error ?? "")
        }`
      );
    }
    if (cookieMap.auth_token) log?.(`[x-cookies] auth_token: ${cookieMap.auth_token.slice(0, 20)}...`);
    if (cookieMap.ct0) log?.(`[x-cookies] ct0: ${cookieMap.ct0.slice(0, 20)}...`);
    return cookieMap;
  } catch (error) {
    log?.(
      `[x-cookies] Failed to load cookies via Chrome DevTools Protocol: ${
        error instanceof Error ? error.message : String(error ?? "")
      }`
    );
    return {};
  }
}

export async function loadXCookies(log?: (message: string) => void): Promise<Record<string, string>> {
  const inlineMap = await loadXCookiesFromInline(log);
  const fileMap = await loadXCookiesFromFile(log);
  const combined = { ...fileMap, ...inlineMap };

  if (hasRequiredXCookies(combined)) return combined;

  const cdpMap = await loadXCookiesFromCdp(log);
  return { ...fileMap, ...cdpMap, ...inlineMap };
}

export async function refreshXCookies(log?: (message: string) => void): Promise<Record<string, string>> {
  return loadXCookiesFromCdp(log);
}

export function buildCookieHeader(cookieMap: Record<string, string>): string | undefined {
  const entries = Object.entries(cookieMap).filter(([, value]) => value);
  if (entries.length === 0) return undefined;
  return entries.map(([key, value]) => `${key}=${value}`).join("; ");
}

import {
  DEFAULT_BEARER_TOKEN,
  DEFAULT_USER_AGENT,
  FALLBACK_FEATURE_SWITCHES,
  FALLBACK_FIELD_TOGGLES,
  FALLBACK_QUERY_ID,
  FALLBACK_TWEET_DETAIL_FEATURE_DEFAULTS,
  FALLBACK_TWEET_DETAIL_FEATURE_SWITCHES,
  FALLBACK_TWEET_DETAIL_FIELD_TOGGLES,
  FALLBACK_TWEET_DETAIL_QUERY_ID,
  FALLBACK_TWEET_FEATURE_SWITCHES,
  FALLBACK_TWEET_FIELD_TOGGLES,
  FALLBACK_TWEET_QUERY_ID,
} from "./constants.js";
import {
  buildFeatureMap,
  buildFieldToggleMap,
  buildRequestHeaders,
  buildTweetFieldToggleMap,
  fetchHomeHtml,
  fetchText,
  parseStringList,
} from "./http.js";
import type { ArticleQueryInfo } from "./types.js";

function isNonEmptyObject(value: unknown): value is Record<string, unknown> {
  return Boolean(value && typeof value === "object" && Object.keys(value as Record<string, unknown>).length > 0);
}

function unwrapTweetResult(result: any): any {
  if (!result) return null;
  if (result.__typename === "TweetWithVisibilityResults" && result.tweet) {
    return result.tweet;
  }
  return result;
}

function extractArticleFromTweet(payload: unknown): unknown {
  const root = (payload as { data?: any }).data ?? payload;
  const result = root?.tweetResult?.result ?? root?.tweet_result?.result ?? root?.tweet_result;
  const tweet = unwrapTweetResult(result);
  const legacy = tweet?.legacy ?? {};
  const article = legacy?.article ?? tweet?.article;
  return (
    article?.article_results?.result ??
    legacy?.article_results?.result ??
    tweet?.article_results?.result ??
    null
  );
}

function extractTweetFromPayload(payload: unknown): unknown {
  const root = (payload as { data?: any }).data ?? payload;
  const result = root?.tweetResult?.result ?? root?.tweet_result?.result ?? root?.tweet_result;
  return unwrapTweetResult(result);
}

function extractArticleFromEntity(payload: unknown): unknown {
  const root = (payload as { data?: any }).data ?? payload;
  return (
    root?.article_result_by_rest_id?.result ??
    root?.article_result_by_rest_id ??
    root?.article_entity_result?.result ??
    null
  );
}

async function resolveArticleQueryInfo(userAgent: string): Promise<ArticleQueryInfo> {
  const html = await fetchHomeHtml(userAgent);

  const bundleMatch = html.match(/"bundle\\.TwitterArticles":"([a-z0-9]+)"/);
  if (!bundleMatch) {
    return {
      queryId: FALLBACK_QUERY_ID,
      featureSwitches: FALLBACK_FEATURE_SWITCHES,
      fieldToggles: FALLBACK_FIELD_TOGGLES,
      html,
    };
  }

  const bundleHash = bundleMatch[1];
  const chunkUrl = `https://abs.twimg.com/responsive-web/client-web/bundle.TwitterArticles.${bundleHash}a.js`;
  const chunk = await fetchText(chunkUrl, {
    headers: {
      "user-agent": userAgent,
    },
  });

  const queryIdMatch = chunk.match(/queryId:\"([^\"]+)\",operationName:\"ArticleEntityResultByRestId\"/);
  const featureMatch = chunk.match(
    /operationName:\"ArticleEntityResultByRestId\"[\s\S]*?featureSwitches:\[(.*?)\]/
  );
  const fieldToggleMatch = chunk.match(
    /operationName:\"ArticleEntityResultByRestId\"[\s\S]*?fieldToggles:\[(.*?)\]/
  );

  const featureSwitches = parseStringList(featureMatch?.[1]);
  const fieldToggles = parseStringList(fieldToggleMatch?.[1]);

  return {
    queryId: queryIdMatch?.[1] ?? FALLBACK_QUERY_ID,
    featureSwitches: featureSwitches.length > 0 ? featureSwitches : FALLBACK_FEATURE_SWITCHES,
    fieldToggles: fieldToggles.length > 0 ? fieldToggles : FALLBACK_FIELD_TOGGLES,
    html,
  };
}

function resolveMainChunkHash(html: string): string | null {
  const match = html.match(/main\\.([a-z0-9]+)\\.js/);
  return match?.[1] ?? null;
}

function resolveApiChunkHash(html: string): string | null {
  const match = html.match(/api:\"([a-zA-Z0-9_-]+)\"/);
  return match?.[1] ?? null;
}

async function resolveTweetDetailQueryInfo(userAgent: string): Promise<ArticleQueryInfo> {
  const html = await fetchHomeHtml(userAgent);
  const apiHash = resolveApiChunkHash(html);
  if (!apiHash) {
    return {
      queryId: FALLBACK_TWEET_DETAIL_QUERY_ID,
      featureSwitches: FALLBACK_TWEET_DETAIL_FEATURE_SWITCHES,
      fieldToggles: FALLBACK_TWEET_DETAIL_FIELD_TOGGLES,
      html,
    };
  }

  const chunkUrl = `https://abs.twimg.com/responsive-web/client-web/api.${apiHash}a.js`;
  const chunk = await fetchText(chunkUrl, {
    headers: {
      "user-agent": userAgent,
    },
  });

  const queryIdMatch = chunk.match(/queryId:\"([^\"]+)\",operationName:\"TweetDetail\"/);
  const featureMatch = chunk.match(
    /operationName:\"TweetDetail\"[\s\S]*?featureSwitches:\[(.*?)\]/
  );
  const fieldToggleMatch = chunk.match(
    /operationName:\"TweetDetail\"[\s\S]*?fieldToggles:\[(.*?)\]/
  );

  const featureSwitches = parseStringList(featureMatch?.[1]);
  const fieldToggles = parseStringList(fieldToggleMatch?.[1]);

  return {
    queryId: queryIdMatch?.[1] ?? FALLBACK_TWEET_DETAIL_QUERY_ID,
    featureSwitches: featureSwitches.length > 0 ? featureSwitches : FALLBACK_TWEET_DETAIL_FEATURE_SWITCHES,
    fieldToggles: fieldToggles.length > 0 ? fieldToggles : FALLBACK_TWEET_DETAIL_FIELD_TOGGLES,
    html,
  };
}

function buildTweetDetailFieldToggleMap(keys: string[]): Record<string, boolean> {
  const toggles = buildFieldToggleMap(keys);
  if (Object.prototype.hasOwnProperty.call(toggles, "withArticlePlainText")) {
    toggles.withArticlePlainText = false;
  }
  if (Object.prototype.hasOwnProperty.call(toggles, "withGrokAnalyze")) {
    toggles.withGrokAnalyze = false;
  }
  if (Object.prototype.hasOwnProperty.call(toggles, "withDisallowedReplyControls")) {
    toggles.withDisallowedReplyControls = false;
  }
  return toggles;
}

async function resolveTweetQueryInfo(userAgent: string): Promise<ArticleQueryInfo> {
  const html = await fetchHomeHtml(userAgent);
  const mainHash = resolveMainChunkHash(html);
  if (!mainHash) {
    return {
      queryId: FALLBACK_TWEET_QUERY_ID,
      featureSwitches: FALLBACK_TWEET_FEATURE_SWITCHES,
      fieldToggles: FALLBACK_TWEET_FIELD_TOGGLES,
      html,
    };
  }

  const chunkUrl = `https://abs.twimg.com/responsive-web/client-web/main.${mainHash}.js`;
  const chunk = await fetchText(chunkUrl, {
    headers: {
      "user-agent": userAgent,
    },
  });

  const queryIdMatch = chunk.match(/queryId:\"([^\"]+)\",operationName:\"TweetResultByRestId\"/);
  const featureMatch = chunk.match(
    /operationName:\"TweetResultByRestId\"[\s\S]*?featureSwitches:\[(.*?)\]/
  );
  const fieldToggleMatch = chunk.match(
    /operationName:\"TweetResultByRestId\"[\s\S]*?fieldToggles:\[(.*?)\]/
  );

  const featureSwitches = parseStringList(featureMatch?.[1]);
  const fieldToggles = parseStringList(fieldToggleMatch?.[1]);

  return {
    queryId: queryIdMatch?.[1] ?? FALLBACK_TWEET_QUERY_ID,
    featureSwitches: featureSwitches.length > 0 ? featureSwitches : FALLBACK_TWEET_FEATURE_SWITCHES,
    fieldToggles: fieldToggles.length > 0 ? fieldToggles : FALLBACK_TWEET_FIELD_TOGGLES,
    html,
  };
}

async function fetchTweetResult(
  tweetId: string,
  cookieMap: Record<string, string>,
  userAgent: string,
  bearerToken: string
): Promise<unknown> {
  const queryInfo = await resolveTweetQueryInfo(userAgent);
  const features = buildFeatureMap(queryInfo.html, queryInfo.featureSwitches);
  const fieldToggles = buildTweetFieldToggleMap(queryInfo.fieldToggles);

  const url = new URL(`https://x.com/i/api/graphql/${queryInfo.queryId}/TweetResultByRestId`);
  url.searchParams.set(
    "variables",
    JSON.stringify({
      tweetId,
      withCommunity: false,
      includePromotedContent: false,
      withVoice: true,
    })
  );
  if (Object.keys(features).length > 0) {
    url.searchParams.set("features", JSON.stringify(features));
  }
  if (Object.keys(fieldToggles).length > 0) {
    url.searchParams.set("fieldToggles", JSON.stringify(fieldToggles));
  }

  const response = await fetch(url.toString(), {
    headers: buildRequestHeaders(cookieMap, userAgent, bearerToken),
  });

  const text = await response.text();
  if (!response.ok) {
    throw new Error(`X API error (${response.status}): ${text.slice(0, 400)}`);
  }

  try {
    return JSON.parse(text);
  } catch (error) {
    throw new Error(`Failed to parse response JSON: ${error instanceof Error ? error.message : String(error)}`);
  }
}

export async function fetchTweetDetail(
  tweetId: string,
  cookieMap: Record<string, string>,
  cursor?: string
): Promise<unknown> {
  const userAgent = process.env.X_USER_AGENT?.trim() || DEFAULT_USER_AGENT;
  const bearerToken = process.env.X_BEARER_TOKEN?.trim() || DEFAULT_BEARER_TOKEN;
  const queryInfo = await resolveTweetDetailQueryInfo(userAgent);
  const features = buildFeatureMap(
    queryInfo.html,
    queryInfo.featureSwitches,
    FALLBACK_TWEET_DETAIL_FEATURE_DEFAULTS
  );
  const fieldToggles = buildTweetDetailFieldToggleMap(queryInfo.fieldToggles);

  const url = new URL(`https://x.com/i/api/graphql/${queryInfo.queryId}/TweetDetail`);
  url.searchParams.set(
    "variables",
    JSON.stringify({
      focalTweetId: tweetId,
      cursor,
      referrer: cursor ? "tweet" : undefined,
      with_rux_injections: false,
      includePromotedContent: true,
      withCommunity: true,
      withQuickPromoteEligibilityTweetFields: true,
      withBirdwatchNotes: true,
      withVoice: true,
      withV2Timeline: true,
      withDownvotePerspective: false,
      withReactionsMetadata: false,
      withReactionsPerspective: false,
      withSuperFollowsTweetFields: false,
      withSuperFollowsUserFields: false,
    })
  );
  if (Object.keys(features).length > 0) {
    url.searchParams.set("features", JSON.stringify(features));
  }
  if (Object.keys(fieldToggles).length > 0) {
    url.searchParams.set("fieldToggles", JSON.stringify(fieldToggles));
  }

  const response = await fetch(url.toString(), {
    headers: buildRequestHeaders(cookieMap, userAgent, bearerToken),
  });

  const text = await response.text();
  if (!response.ok) {
    throw new Error(`X API error (${response.status}): ${text.slice(0, 400)}`);
  }

  try {
    return JSON.parse(text);
  } catch (error) {
    throw new Error(`Failed to parse response JSON: ${error instanceof Error ? error.message : String(error)}`);
  }
}

async function fetchArticleEntityById(
  articleEntityId: string,
  cookieMap: Record<string, string>,
  userAgent: string,
  bearerToken: string
): Promise<unknown> {
  const queryInfo = await resolveArticleQueryInfo(userAgent);
  const features = buildFeatureMap(queryInfo.html, queryInfo.featureSwitches);
  const fieldToggles = buildFieldToggleMap(queryInfo.fieldToggles);

  const url = new URL(`https://x.com/i/api/graphql/${queryInfo.queryId}/ArticleEntityResultByRestId`);
  url.searchParams.set("variables", JSON.stringify({ articleEntityId }));
  if (Object.keys(features).length > 0) {
    url.searchParams.set("features", JSON.stringify(features));
  }
  if (Object.keys(fieldToggles).length > 0) {
    url.searchParams.set("fieldToggles", JSON.stringify(fieldToggles));
  }

  const response = await fetch(url.toString(), {
    headers: buildRequestHeaders(cookieMap, userAgent, bearerToken),
  });

  const text = await response.text();
  if (!response.ok) {
    throw new Error(`X API error (${response.status}): ${text.slice(0, 400)}`);
  }

  try {
    return JSON.parse(text);
  } catch (error) {
    throw new Error(`Failed to parse response JSON: ${error instanceof Error ? error.message : String(error)}`);
  }
}

export async function fetchXArticle(
  articleId: string,
  cookieMap: Record<string, string>,
  raw: boolean
): Promise<unknown> {
  const userAgent = process.env.X_USER_AGENT?.trim() || DEFAULT_USER_AGENT;
  const bearerToken = process.env.X_BEARER_TOKEN?.trim() || DEFAULT_BEARER_TOKEN;

  const tweetPayload = await fetchTweetResult(articleId, cookieMap, userAgent, bearerToken);
  if (raw) {
    return tweetPayload;
  }

  const articleFromTweet = extractArticleFromTweet(tweetPayload);
  if (isNonEmptyObject(articleFromTweet)) {
    return articleFromTweet;
  }

  const articlePayload = await fetchArticleEntityById(articleId, cookieMap, userAgent, bearerToken);
  const articleFromEntity = extractArticleFromEntity(articlePayload);
  if (isNonEmptyObject(articleFromEntity)) {
    return articleFromEntity;
  }
  return articleFromEntity ?? articlePayload;
}

export async function fetchXTweet(
  tweetId: string,
  cookieMap: Record<string, string>,
  raw: boolean
): Promise<unknown> {
  const userAgent = process.env.X_USER_AGENT?.trim() || DEFAULT_USER_AGENT;
  const bearerToken = process.env.X_BEARER_TOKEN?.trim() || DEFAULT_BEARER_TOKEN;

  const tweetPayload = await fetchTweetResult(tweetId, cookieMap, userAgent, bearerToken);
  if (raw) {
    return tweetPayload;
  }

  const tweet = extractTweetFromPayload(tweetPayload);
  if (isNonEmptyObject(tweet)) {
    return tweet;
  }
  return tweet ?? tweetPayload;
}

import { buildCookieHeader } from "./cookies.js";

let cachedHomeHtml: { userAgent: string; html: string } | null = null;

export async function fetchText(url: string, init?: RequestInit): Promise<string> {
  const response = await fetch(url, init);
  const text = await response.text();
  if (!response.ok) {
    throw new Error(`Request failed (${response.status}) for ${url}: ${text.slice(0, 200)}`);
  }
  return text;
}

export async function fetchHomeHtml(userAgent: string): Promise<string> {
  if (cachedHomeHtml?.userAgent === userAgent) {
    return cachedHomeHtml.html;
  }
  const html = await fetchText("https://x.com", {
    headers: {
      "user-agent": userAgent,
    },
  });
  cachedHomeHtml = { userAgent, html };
  return html;
}

export function parseStringList(raw: string | undefined): string[] {
  if (!raw) return [];
  return raw
    .split(",")
    .map((item) => item.trim())
    .filter(Boolean)
    .map((item) => item.replace(/^\"|\"$/g, ""));
}

export function resolveFeatureValue(html: string, key: string): boolean | undefined {
  const keyPattern = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
  const unescaped = new RegExp(`"${keyPattern}"\\s*:\\s*\\{"value"\\s*:\\s*(true|false)`);
  const escaped = new RegExp(`\\\\"${keyPattern}\\\\"\\s*:\\s*\\\\{\\\\"value\\\\"\\s*:\\s*(true|false)`);
  const match = html.match(unescaped) ?? html.match(escaped);
  if (!match) return undefined;
  return match[1] === "true";
}

export function buildFeatureMap(
  html: string,
  keys: string[],
  defaults?: Record<string, boolean>
): Record<string, boolean> {
  const features: Record<string, boolean> = {};
  for (const key of keys) {
    const value = resolveFeatureValue(html, key);
    if (value !== undefined) {
      features[key] = value;
    } else if (defaults && Object.prototype.hasOwnProperty.call(defaults, key)) {
      features[key] = defaults[key] ?? true;
    } else {
      features[key] = true;
    }
  }
  if (!Object.prototype.hasOwnProperty.call(features, "responsive_web_graphql_exclude_directive_enabled")) {
    features.responsive_web_graphql_exclude_directive_enabled = true;
  }
  return features;
}

export function buildFieldToggleMap(keys: string[]): Record<string, boolean> {
  const toggles: Record<string, boolean> = {};
  for (const key of keys) {
    toggles[key] = true;
  }
  return toggles;
}

export function buildTweetFieldToggleMap(keys: string[]): Record<string, boolean> {
  const toggles: Record<string, boolean> = {};
  for (const key of keys) {
    if (key === "withGrokAnalyze" || key === "withDisallowedReplyControls") {
      toggles[key] = false;
    } else {
      toggles[key] = true;
    }
  }
  return toggles;
}

export function buildRequestHeaders(
  cookieMap: Record<string, string>,
  userAgent: string,
  bearerToken: string
): Record<string, string> {
  const headers: Record<string, string> = {
    authorization: bearerToken,
    "user-agent": userAgent,
    accept: "application/json",
    "x-twitter-active-user": "yes",
    "x-twitter-client-language": "en",
    "accept-language": "en",
  };

  if (cookieMap.auth_token) {
    headers["x-twitter-auth-type"] = "OAuth2Session";
  }

  const cookieHeader = buildCookieHeader(cookieMap);
  if (cookieHeader) {
    headers.cookie = cookieHeader;
  }
  if (cookieMap.ct0) {
    headers["x-csrf-token"] = cookieMap.ct0;
  }
  if (process.env.X_CLIENT_TRANSACTION_ID?.trim()) {
    headers["x-client-transaction-id"] = process.env.X_CLIENT_TRANSACTION_ID.trim();
  }

  return headers;
}

import fs from "node:fs";
import path from "node:path";
import readline from "node:readline";
import process from "node:process";
import { mkdir, readFile, writeFile } from "node:fs/promises";

import { fetchXArticle } from "./graphql.js";
import { formatArticleMarkdown } from "./markdown.js";
import { localizeMarkdownMedia, type LocalizeMarkdownMediaResult } from "./media-localizer.js";
import { resolveReferencedTweetsFromArticle } from "./referenced-tweets.js";
import { hasRequiredXCookies, loadXCookies, refreshXCookies } from "./cookies.js";
import { resolveXToMarkdownConsentPath } from "./paths.js";
import { tweetToMarkdown } from "./tweet-to-markdown.js";

type CliArgs = {
  url: string | null;
  output: string | null;
  json: boolean;
  login: boolean;
  downloadMedia: boolean;
  help: boolean;
};

type ConsentRecord = {
  version: number;
  accepted: boolean;
  acceptedAt: string;
  disclaimerVersion: string;
};

const DISCLAIMER_VERSION = "1.0";

function formatScriptCommand(fallback: string): string {
  const raw = process.argv[1];
  const displayPath = raw
    ? (() => {
        const relative = path.relative(process.cwd(), raw);
        return relative && !relative.startsWith("..") ? relative : raw;
      })()
    : fallback;
  const quotedPath = displayPath.includes(" ")
    ? `"${displayPath.replace(/"/g, '\\"')}"`
    : displayPath;
  return `npx -y bun ${quotedPath}`;
}

function printUsage(exitCode: number): never {
  const cmd = formatScriptCommand("scripts/main.ts");
  console.log(`X (Twitter) to Markdown

Usage:
  ${cmd} <url>
  ${cmd} --url <url>

Options:
  --output <path>, -o  Output path (file or dir). Default: ./x-to-markdown/<slug>/
  --json               Output as JSON
  --download-media     Download images/videos to local ./imgs and ./videos next to markdown
  --login              Refresh cookies only, then exit
  --help, -h           Show help

Examples:
  ${cmd} https://x.com/username/status/1234567890
  ${cmd} https://x.com/i/article/1234567890 -o ./article.md
  ${cmd} https://x.com/username/status/1234567890 -o ./out/
  ${cmd} https://x.com/username/status/1234567890 --download-media
  ${cmd} https://x.com/username/status/1234567890 --json | jq -r '.markdownPath'
  ${cmd} --login
`);
  process.exit(exitCode);
}

function parseArgs(argv: string[]): CliArgs {
  const out: CliArgs = {
    url: null,
    output: null,
    json: false,
    login: false,
    downloadMedia: false,
    help: false,
  };

  const positional: string[] = [];

  for (let i = 0; i < argv.length; i++) {
    const a = argv[i]!;

    if (a === "--help" || a === "-h") {
      out.help = true;
      continue;
    }

    if (a === "--json") {
      out.json = true;
      continue;
    }

    if (a === "--login") {
      out.login = true;
      continue;
    }

    if (a === "--download-media") {
      out.downloadMedia = true;
      continue;
    }

    if (a === "--url") {
      const v = argv[++i];
      if (!v) throw new Error("Missing value for --url");
      out.url = v;
      continue;
    }

    if (a === "--output" || a === "-o") {
      const v = argv[++i];
      if (!v) throw new Error(`Missing value for ${a}`);
      out.output = v;
      continue;
    }

    if (a.startsWith("-")) {
      throw new Error(`Unknown option: ${a}`);
    }

    positional.push(a);
  }

  if (!out.url && positional.length > 0) {
    out.url = positional[0]!;
  }

  return out;
}

function normalizeInputUrl(input: string): string {
  const trimmed = input.trim();
  if (!trimmed) return "";
  try {
    return new URL(trimmed).toString();
  } catch {
    return trimmed;
  }
}

function parseArticleId(input: string): string | null {
  const trimmed = input.trim();
  if (!trimmed) return null;

  try {
    const parsed = new URL(trimmed);
    const match = parsed.pathname.match(/\/(?:i\/)?article\/(\d+)/);
    if (match?.[1]) return match[1];
  } catch {
    return null;
  }

  return null;
}

function parseTweetId(input: string): string | null {
  const trimmed = input.trim();
  if (!trimmed) return null;
  if (/^\d+$/.test(trimmed)) return trimmed;

  try {
    const parsed = new URL(trimmed);
    const match = parsed.pathname.match(/\/status(?:es)?\/(\d+)/);
    if (match?.[1]) return match[1];
  } catch {
    return null;
  }

  return null;
}

function parseTweetUsername(input: string): string | null {
  const trimmed = input.trim();
  if (!trimmed) return null;
  try {
    const parsed = new URL(trimmed);
    const match = parsed.pathname.match(/^\/([^/]+)\/status(?:es)?\/\d+/);
    if (match?.[1]) return match[1];
  } catch {
    return null;
  }
  return null;
}

function sanitizeSlug(input: string): string {
  return input
    .trim()
    .replace(/^@/, "")
    .replace(/[^a-zA-Z0-9_-]+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^[-_]+|[-_]+$/g, "")
    .slice(0, 120);
}

function extractContentSlug(markdown: string): string {
  const headingMatch = markdown.match(/^#\s+(.+)$/m);
  if (headingMatch?.[1]) {
    return sanitizeSlug(headingMatch[1].slice(0, 60)).toLowerCase();
  }
  const lines = markdown.split("\n");
  let inFrontmatter = false;
  for (const line of lines) {
    if (line === "---") {
      inFrontmatter = !inFrontmatter;
      continue;
    }
    if (inFrontmatter) continue;
    const trimmed = line.trim();
    if (trimmed) {
      return sanitizeSlug(trimmed.slice(0, 60)).toLowerCase();
    }
  }
  return "untitled";
}

function resolveSlugAndId(normalizedUrl: string, kind: "tweet" | "article"): { slug: string; idPart: string } {
  const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
  const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
  const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;

  const idPart = articleId ?? tweetId ?? String(Date.now());
  const userSlug = username ? sanitizeSlug(username) : null;
  const slug = userSlug ?? idPart;
  return { slug, idPart };
}

function extractFrontmatterUrls(markdown: string): string[] {
  const match = markdown.match(/^---\n([\s\S]*?)\n---/);
  if (!match?.[1]) return [];

  const lines = match[1].split("\n");
  const urls: string[] = [];
  for (const line of lines) {
    const m = line.match(/^(url|requestedUrl):\s*["']([^"']+)["']\s*$/);
    if (m?.[2]) {
      urls.push(m[2]);
    }
  }
  return urls;
}

function frontmatterMatchesTarget(
  markdown: string,
  normalizedUrl: string,
  kind: "tweet" | "article"
): boolean {
  const urls = extractFrontmatterUrls(markdown);
  if (urls.length === 0) return false;

  const targetId = kind === "article" ? parseArticleId(normalizedUrl) : parseTweetId(normalizedUrl);
  if (!targetId) return false;

  for (const url of urls) {
    const candidateId = kind === "article" ? parseArticleId(url) : parseTweetId(url);
    if (candidateId && candidateId === targetId) {
      return true;
    }
  }

  return false;
}

function listMarkdownFiles(dirPath: string): string[] {
  try {
    return fs
      .readdirSync(dirPath)
      .filter((name) => name.toLowerCase().endsWith(".md"))
      .map((name) => path.join(dirPath, name))
      .sort();
  } catch {
    return [];
  }
}

function resolveExistingMarkdownPath(
  normalizedUrl: string,
  kind: "tweet" | "article",
  argsOutput: string | null
): string | null {
  const { slug, idPart } = resolveSlugAndId(normalizedUrl, kind);
  const candidateDirs = new Set<string>();
  const candidateFiles = new Set<string>();

  if (argsOutput) {
    const resolved = path.resolve(argsOutput);
    const looksDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
    try {
      if (fs.existsSync(resolved)) {
        const stat = fs.statSync(resolved);
        if (stat.isFile()) {
          candidateFiles.add(resolved);
        } else if (stat.isDirectory()) {
          candidateDirs.add(path.join(resolved, slug, idPart));
          candidateDirs.add(resolved);
        }
      } else if (looksDir) {
        candidateDirs.add(path.join(resolved, slug, idPart));
      }
    } catch {
      // ignore and continue
    }
  } else {
    candidateDirs.add(path.resolve(process.cwd(), "x-to-markdown", slug, idPart));
  }

  for (const filePath of candidateFiles) {
    if (!filePath.toLowerCase().endsWith(".md")) continue;
    try {
      const markdown = fs.readFileSync(filePath, "utf8");
      if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
        return filePath;
      }
    } catch {
      // ignore and continue
    }
  }

  for (const dirPath of candidateDirs) {
    if (!fs.existsSync(dirPath)) continue;
    let stat: fs.Stats;
    try {
      stat = fs.statSync(dirPath);
    } catch {
      continue;
    }
    if (!stat.isDirectory()) continue;

    const markdownFiles = listMarkdownFiles(dirPath);
    for (const filePath of markdownFiles) {
      try {
        const markdown = fs.readFileSync(filePath, "utf8");
        if (frontmatterMatchesTarget(markdown, normalizedUrl, kind)) {
          return filePath;
        }
      } catch {
        // ignore and continue
      }
    }
  }

  return null;
}

async function resolveOutputPath(
  normalizedUrl: string,
  kind: "tweet" | "article",
  argsOutput: string | null,
  contentSlug: string,
  log: (message: string) => void
): Promise<{ outputDir: string; markdownPath: string; slug: string }> {
  const articleId = kind === "article" ? parseArticleId(normalizedUrl) : null;
  const tweetId = kind === "tweet" ? parseTweetId(normalizedUrl) : null;
  const username = kind === "tweet" ? parseTweetUsername(normalizedUrl) : null;

  const userSlug = username ? sanitizeSlug(username) : null;
  const idPart = articleId ?? tweetId ?? String(Date.now());
  const slug = userSlug ?? idPart;

  const defaultFileName = `${contentSlug}.md`;

  if (argsOutput) {
    const wantsDir = argsOutput.endsWith("/") || argsOutput.endsWith("\\");
    const resolved = path.resolve(argsOutput);
    try {
      if (wantsDir || (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory())) {
        const outputDir = path.join(resolved, slug, idPart);
        await mkdir(outputDir, { recursive: true });
        return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
      }
    } catch {
      // treat as file path
    }

    const outputDir = path.dirname(resolved);
    await mkdir(outputDir, { recursive: true });
    return { outputDir, markdownPath: resolved, slug };
  }

  const outputDir = path.resolve(process.cwd(), "x-to-markdown", slug, idPart);
  await mkdir(outputDir, { recursive: true });
  return { outputDir, markdownPath: path.join(outputDir, defaultFileName), slug };
}

function formatMetaMarkdown(meta: Record<string, string | number | null | undefined>): string {
  const lines = ["---"];
  for (const [key, value] of Object.entries(meta)) {
    if (value === undefined || value === null || value === "") continue;
    if (typeof value === "number") {
      lines.push(`${key}: ${value}`);
    } else {
      lines.push(`${key}: ${JSON.stringify(value)}`);
    }
  }
  lines.push("---");
  return lines.join("\n");
}

async function promptYesNo(question: string): Promise<boolean> {
  if (!process.stdin.isTTY) return false;

  const rl = readline.createInterface({
    input: process.stdin,
    output: process.stderr,
  });

  try {
    const answer = await new Promise<string>((resolve) => rl.question(question, resolve));
    const normalized = answer.trim().toLowerCase();
    return normalized === "y" || normalized === "yes";
  } finally {
    rl.close();
  }
}

function isValidConsent(value: unknown): value is ConsentRecord {
  if (!value || typeof value !== "object") return false;
  const record = value as Partial<ConsentRecord>;
  return (
    record.accepted === true &&
    record.disclaimerVersion === DISCLAIMER_VERSION &&
    typeof record.acceptedAt === "string" &&
    record.acceptedAt.length > 0
  );
}

async function ensureConsent(log: (message: string) => void): Promise<void> {
  const consentPath = resolveXToMarkdownConsentPath();

  try {
    if (fs.existsSync(consentPath) && fs.statSync(consentPath).isFile()) {
      const raw = await readFile(consentPath, "utf8");
      const parsed = JSON.parse(raw) as unknown;
      if (isValidConsent(parsed)) {
        log(
          `⚠️  Warning: Using reverse-engineered X API (not official). Accepted on: ${(parsed as ConsentRecord).acceptedAt}`
        );
        return;
      }
    }
  } catch {
    // fall through to prompt
  }

  log(`⚠️  DISCLAIMER

This tool uses a reverse-engineered X (Twitter) API, NOT an official API.

Risks:
- May break without notice if X changes their API
- No official support or guarantees
- Account restrictions possible if API usage detected
- Use at your own risk
`);

  if (!process.stdin.isTTY) {
    throw new Error(
      `Consent required. Run in a TTY or create ${consentPath} with accepted: true and disclaimerVersion: ${DISCLAIMER_VERSION}`
    );
  }

  const accepted = await promptYesNo("Do you accept these terms and wish to continue? (y/N): ");
  if (!accepted) {
    throw new Error("User declined the disclaimer. Exiting.");
  }

  await mkdir(path.dirname(consentPath), { recursive: true });
  const payload: ConsentRecord = {
    version: 1,
    accepted: true,
    acceptedAt: new Date().toISOString(),
    disclaimerVersion: DISCLAIMER_VERSION,
  };
  await writeFile(consentPath, JSON.stringify(payload, null, 2), "utf8");
  log(`[x-to-markdown] Consent saved to: ${consentPath}`);
}

async function convertArticleToMarkdown(
  inputUrl: string,
  articleId: string,
  log: (message: string) => void
): Promise<string> {
  log("[x-to-markdown] Loading cookies...");
  const cookieMap = await loadXCookies(log);
  if (!hasRequiredXCookies(cookieMap)) {
    throw new Error("Missing auth cookies. Provide X_AUTH_TOKEN and X_CT0 or log in via Chrome.");
  }

  log(`[x-to-markdown] Fetching article ${articleId}...`);
  const article = await fetchXArticle(articleId, cookieMap, false);
  const referencedTweets = await resolveReferencedTweetsFromArticle(article, cookieMap, { log });
  const { markdown: body, coverUrl } = formatArticleMarkdown(article, { referencedTweets });

  const title = typeof (article as any)?.title === "string" ? String((article as any).title).trim() : "";
  const meta = formatMetaMarkdown({
    url: `https://x.com/i/article/${articleId}`,
    requestedUrl: inputUrl,
    title: title || null,
    coverImage: coverUrl,
  });

  return [meta, body.trimEnd()].filter(Boolean).join("\n\n").trimEnd();
}

async function main(): Promise<void> {
  const args = parseArgs(process.argv.slice(2));
  if (args.help) printUsage(0);
  if (!args.login && !args.url) printUsage(1);

  const log = (message: string) => console.error(message);
  await ensureConsent(log);

  if (args.login) {
    log("[x-to-markdown] Refreshing cookies via browser login...");
    const cookieMap = await refreshXCookies(log);
    if (!hasRequiredXCookies(cookieMap)) {
      throw new Error("Missing auth cookies after login. Please ensure you are logged in to X.");
    }
    log("[x-to-markdown] Cookies refreshed.");
    return;
  }

  const normalizedUrl = normalizeInputUrl(args.url ?? "");
  const articleId = parseArticleId(normalizedUrl);
  const tweetId = parseTweetId(normalizedUrl);
  if (!articleId && !tweetId) {
    throw new Error("Invalid X url. Examples: https://x.com/<user>/status/<id> or https://x.com/i/article/<id>");
  }

  const kind = articleId ? ("article" as const) : ("tweet" as const);

  if (args.downloadMedia) {
    const existingMarkdownPath = resolveExistingMarkdownPath(normalizedUrl, kind, args.output);
    if (existingMarkdownPath) {
      log(`[x-to-markdown] Reusing existing markdown: ${existingMarkdownPath}`);
      const existingMarkdown = await readFile(existingMarkdownPath, "utf8");
      const mediaResult = await localizeMarkdownMedia(existingMarkdown, {
        markdownPath: existingMarkdownPath,
        log,
      });
      const didLocalize =
        mediaResult.downloadedImages > 0 ||
        mediaResult.downloadedVideos > 0 ||
        mediaResult.markdown !== existingMarkdown;

      if (didLocalize) {
        await writeFile(existingMarkdownPath, mediaResult.markdown, "utf8");
        log(
          `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
        );
        log(`[x-to-markdown] Saved: ${existingMarkdownPath}`);

        const { slug } = resolveSlugAndId(normalizedUrl, kind);
        if (args.json) {
          console.log(
            JSON.stringify(
              {
                url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
                requestedUrl: normalizedUrl,
                type: kind,
                slug,
                outputDir: path.dirname(existingMarkdownPath),
                markdownPath: existingMarkdownPath,
                downloadMedia: true,
                downloadedImages: mediaResult.downloadedImages,
                downloadedVideos: mediaResult.downloadedVideos,
                imageDir: mediaResult.imageDir,
                videoDir: mediaResult.videoDir,
              },
              null,
              2
            )
          );
        } else {
          console.log(existingMarkdownPath);
        }
        return;
      }

      log("[x-to-markdown] Existing markdown already localized; rebuilding content to refresh placement.");
    }
  }

  let markdown =
    kind === "article" && articleId
      ? await convertArticleToMarkdown(normalizedUrl, articleId, log)
      : await tweetToMarkdown(normalizedUrl, { log });

  const contentSlug = extractContentSlug(markdown);
  const { outputDir, markdownPath, slug } = await resolveOutputPath(normalizedUrl, kind, args.output, contentSlug, log);

  let mediaResult: LocalizeMarkdownMediaResult | null = null;

  if (args.downloadMedia) {
    mediaResult = await localizeMarkdownMedia(markdown, {
      markdownPath,
      log,
    });
    markdown = mediaResult.markdown;
    log(
      `[x-to-markdown] Media localized: images=${mediaResult.downloadedImages}, videos=${mediaResult.downloadedVideos}`
    );
  }

  await writeFile(markdownPath, markdown, "utf8");
  log(`[x-to-markdown] Saved: ${markdownPath}`);

  if (args.json) {
    console.log(
      JSON.stringify(
        {
          url: articleId ? `https://x.com/i/article/${articleId}` : normalizedUrl,
          requestedUrl: normalizedUrl,
          type: kind,
          slug,
          outputDir,
          markdownPath,
          downloadMedia: args.downloadMedia,
          downloadedImages: mediaResult?.downloadedImages ?? 0,
          downloadedVideos: mediaResult?.downloadedVideos ?? 0,
          imageDir: mediaResult?.imageDir ?? null,
          videoDir: mediaResult?.videoDir ?? null,
        },
        null,
        2
      )
    );
  } else {
    console.log(markdownPath);
  }
}

await main().catch((error) => {
  console.error(error instanceof Error ? error.message : String(error ?? ""));
  process.exit(1);
});