Feishu Doc Scraper

Security scan passed
Scanned at: 2026-05-17T16:03:13.931043
Tool: gitleaks + pattern-based validation
Content hash: 490c7a25af60db17912e78bc1932b2bc7773bcfd1c50c04c62cd501a08b6551e

#!/usr/bin/env python3
"""
Render a structured Feishu capture manifest into Markdown.
"""

from __future__ import annotations

import argparse
import json
import sys
from pathlib import Path


def to_list(value):
    if value is None:
        return []
    if isinstance(value, list):
        return [str(item) for item in value if str(item).strip()]
    return [str(value)]


def yaml_lines(manifest):
    lines = ["---"]
    simple_fields = [
        "title",
        "source",
        "published",
        "created",
        "description",
    ]
    for field in simple_fields:
        value = manifest.get(field, "")
        lines.append(f'{field}: "{str(value).replace(chr(34), chr(39))}"' if value else f"{field}:")

    for key in ("author", "tags"):
        values = to_list(manifest.get(key))
        if values:
            lines.append(f"{key}:")
            for value in values:
                lines.append(f'  - "{value.replace(chr(34), chr(39))}"')
        else:
            lines.append(f"{key}:")

    lines.append("---")
    return lines


def normalize_body(body):
    if body is None:
        return []
    if isinstance(body, list):
        return [str(block).strip() for block in body if str(block).strip()]
    text = str(body).strip()
    return [text] if text else []


def section_lines(section):
    level = int(section.get("heading_level", 2))
    level = max(1, min(level, 6))
    heading = str(section.get("heading", "")).strip()
    if not heading:
        raise ValueError("Section heading is required")

    lines = [f'{"#" * level} {heading}']
    body_blocks = normalize_body(section.get("body"))
    if body_blocks:
        lines.append("")
        lines.extend(body_blocks)
    return lines


def render_markdown(manifest):
    title = str(manifest.get("title", "")).strip()
    if not title:
        raise ValueError("Manifest title is required")

    sections = manifest.get("sections")
    if not isinstance(sections, list) or not sections:
        raise ValueError("Manifest sections must be a non-empty list")

    lines = yaml_lines(manifest)
    lines.extend(["", f"# {title}"])

    source = str(manifest.get("source", "")).strip()
    if source:
        lines.extend(["", f"Source: <{source}>"])

    for section in sections:
        lines.extend(["", *section_lines(section)])

    return "\n".join(lines).rstrip() + "\n"


def parse_args():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--input", required=True, help="Path to capture manifest JSON")
    parser.add_argument("--output", help="Optional output markdown path")
    return parser.parse_args()


def main():
    args = parse_args()
    manifest_path = Path(args.input)
    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
    markdown = render_markdown(manifest)

    if args.output:
        output_path = Path(args.output)
        output_path.write_text(markdown, encoding="utf-8")
        print(f"Wrote {output_path}")
    else:
        sys.stdout.write(markdown)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Check that expected Feishu headings are present in the final Markdown output.
"""

from __future__ import annotations

import argparse
import re
import sys
from pathlib import Path

NOISE_PATTERNS = (
    "you may also ask",
    "recommended content",
    "upload logs",
    "contact support",
    "comments",
)


def normalize(text: str) -> str:
    text = text.strip().lower()
    text = re.sub(r"^#+\s*", "", text)
    text = re.sub(r"\s+", "", text)
    # Remove punctuation and special characters.  Using a set avoids the
    # regex escaping trap (the previous character class terminated early
    # because \\] was interpreted as a literal backslash followed by ]).
    _REMOVE_CHARS = set(
        chr(c) for c in (
            0x60, 0x7E, 0x7C, 0x2C, 0x2E, 0x21, 0x3F, 0x28, 0x29,
            0x5B, 0x5D, 0x3C, 0x3E, 0x300A, 0x300B,
            0x22, 0x27, 0x201C, 0x201D, 0x2018, 0x2019,
            0x5C, 0x2D, 0x2B,
            0x3A, 0xFF1A,
            0x3002, 0xFF0C,
            0xFF01, 0xFF1F,
            0xFF08, 0xFF09,
            0x2014, 0x2013,
        )
    )
    text = "".join(c for c in text if c not in _REMOVE_CHARS)
    return text


def load_expected(headings_file: Path) -> list[str]:
    return [line.strip() for line in headings_file.read_text(encoding="utf-8").splitlines() if line.strip()]


def extract_headings(markdown_text: str) -> list[str]:
    headings = []
    for line in markdown_text.splitlines():
        if re.match(r"^#{1,6}\s+\S", line):
            headings.append(line.strip())
    return headings


def detect_noise(markdown_text: str) -> list[str]:
    lowered = markdown_text.lower()
    return [pattern for pattern in NOISE_PATTERNS if pattern in lowered]


def parse_args():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--markdown-file", required=True, help="Generated markdown file")
    parser.add_argument("--headings-file", required=True, help="Plain text file with one expected heading per line")
    return parser.parse_args()


def main():
    args = parse_args()
    markdown_path = Path(args.markdown_file)
    headings_path = Path(args.headings_file)

    markdown_text = markdown_path.read_text(encoding="utf-8")
    expected = load_expected(headings_path)
    found = extract_headings(markdown_text)

    found_index = {normalize(item): item for item in found}
    missing = [item for item in expected if normalize(item) not in found_index]
    noise_hits = detect_noise(markdown_text)

    print(f"Expected headings: {len(expected)}")
    print(f"Found markdown headings: {len(found)}")

    if missing:
        print("Missing headings:")
        for item in missing:
            print(f"  - {item}")

    if noise_hits:
        print("Noise patterns detected:")
        for item in noise_hits:
            print(f"  - {item}")

    if missing or noise_hits:
        sys.exit(1)

    print("Heading coverage check passed.")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Download images from Feishu/Lark documents via SSR HTML extraction.

When browser automation (AppleScript, JXA, Chrome DevTools) is unavailable,
this script extracts authenticated image URLs directly from the initial HTML
response and downloads them with session cookies.

Dependencies: pip install browser_cookie3 requests

Usage (single document):
    python3 download_feishu_images.py \
        --url "https://my.feishu.cn/wiki/..." \
        --doc-name "my-document" \
        --output-dir "assets/"

Usage (batch from file):
    python3 download_feishu_images.py \
        --batch-file urls.txt \
        --output-dir "assets/"

The urls.txt format (one per line, optional doc-name prefix):
    my-document|https://my.feishu.cn/wiki/...
    another-doc|https://my.feishu.cn/wiki/...

Output: downloaded images + markdown image references printed to stdout.
"""

from __future__ import annotations

import argparse
import os
import re
import sys
from pathlib import Path
from urllib.parse import urlparse

try:
    import browser_cookie3
    import requests
except ImportError as e:
    print(f"Missing dependency: {e}", file=sys.stderr)
    print("Install: pip install browser_cookie3 requests", file=sys.stderr)
    sys.exit(1)

IMAGE_URL_RE = re.compile(r'https?://internal-api-drive-stream[^\s"\'<>]+')

DEFAULT_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
    ),
    "Accept": (
        "text/html,application/xhtml+xml,application/xml;q=0.9,"
        "image/avif,image/webp,image/apng,*/*;q=0.8"
    ),
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
}


def sanitize_name(name: str) -> str:
    """Keep alphanumerics, Chinese chars, underscore, hyphen. Max 40 chars."""
    cleaned = re.sub(r"[^a-zA-Z0-9一-鿿_-]", "", name)
    return cleaned[:40]


def extract_image_urls(html_text: str) -> list[str]:
    """Extract authenticated Feishu image URLs from raw HTML."""
    seen: set[str] = set()
    unique: list[str] = []
    for url in IMAGE_URL_RE.findall(html_text):
        if url not in seen:
            seen.add(url)
            unique.append(url)
    return unique


def download_image(url: str, cookies, referer: str) -> tuple[bytes, str]:
    """Download a single image with session cookies. Returns (content, content_type)."""
    headers = {"Referer": referer}
    resp = requests.get(url, cookies=cookies, headers=headers, timeout=30)
    resp.raise_for_status()
    content_type = resp.headers.get("content-type", "image/png")
    return resp.content, content_type


def ext_from_content_type(content_type: str) -> str:
    """Map content-type to file extension."""
    ct = content_type.lower()
    if "gif" in ct:
        return "gif"
    if "jpeg" in ct or "jpg" in ct:
        return "jpg"
    if "webp" in ct:
        return "webp"
    if "svg" in ct:
        return "svg"
    return "png"


def process_document(
    url: str,
    doc_name: str,
    output_dir: Path,
    cookies,
    dry_run: bool = False,
) -> dict:
    """Download all images from a single Feishu document."""
    result = {
        "url": url,
        "doc_name": doc_name,
        "found": 0,
        "downloaded": 0,
        "errors": 0,
        "files": [],
        "markdown_refs": [],
    }

    try:
        resp = requests.get(url, cookies=cookies, headers=DEFAULT_HEADERS, timeout=30)
        resp.raise_for_status()
    except requests.RequestException as e:
        print(f"  ERROR fetching page: {e}", file=sys.stderr)
        result["errors"] += 1
        return result

    image_urls = extract_image_urls(resp.text)
    result["found"] = len(image_urls)

    if not image_urls:
        print("  No image URLs found in page HTML.")
        return result

    parsed = urlparse(url)
    referer = f"{parsed.scheme}://{parsed.netloc}/"
    safe_name = sanitize_name(doc_name)
    output_dir.mkdir(parents=True, exist_ok=True)

    for i, img_url in enumerate(image_urls):
        ext = "png"
        local_name = f"{safe_name}-{i}.{ext}"
        local_path = output_dir / local_name

        try:
            if dry_run:
                print(f"  DRY-RUN: would download -> {local_name}")
            else:
                content, content_type = download_image(img_url, cookies, referer=referer)
                ext = ext_from_content_type(content_type)
                local_name = f"{safe_name}-{i}.{ext}"
                local_path = output_dir / local_name
                local_path.write_bytes(content)
                print(f"  OK: {local_name} ({len(content)} bytes)")

            result["downloaded"] += 1
            result["files"].append(str(local_path))
            result["markdown_refs"].append(f"![](assets/{local_name})")
        except requests.RequestException as e:
            print(f"  ERROR downloading image {i}: {e}", file=sys.stderr)
            result["errors"] += 1

    return result


def parse_batch_file(path: Path) -> list[tuple[str, str]]:
    """Parse batch file. Format: doc-name|url (or just url)."""
    entries: list[tuple[str, str]] = []
    for line in path.read_text(encoding="utf-8").splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        if "|" in line:
            doc_name, url = line.split("|", 1)
            entries.append((doc_name.strip(), url.strip()))
        else:
            parsed = urlparse(line)
            doc_name = parsed.path.strip("/").split("/")[-1] or "doc"
            entries.append((doc_name, line))
    return entries


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--url", help="Single Feishu document URL")
    parser.add_argument("--doc-name", default="doc", help="Document name for image files")
    parser.add_argument("--output-dir", default="assets", help="Directory to save images")
    parser.add_argument("--batch-file", help="File with doc-name|url lines for batch processing")
    parser.add_argument("--dry-run", action="store_true", help="Print what would be done without downloading")
    return parser.parse_args()


def main() -> int:
    args = parse_args()

    if not args.url and not args.batch_file:
        print("Error: specify --url or --batch-file", file=sys.stderr)
        return 1

    try:
        cookies = browser_cookie3.chrome()
    except Exception as e:
        print(f"Error loading Chrome cookies: {e}", file=sys.stderr)
        return 1

    output_dir = Path(args.output_dir)
    total_found = 0
    total_downloaded = 0
    total_errors = 0
    all_markdown_refs: list[str] = []

    if args.batch_file:
        entries = parse_batch_file(Path(args.batch_file))
        for doc_name, url in entries:
            print(f"\n[{doc_name}] {url}")
            result = process_document(url, doc_name, output_dir, cookies, dry_run=args.dry_run)
            total_found += result["found"]
            total_downloaded += result["downloaded"]
            total_errors += result["errors"]
            all_markdown_refs.extend(result["markdown_refs"])
    else:
        print(f"\n[{args.doc_name}] {args.url}")
        result = process_document(args.url, args.doc_name, output_dir, cookies, dry_run=args.dry_run)
        total_found = result["found"]
        total_downloaded = result["downloaded"]
        total_errors = result["errors"]
        all_markdown_refs = result["markdown_refs"]

    if all_markdown_refs:
        print("\n--- Markdown references ---")
        for ref in all_markdown_refs:
            print(ref)

    print("\n--- Summary ---")
    print(f"Images found: {total_found}")
    print(f"Images downloaded: {total_downloaded}")
    print(f"Errors: {total_errors}")

    return 0 if total_errors == 0 else 1


if __name__ == "__main__":
    sys.exit(main())

// feishu_dom_capture.js — Injectable DOM capture script for Feishu/Lark documents.
// Inject via chrome-devtools evaluate_script or Browser Use javascript_tool.
// After injection, call window.__feishuCapture.run() to execute the full pipeline.
// Result: window.__feishuCapture.manifest (JSON) and window.__feishuCapture.markdown (string).

(() => {
  'use strict';

  // ── Noise patterns (Feishu UI chrome that leaks into innerText) ──
  const NOISE_EXACT = new Set([
    'Unable to print', 'Group card (Log in to view)', 'Group card',
    'Copy', 'Code block', '代码块',
    'Plain Text', 'Shell', 'JSON', 'Bash', 'TypeScript', 'JavaScript',
  ]);
  const NOISE_RE = /^Unable to print|^Group card|^Modified [A-Z][a-z]+ \d+/;

  function stripNoise(s) {
    if (typeof s !== 'string') return s;
    return s
      .replace(/Unable to print[^\s]*(\d+%)?/g, '')
      .replace(/Group card \(Log in to view\)/g, '')
      .replace(/Group card/g, '')
      .replace(/Modified [A-Z][a-z]+ \d+/g, '')
      .replace(/\s+/g, ' ')
      .trim();
  }

  function isNoise(text) {
    if (!text) return true;
    if (NOISE_EXACT.has(text)) return true;
    if (NOISE_RE.test(text)) return true;
    return false;
  }

  // ── Inline markdown: convert DOM inline tags to markdown syntax ──
  function inlineMarkdown(node) {
    let result = '';
    for (const child of node.childNodes) {
      if (child.nodeType === 3) {
        result += child.textContent;
      } else if (child.nodeType === 1) {
        const tag = child.tagName.toLowerCase();
        if (tag === 'br') { result += '\n'; continue; }
        const inner = inlineMarkdown(child);
        if (tag === 'b' || tag === 'strong') result += `**${inner}**`;
        else if (tag === 'i' || tag === 'em') result += `*${inner}*`;
        else if (tag === 'u') result += `<u>${inner}</u>`;
        else if (tag === 'code' && !child.parentElement?.querySelector('pre')) result += `\`${inner}\``;
        else if (tag === 'a' && child.getAttribute('href')) result += `[${inner}](${child.getAttribute('href')})`;
        else result += inner;
      }
    }
    return result.replace(/[​]/g, '');
  }

  // ── Table / bullet helpers ──
  function isInsideTable(el) {
    return !!el.parentElement?.closest('.docx-table-block, .table-block');
  }

  function extractBullets(el, depth = 0) {
    const results = [];
    const listContent = el.querySelector(':scope > .list-wrapper, :scope > .list-content, :scope > .ace-line');
    let textNode = listContent;
    if (!textNode) {
      const aceLines = el.querySelectorAll('.ace-line');
      for (const a of aceLines) {
        if (a.closest('.docx-bullet-block, .docx-list-block') === el) { textNode = a; break; }
      }
    }
    if (textNode) {
      const text = inlineMarkdown(textNode).replace(/^[•◦·]\s*/, '').trim();
      if (text) results.push({ depth, text });
    }
    const allNested = el.querySelectorAll('.docx-bullet-block, .docx-list-block');
    const directChildren = Array.from(allNested).filter(b => {
      const parent = b.parentElement?.closest('.docx-bullet-block, .docx-list-block');
      return b !== el && parent === el;
    });
    directChildren.forEach(child => results.push(...extractBullets(child, depth + 1)));
    return results;
  }

  function extractTable(tableBlock) {
    const rows = [];
    tableBlock.querySelectorAll('tr, .docx-table-tr').forEach(rowEl => {
      const cells = Array.from(rowEl.querySelectorAll('td, .table-cell-block, .docx-table_cell-block, [class*="table-cell"]'))
        .map(c => (c.innerText || '').replace(/[​\n]/g, ' ').trim());
      if (cells.length > 0) rows.push(cells);
    });
    return rows;
  }

  // ── Core capture ──
  const capturedBlocks = new Map();

  function captureVisibleBlocks() {
    const blocks = document.querySelectorAll('.block');
    let newCount = 0;
    for (const block of blocks) {
      const bid = block.getAttribute('data-block-id');
      if (!bid || capturedBlocks.has(bid)) continue;
      if (isInsideTable(block) && !block.className.includes('docx-table-block')) continue;
      const parentBullet = block.parentElement?.closest('.docx-bullet-block, .docx-list-block');
      if ((block.className.includes('docx-bullet-block') || block.className.includes('docx-list-block')) && parentBullet) continue;
      // Skip quote container child render units (they duplicate the container's content)
      if (block.className.includes('quote-container-render-unit')) continue;

      const cls = block.className || '';
      const text = (block.innerText || '').replace(/[​]/g, '').trim();
      let type = 'text', payload = null;

      if (cls.includes('docx-heading1-block')) { type = 'h1'; payload = text; }
      else if (cls.includes('docx-heading2-block')) { type = 'h2'; payload = text; }
      else if (cls.includes('docx-heading3-block')) { type = 'h3'; payload = text; }
      else if (cls.includes('docx-heading4-block')) { type = 'h4'; payload = text; }
      else if (cls.includes('docx-table-block')) { type = 'table'; payload = extractTable(block); }
      else if (cls.includes('docx-bullet-block') || cls.includes('docx-list-block')) { type = 'bullets'; payload = extractBullets(block, 0); }
      else if (cls.includes('docx-code-block')) {
        type = 'code';
        const langEl = block.querySelector('[class*="lang"], [class*="language"]');
        payload = { lang: langEl?.innerText?.trim() || '', text };
      } else if (cls.includes('docx-quote_container-block') || cls.includes('docx-quote-block') || cls.includes('docx-callout-block')) {
        type = 'quote'; payload = inlineMarkdown(block).trim();
      } else if (cls.includes('docx-image-block') || cls.includes('docx-image')) {
        type = 'image';
        const img = block.querySelector('img');
        payload = { src: img?.src || '', alt: img?.alt || '' };
      } else if (cls.includes('docx-divider-block')) {
        type = 'divider'; payload = '---';
      } else {
        payload = inlineMarkdown(block).trim();
        if (!payload) continue;
      }

      capturedBlocks.set(bid, { id: bid, idNum: parseInt(bid, 10), type, payload });
      newCount++;
    }
    return { newCount, total: capturedBlocks.size };
  }

  // ── TOC-driven capture loop ──
  async function tocDrivenCapture() {
    const sleep = ms => new Promise(r => setTimeout(r, ms));
    const tocItems = Array.from(document.querySelectorAll('.catalogue__list-item'));
    const scrollContainer = document.querySelector('.bear-web-x-container, .page-main, .content-scroller, [class*="docx-width"]');

    for (const item of tocItems) {
      const clickTarget = item.querySelector('a, button, [role="button"]') || item;
      clickTarget.click();
      await sleep(800);
      captureVisibleBlocks();
      if (scrollContainer) {
        for (let s = 0; s < 3; s++) {
          scrollContainer.scrollBy(0, scrollContainer.clientHeight * 0.6);
          await sleep(400);
          captureVisibleBlocks();
        }
      }
    }
    // Final sweep
    if (scrollContainer) {
      for (let s = 0; s < 8; s++) {
        scrollContainer.scrollBy(0, scrollContainer.clientHeight * 0.8);
        await sleep(500);
        captureVisibleBlocks();
      }
    }
  }

  // ── Image download via fetch + session cookie ──
  async function downloadImages(docName = 'doc') {
    const imageBlocks = Array.from(capturedBlocks.values()).filter(b => b.type === 'image' && b.payload?.src);
    const downloaded = [];
    for (let i = 0; i < imageBlocks.length; i++) {
      const src = imageBlocks[i].payload.src;
      if (src.startsWith('blob:') || src.startsWith('data:')) continue;
      try {
        const resp = await fetch(src, { credentials: 'include' });
        if (!resp.ok) { downloaded.push({ i, src: src.substring(0, 80), ok: false }); continue; }
        const contentType = resp.headers.get('content-type') || 'image/png';
        const blob = await resp.blob();
        const reader = new FileReader();
        const dataUrl = await new Promise(resolve => {
          reader.onloadend = () => resolve(reader.result);
          reader.readAsDataURL(blob);
        });
        const ext = contentType.includes('gif') ? 'gif' : contentType.includes('jpeg') ? 'jpg' : 'png';
        // Per-document naming: never share generic img-0.png across documents
        const safeName = docName.replace(/[^a-zA-Z0-9一-龥_-]/g, '').substring(0, 40);
        imageBlocks[i].payload.localName = `${safeName}-${i}.${ext}`;
        imageBlocks[i].payload.dataUrl = dataUrl;
        imageBlocks[i].payload.size = blob.size;
        downloaded.push({ i, ext, size: blob.size, ok: true });
      } catch (e) {
        downloaded.push({ i, error: e.message, ok: false });
      }
    }
    return downloaded;
  }

  // ── Clean + deduplicate + sort ──
  function cleanAndSort() {
    const all = Array.from(capturedBlocks.values());
    all.sort((a, b) => a.idNum - b.idNum);

    // Build covered-text set for dedup
    const coveredTexts = new Set();
    for (const b of all) {
      if (b.type === 'table' && Array.isArray(b.payload))
        b.payload.forEach(row => row.forEach(cell => { if (cell.trim()) coveredTexts.add(cell.trim()); }));
      if (b.type === 'bullets' && Array.isArray(b.payload))
        b.payload.forEach(item => { if (item.text?.trim()) coveredTexts.add(item.text.trim()); });
      if (b.type === 'quote' && typeof b.payload === 'string' && b.payload.trim())
        coveredTexts.add(stripNoise(b.payload));
    }

    const seenText = new Set();
    return all.filter(b => {
      // Drop aggregation artifacts (> 350 chars text blocks are page-main innerText lumps)
      if (b.type === 'text' && typeof b.payload === 'string' && b.payload.length > 350) return false;
      // Drop empty bullets
      if (b.type === 'bullets' && (!Array.isArray(b.payload) || b.payload.length === 0 || b.payload.every(it => !it.text?.trim()))) return false;

      const text = typeof b.payload === 'string' ? stripNoise(b.payload) : '';
      if (b.type === 'text' && (!text || text.length < 3)) return false;
      if (b.type === 'text' && isNoise(text)) return false;
      // Dedup: text covered by quote/table/bullets
      if (b.type === 'text' && coveredTexts.has(text)) return false;
      // Dedup by exact content
      let key = null;
      if (b.type === 'text' || b.type === 'quote') key = b.type + ':' + text;
      else if (b.type === 'bullets' && Array.isArray(b.payload)) key = 'bullets:' + b.payload.map(it => stripNoise(it.text)).join('|');
      else if (b.type === 'image' && b.payload?.src) key = 'image:' + b.payload.src;
      else if (b.type === 'code' && b.payload?.text) key = 'code:' + b.payload.text.trim();
      else if (b.type.startsWith('h')) key = b.type + ':' + b.payload;
      if (key && seenText.has(key)) return false;
      if (key) seenText.add(key);

      // Clean code-block noise
      if (b.type === 'code' && b.payload?.text) {
        let t = b.payload.text;
        t = t.split('\n').filter(l => !['Copy', 'Code block', '代码块'].includes(l.trim())).join('\n');
        t = t.replace(/^(plaintext|shell|bash|json|typescript|javascript|python)\s*\n/i, m => {
          if (!b.payload.lang) b.payload.lang = m.trim().toLowerCase();
          return '';
        });
        b.payload.text = t.trim();
      }
      return true;
    });
  }

  // ── Build manifest JSON ──
  function buildManifest(blocks, meta = {}) {
    const sections = [];
    let currentSection = null;
    const levelMap = { h1: 2, h2: 3, h3: 4, h4: 5 };

    for (const b of blocks) {
      if (levelMap[b.type]) {
        currentSection = { heading_level: levelMap[b.type], heading: (b.payload || '').trim(), body: [] };
        sections.push(currentSection);
        continue;
      }
      if (!currentSection) continue;

      const text = typeof b.payload === 'string' ? stripNoise(b.payload) : '';
      if (b.type === 'text' && text) currentSection.body.push(text);
      else if (b.type === 'quote' && text) currentSection.body.push('> ' + text);
      else if (b.type === 'bullets' && Array.isArray(b.payload)) {
        for (const item of b.payload) {
          const bt = stripNoise(item.text || '').replace(/^[•◦·]\s*/, '');
          if (bt) currentSection.body.push('  '.repeat(item.depth) + '- ' + bt);
        }
      } else if (b.type === 'code' && b.payload?.text) {
        const lang = (b.payload.lang || '').toLowerCase().replace(/[^a-z0-9]/g, '');
        currentSection.body.push('```' + lang + '\n' + b.payload.text + '\n```');
      } else if (b.type === 'table' && Array.isArray(b.payload) && b.payload.length > 0) {
        const rows = b.payload;
        currentSection.body.push('| ' + rows[0].join(' | ') + ' |');
        currentSection.body.push('|' + rows[0].map(() => '---').join('|') + '|');
        for (let i = 1; i < rows.length; i++) currentSection.body.push('| ' + rows[i].join(' | ') + ' |');
      } else if (b.type === 'image' && b.payload?.localName) {
        currentSection.body.push(`![${b.payload.alt || ''}](assets/${b.payload.localName})`);
      } else if (b.type === 'image' && b.payload?.src && !b.payload.src.startsWith('blob:')) {
        currentSection.body.push(`![${b.payload.alt || ''}](${b.payload.src})`);
      } else if (b.type === 'divider') {
        currentSection.body.push('---');
      }
    }

    return {
      title: meta.title || document.title.replace(/ - Feishu Docs$/, '').trim(),
      source: meta.source || location.href,
      author: meta.author || [],
      published: meta.published || '',
      created: new Date().toISOString().substring(0, 10),
      description: meta.description || '',
      tags: meta.tags || [],
      sections,
    };
  }

  // ── Public API ──
  window.__feishuCapture = {
    capturedBlocks,
    captureVisibleBlocks,
    tocDrivenCapture,
    downloadImages,
    cleanAndSort,
    buildManifest,
    stripNoise,
    inlineMarkdown,

    async run(meta = {}) {
      capturedBlocks.clear();
      captureVisibleBlocks();
      await tocDrivenCapture();
      const docName = meta.docName || meta.title || document.title.replace(/ - Feishu Docs$/, '').trim() || 'doc';
      const imgResults = await downloadImages(docName);
      const cleaned = cleanAndSort();
      const manifest = buildManifest(cleaned, meta);
      this.manifest = manifest;
      this.cleanedBlocks = cleaned;
      this.imageResults = imgResults;
      return {
        totalCaptured: capturedBlocks.size,
        afterClean: cleaned.length,
        sections: manifest.sections.length,
        images: imgResults.length,
        imagesOk: imgResults.filter(r => r.ok).length,
      };
    },
  };
})();

#!/usr/bin/env python3
"""Enumerate every rich-media reference in a fetched Feishu Markdown body.

This is the recursion engine's core for Path A (lark-cli API extraction). A
collection/hub is a doc whose body references other docs; missing one
reference means a missing document — the single biggest hub-scraping failure.
Hand-rolled `grep | sed` pipelines repeatedly missed the `my.feishu.cn`
personal-space pattern, so this enumeration is centralized and tested here.

Input : a Markdown file produced by `lark-cli docs +fetch ... | jq -r .data.markdown`.
Output: JSON array on stdout, one object per *distinct* reference:
        {"type": ..., "ref": <token-or-url>, "title": ..., "dispatch": <hint>}
        plus a human summary on stderr.

It only *enumerates*. Dispatching/fetching each reference is the caller's job
(see references/lark-cli-api-extraction.md, Step 5 dispatch table).

Usage:
    python3 feishu_extract_refs.py FETCHED_BODY.md
    python3 feishu_extract_refs.py FETCHED_BODY.md --type docx   # filter
"""
from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path

# Feishu/Lark hosts. feishu.cn = mainland tenants + my.feishu.cn personal space;
# larksuite.com = international Lark. Both serve the same /docx /wiki /sheets
# /minutes /base /file path scheme.
_HOST = r"[a-z0-9-]+\.(?:feishu\.cn|larksuite\.com)"

# Inline rich-media tags emitted by `docs +fetch` Markdown.
RE_MENTION_DOC = re.compile(
    r'<mention-doc\s+token="([^"]+)"\s+type="([^"]+)"\s*>([^<]*)</mention-doc>'
)
RE_SHEET_TAG = re.compile(r'<sheet\s+token="([^"]+)"\s*/?>')
RE_IMAGE_TAG = re.compile(r'<image\s+token="([^"]+)"')
RE_FILE_TAG = re.compile(r'<file\s+token="([^"]+)"[^>]*>([^<]*)</file>')
RE_LARK_TABLE = re.compile(r"<lark-table\b")

# URLs that appear in the body (cross-tenant / personal space / minutes /
# Tencent Meeting). One regex covers mainland, international and personal
# (my.feishu.cn) because the host group accepts any sub-domain.
RE_FEISHU_URL = re.compile(
    r"https://(" + _HOST + r")/(docx|wiki|sheets|base|file|minutes)/([A-Za-z0-9]+)"
)
RE_TENCENT_MEETING = re.compile(
    r"https://meeting\.tencent\.com/crm/([A-Za-z0-9]+)"
)

# How the caller should handle each type (mirrors the reference's dispatch
# table, surfaced here so the caller does not have to re-derive it).
DISPATCH = {
    "mention-doc-docx": "docs +fetch --doc <token>",
    "mention-doc-wiki": "wiki spaces get_node then docs +fetch",
    "mention-doc-sheet": "sheets +read",
    "url-docx": "docs +fetch --doc <token>",
    "url-wiki": "wiki spaces get_node then docs +fetch",
    "url-sheets": "sheets +read (split token on '_' -> SP, SID)",
    "url-base": "Bitable API (outside this skill) — record token",
    "url-file": "attachment — record token + name; treat like image gap",
    "url-minutes": "native transcript API (feishu-minutes-transcript.md)",
    "sheet-tag": "sheets +read (split token on '_' -> SP, SID)",
    "image": "register token; lark-cli cannot download docx images",
    "file": "attachment — record token + name; treat like image gap",
    "tencent-meeting": "Tencent Meeting native transcript (never download+re-ASR)",
    "lark-table": "inline content — render in place to a Markdown table",
}


def _read_text(path: Path) -> str:
    """Read the body strictly as UTF-8.

    We deliberately do NOT use errors='replace': a decode failure means an
    upstream step corrupted the text, and the skill's acceptance contract
    checks for U+FFFD. Masking it here would hide exactly the failure the
    pipeline is trying to detect, so fail loudly instead.
    """
    try:
        raw = path.read_bytes()
    except FileNotFoundError:
        sys.exit(f"error: file not found: {path}")
    except PermissionError:
        sys.exit(f"error: cannot read (permission): {path}")
    if not raw.strip():
        sys.exit(f"error: file is empty: {path} (fetch likely failed upstream)")
    try:
        return raw.decode("utf-8")
    except UnicodeDecodeError as exc:
        sys.exit(
            f"error: {path} is not valid UTF-8 ({exc}); an upstream extraction "
            f"step corrupted the body — re-fetch with `lark-cli docs +fetch "
            f"--format json` and `jq -r .data.markdown`, do not 'fix' encoding here."
        )


def extract(text: str) -> list[dict]:
    refs: list[dict] = []

    for token, doc_type, title in RE_MENTION_DOC.findall(text):
        t = doc_type.strip().lower()
        kind = "mention-doc-sheet" if t in ("sheet", "bitable") else (
            "mention-doc-wiki" if t == "wiki" else "mention-doc-docx"
        )
        refs.append({
            "type": kind,
            "ref": token,
            "title": title.strip(),
            "dispatch": DISPATCH[kind],
        })

    for token in RE_SHEET_TAG.findall(text):
        refs.append({
            "type": "sheet-tag",
            "ref": token,
            "title": "",
            "dispatch": DISPATCH["sheet-tag"],
        })

    for token in RE_IMAGE_TAG.findall(text):
        refs.append({
            "type": "image",
            "ref": token,
            "title": "",
            "dispatch": DISPATCH["image"],
        })

    for token, name in RE_FILE_TAG.findall(text):
        refs.append({
            "type": "file",
            "ref": token,
            "title": name.strip(),
            "dispatch": DISPATCH["file"],
        })

    for host, seg, token in RE_FEISHU_URL.findall(text):
        kind = f"url-{seg}"
        refs.append({
            "type": kind,
            "ref": f"https://{host}/{seg}/{token}",
            "title": "",
            "dispatch": DISPATCH.get(kind, "record token"),
        })

    for mid in RE_TENCENT_MEETING.findall(text):
        refs.append({
            "type": "tencent-meeting",
            "ref": f"https://meeting.tencent.com/crm/{mid}",
            "title": "",
            "dispatch": DISPATCH["tencent-meeting"],
        })

    n_tables = len(RE_LARK_TABLE.findall(text))
    if n_tables:
        # Inline content, not a link to follow — surfaced so the caller knows
        # to render it in place (pandas.read_html handles colspan/rowspan).
        refs.append({
            "type": "lark-table",
            "ref": f"(inline x{n_tables})",
            "title": "",
            "dispatch": DISPATCH["lark-table"],
        })

    # De-duplicate on (type, ref); keep first title seen.
    seen: dict[tuple[str, str], dict] = {}
    for r in refs:
        key = (r["type"], r["ref"])
        if key not in seen:
            seen[key] = r
    return list(seen.values())


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("markdown_file", help="fetched Feishu body (.md)")
    ap.add_argument("--type", help="only emit refs of this type (e.g. docx, image)")
    args = ap.parse_args()

    text = _read_text(Path(args.markdown_file))
    refs = extract(text)
    if args.type:
        refs = [r for r in refs if args.type in r["type"]]

    json.dump(refs, sys.stdout, ensure_ascii=False, indent=2)
    sys.stdout.write("\n")

    # Summary to stderr so stdout stays pure JSON for piping.
    by_type: dict[str, int] = {}
    for r in refs:
        by_type[r["type"]] = by_type.get(r["type"], 0) + 1
    if by_type:
        summary = ", ".join(f"{k}={v}" for k, v in sorted(by_type.items()))
        print(f"[feishu_extract_refs] {len(refs)} distinct refs: {summary}",
              file=sys.stderr)
    else:
        print("[feishu_extract_refs] no references found — this is a leaf doc "
              "(nothing further to recurse).", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Restore heading hierarchy and highlights lost when pandoc converts a
Feishu-exported .docx (Path B).

Feishu-exported docx does not use Word heading styles — it lays out headings
with font size + bold on normal paragraphs, and marks emphasis with run
shading (`w:shd@fill`), not `w:highlight`. pandoc therefore produces zero
Markdown headings (every heading becomes flat `**bold**`) and drops every
highlight. A text-level check ("no errors, word count matches") passes while
the document's entire structure is gone — only visual verification catches it.

This script repairs the pandoc Markdown WITHOUT retyping the body:
  * heading levels are derived from the docx's own font-size distribution
    (largest sizes -> H1..Hn, descending) and applied as `#` prefixes;
  * run shading fills are restored as Obsidian `==highlight==`.

Body text is never reconstructed — only `#` prefixes and `==` wrappers are
added to the existing pandoc lines. This keeps the API/pandoc text byte-exact
(the fidelity invariant) while giving back the structure a human sees.

Usage:
    python3 restore_docx_headings.py --docx SRC.docx --md PANDOC.md --out FINAL.md
    python3 restore_docx_headings.py --docx SRC.docx --md PANDOC.md --dry-run

`--dry-run` prints the derived size->level mapping and match counts without
writing — verify the plan before applying it (plan / validate / execute).
"""
from __future__ import annotations

import argparse
import re
import sys
from collections import Counter
from pathlib import Path

try:
    from docx import Document
    from docx.oxml.ns import qn
except ModuleNotFoundError:
    sys.exit(
        "error: python-docx is not installed.\n"
        "  run with uv:  uv run --with python-docx python3 "
        "scripts/restore_docx_headings.py ...\n"
        "  or:           pip install python-docx"
    )

# Run-shading fills that are page/background, not emphasis. Everything else
# applied at run level by Feishu is an intentional highlight. Deriving
# "highlight = any non-background run fill" from the document avoids
# hard-coding specific colors; the values verified in practice were
# ffe928 (yellow) and 935af6 (purple) — kept here only as the known examples,
# not as a closed allow-list.
_BACKGROUND_FILLS = {"auto", "ffffff", "000000", ""}
_ZERO_WIDTH = "​‌‍"


def _norm(s: str) -> str:
    """Normalize a line for cross-format text matching.

    pandoc may wrap a heading as `**text**`; the source paragraph is `text`.
    Strip emphasis/heading markers, zero-width chars, and collapse whitespace
    so the same logical line matches across the two representations.
    """
    s = s.translate({ord(c): None for c in _ZERO_WIDTH})
    s = re.sub(r"[*_#`]", "", s)
    s = re.sub(r"\s+", " ", s)
    return s.strip()


def _doc_default_pt(doc) -> float:
    """Resolve the document's default body point size.

    Critical: body paragraphs in a Feishu/pandoc docx frequently carry NO
    explicit run size — they inherit from the Normal style or docDefaults.
    If such paragraphs are bucketed as "unknown" and excluded, the modal
    size becomes a *heading* size and every real heading is demoted to body
    (verified failure). So every paragraph must get a numeric size, falling
    back to this resolved default, so the modal size is the true body size.

    Resolution order: Normal style -> docDefaults rPr sz -> 11.0pt.
    11.0pt is the de-facto Word default for the .docx era (Calibri 11); it
    is only the last resort when the file declares no default at all.
    """
    try:
        sz = doc.styles["Normal"].font.size
        if sz is not None:
            return sz.pt
    except (KeyError, AttributeError, ValueError):
        pass
    try:
        sz_el = doc.styles.element.find(
            qn("w:docDefaults") + "/" + qn("w:rPrDefault")
            + "/" + qn("w:rPr") + "/" + qn("w:sz")
        )
        if sz_el is not None:
            val = sz_el.get(qn("w:val"))
            if val:
                return int(val) / 2.0  # OOXML sz is in half-points
    except (AttributeError, ValueError, TypeError):
        pass
    return 11.0


def _para_font_pt(para, default_pt: float) -> float:
    """Effective point size of a paragraph — never None.

    Headings here have all runs at one large size. Take the max run size;
    fall back to the paragraph style's size; finally to the resolved
    document default so unsized body paragraphs land in the body bucket
    (not the 'unknown' void that corrupts the modal-size heuristic).
    """
    sizes = [r.font.size.pt for r in para.runs if r.font.size is not None]
    if sizes:
        return max(sizes)
    try:
        if para.style and para.style.font and para.style.font.size:
            return para.style.font.size.pt
    except (AttributeError, ValueError):
        pass
    return default_pt


def _run_highlight_fill(run) -> str | None:
    """Return the run's shading fill if it is an emphasis highlight, else None."""
    rpr = run._element.rPr
    if rpr is None:
        return None
    shd = rpr.find(qn("w:shd"))
    if shd is None:
        return None
    fill = (shd.get(qn("w:fill")) or "").lower()
    if fill in _BACKGROUND_FILLS:
        return None
    return fill


def build_plan(docx_path: Path):
    """Walk the docx once, returning the heading plan and highlight plan.

    heading_plan : list of (normalized_text, level, raw_text) in doc order
    highlight_plan: list of (normalized_text, [run_text, ...]) in doc order
    size_to_level: derived mapping for --dry-run reporting
    """
    try:
        doc = Document(str(docx_path))
    except Exception as exc:  # python-docx raises various errors for bad files
        sys.exit(f"error: cannot open docx ({exc}). Confirm with `file -b`; an "
                 f"exported .docx is sometimes mislabeled.")

    paras = list(doc.paragraphs)
    default_pt = _doc_default_pt(doc)

    # Every non-empty paragraph gets a numeric size (unsized -> resolved
    # default), so the modal size is the true body size. Sizes strictly
    # larger than body, descending, become H1..Hn.
    size_counts = Counter(
        round(_para_font_pt(p, default_pt), 1)
        for p in paras if p.text.strip()
    )
    if not size_counts:
        # No text paragraphs at all — nothing to restore; let the caller
        # pass the markdown through unchanged rather than abort.
        print("[restore] no text paragraphs in docx — passthrough.",
              file=sys.stderr)
        return [], [], {}, default_pt
    body_size = size_counts.most_common(1)[0][0]
    heading_sizes = sorted((s for s in size_counts if s > body_size), reverse=True)
    size_to_level = {s: i + 1 for i, s in enumerate(heading_sizes)}
    if not size_to_level:
        # One distinct size only: the doc has no font-size heading hierarchy
        # (it likely already uses Word heading styles, which doc-to-markdown
        # converts natively). Highlights may still need restoring, so warn
        # and continue rather than exit.
        print(f"[restore] no font-size hierarchy above body {body_size}pt — "
              f"doc likely uses Word heading styles already; restoring "
              f"highlights only.", file=sys.stderr)

    heading_plan, highlight_plan = [], []
    for p in paras:
        text = p.text.strip()
        if not text:
            continue
        lvl = size_to_level.get(round(_para_font_pt(p, default_pt), 1))
        if lvl:
            heading_plan.append((_norm(text), lvl, text))
        hi = [r.text for r in p.runs
              if r.text.strip() and _run_highlight_fill(r) is not None]
        if hi:
            highlight_plan.append((_norm(text), hi))

    return heading_plan, highlight_plan, size_to_level, body_size


def apply_plan(md_lines, heading_plan, highlight_plan):
    """Apply heading prefixes and highlight wrappers to the pandoc lines.

    Matching is by normalized text, in document order, with a forward-only
    cursor so repeated identical strings map to successive occurrences.
    Returns (new_lines, n_headings_applied, n_unmatched_headings,
    n_highlights_applied).
    """
    norm_lines = [_norm(l) for l in md_lines]
    out = list(md_lines)

    cursor = 0
    applied_h = unmatched_h = 0
    for ntext, level, _raw in heading_plan:
        if not ntext:
            continue
        found = -1
        for i in range(cursor, len(out)):
            if norm_lines[i] == ntext:
                found = i
                break
        if found == -1:
            unmatched_h += 1
            continue
        # Replace the whole line with a clean heading — drop pandoc's bold
        # since a heading must not also be `**...**`.
        out[found] = "#" * level + " " + ntext
        norm_lines[found] = ntext  # keep in sync for subsequent matches
        cursor = found + 1
        applied_h += 1

    cursor = 0
    applied_hl = 0
    for ntext, run_texts in highlight_plan:
        if not ntext:
            continue
        found = -1
        for i in range(cursor, len(out)):
            if norm_lines[i] == ntext:
                found = i
                break
        if found == -1:
            continue
        line = out[found]
        for rt in run_texts:
            rt = rt.strip()
            if not rt or rt not in line:
                continue
            if ("==" + rt + "==") in line:  # already wrapped
                continue
            line = line.replace(rt, "==" + rt + "==", 1)
        out[found] = line
        cursor = found + 1
        applied_hl += 1

    return out, applied_h, unmatched_h, applied_hl


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--docx", required=True, help="the owner-exported source .docx")
    ap.add_argument("--md", required=True, help="first-pass pandoc/doc-to-markdown .md")
    ap.add_argument("--out", help="output path (required unless --dry-run)")
    ap.add_argument("--dry-run", action="store_true",
                    help="print the size->level mapping and counts; do not write")
    args = ap.parse_args()

    docx_path, md_path = Path(args.docx), Path(args.md)
    if not docx_path.exists():
        sys.exit(f"error: docx not found: {docx_path}")
    if not md_path.exists():
        sys.exit(f"error: markdown not found: {md_path}")
    if not args.dry_run and not args.out:
        sys.exit("error: --out is required unless --dry-run")

    heading_plan, highlight_plan, size_to_level, body_size = build_plan(docx_path)

    print(f"[restore] body size = {body_size}pt (normal text)", file=sys.stderr)
    for sz, lvl in sorted(size_to_level.items(), key=lambda kv: -kv[0]):
        n = sum(1 for t in heading_plan if t[1] == lvl)
        print(f"[restore] {sz}pt -> H{lvl}  ({n} paragraphs)", file=sys.stderr)
    print(f"[restore] {len(highlight_plan)} paragraphs carry run highlights",
          file=sys.stderr)

    md_lines = md_path.read_text(encoding="utf-8").splitlines()
    new_lines, applied_h, unmatched_h, applied_hl = apply_plan(
        md_lines, heading_plan, highlight_plan
    )
    print(f"[restore] headings applied={applied_h} unmatched={unmatched_h}; "
          f"highlight lines applied={applied_hl}", file=sys.stderr)
    if unmatched_h:
        print(f"[restore] WARNING: {unmatched_h} heading paragraph(s) had no "
              f"matching Markdown line — inspect the source vs pandoc output "
              f"for those (often a table caption or an image-only paragraph).",
              file=sys.stderr)

    if args.dry_run:
        print("[restore] dry-run: nothing written.", file=sys.stderr)
        return

    out_path = Path(args.out)
    out_path.write_text("\n".join(new_lines) + "\n", encoding="utf-8")
    print(f"[restore] wrote {out_path}. Next: visually verify against the docx "
          f"render (qlmanage / soffice --convert-to pdf) before accepting.",
          file=sys.stderr)


if __name__ == "__main__":
    main()