Doc To Markdown

Security scan passed
Scanned at: 2026-06-13T19:44:41.778833
Tool: gitleaks + pattern-based validation
Content hash: 186ebedf7650edacd16882f4103e608a7c2b987ad545512f9853750029c65af7

#!/usr/bin/env python3
"""
Convert Windows paths to WSL format.

Usage:
    python convert_path.py "C:\\Users\\<windows-user>\\Downloads\\file.doc"

Output:
    /mnt/c/Users/<windows-user>/Downloads/file.doc
"""

import sys
import re


def convert_windows_to_wsl(windows_path: str) -> str:
    """
    Convert a Windows path to WSL format.

    Args:
        windows_path: Windows path (e.g., "C:\\Users\\<windows-user>\\file.doc")

    Returns:
        WSL path (e.g., "/mnt/c/Users/<windows-user>/file.doc")
    """
    # Remove quotes if present
    path = windows_path.strip('"').strip("'")

    # Handle drive letter (C:\ or C:/)
    drive_pattern = r'^([A-Za-z]):[\\\/]'
    match = re.match(drive_pattern, path)

    if not match:
        # Already a WSL path or relative path
        return path

    drive_letter = match.group(1).lower()
    path_without_drive = path[3:]  # Remove "C:\"

    # Replace backslashes with forward slashes
    path_without_drive = path_without_drive.replace('\\', '/')

    # Construct WSL path
    wsl_path = f"/mnt/{drive_letter}/{path_without_drive}"

    return wsl_path


def main():
    if len(sys.argv) < 2:
        print("Usage: python convert_path.py <windows_path>")
        print('Example: python convert_path.py "C:\\Users\\<windows-user>\\Downloads\\file.doc"')
        sys.exit(1)

    windows_path = sys.argv[1]
    wsl_path = convert_windows_to_wsl(windows_path)
    print(wsl_path)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Multi-tool document to markdown converter with intelligent orchestration.

Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge).
DOCX files get automatic post-processing to fix pandoc artifacts.

Usage:
    # Quick Mode (default) - fast, single best tool
    uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md

    # Heavy Mode - multi-tool parallel execution with merge
    uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md --heavy

    # DOCX deep mode - python-docx direct parsing (experimental)
    uv run --with python-docx scripts/convert.py document.docx -o output.md --docx-deep

    # With image extraction
    uv run --with pymupdf4llm scripts/convert.py document.pdf -o output.md --assets-dir ./images

Dependencies:
    - pymupdf4llm: PDF conversion (LLM-optimized)
    - markitdown: PDF/DOCX/PPTX conversion
    - pandoc: DOCX/PPTX conversion (system install: brew install pandoc)
    - python-docx: DOCX deep parsing (optional, for --docx-deep)
"""

import argparse
import json
import re
import subprocess
import sys
import shutil
import zipfile
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional


@dataclass
class ConversionResult:
    """Result from a single tool conversion."""
    markdown: str
    tool: str
    images: list[str] = field(default_factory=list)
    success: bool = True
    error: str = ""


# ── Post-processing stats ────────────────────────────────────────────────────

@dataclass
class PostProcessStats:
    """Track what the DOCX post-processor fixed."""
    grid_tables_converted: int = 0
    image_paths_fixed: int = 0
    attributes_removed: int = 0
    code_blocks_fixed: int = 0
    escaped_brackets_fixed: int = 0
    double_brackets_fixed: int = 0

    def any_fixes(self) -> bool:
        return any(
            getattr(self, f) > 0
            for f in self.__dataclass_fields__
        )

    def summary(self) -> str:
        parts = []
        if self.grid_tables_converted:
            parts.append(f"grid tables: {self.grid_tables_converted}")
        if self.image_paths_fixed:
            parts.append(f"image paths: {self.image_paths_fixed}")
        if self.attributes_removed:
            parts.append(f"attributes: {self.attributes_removed}")
        if self.code_blocks_fixed:
            parts.append(f"code blocks: {self.code_blocks_fixed}")
        if self.escaped_brackets_fixed:
            parts.append(f"escaped brackets: {self.escaped_brackets_fixed}")
        if self.double_brackets_fixed:
            parts.append(f"double brackets: {self.double_brackets_fixed}")
        return ", ".join(parts) if parts else "no fixes needed"


# ── DOCX post-processing ─────────────────────────────────────────────────────

# Regex patterns compiled once
_RE_GRID_BORDER = re.compile(r"^\+[:=-][-:=]+(?:\+[:=-][-:=]+)*\+$")
_RE_GRID_ROW = re.compile(r"^\|(.+)\|$")
_RE_NESTED_GRID_BORDER = re.compile(r"^\|\s*\+[:=-][-:=]+\+\s*\|$")
_RE_PANDOC_ATTR = re.compile(r"\{[^}]*(?:width|height)\s*=\s*\"[^\"]*\"[^}]*\}")
_RE_PANDOC_CLASS = re.compile(r"\{\.(?:underline|mark)\}")
_RE_DOUBLE_BRACKET_LINK = re.compile(r"\[\[([^\]]+)\]\(([^)]+)\)")
_RE_DOUBLE_BRACKET_CLOSED = re.compile(r"\[\[([^\]]+)\]\]\(([^)]+)\)")
_RE_DOUBLE_BRACKET_ATTR_LINK = re.compile(r"\[\[([^\]]+)\]\{[^}]*\}\]\(([^)]+)\)")
_RE_ESCAPED_BRACKET = re.compile(r"\\(\[|])")
# Matches single-column dashed line: "  ------"
# AND multi-column simple table border: "  ---- -----"
_RE_DASHED_LINE = re.compile(r"^(\s{2,})-{3,}[\s-]*$")
_RE_ESCAPED_QUOTE = re.compile(r'\\"')
# CJK + fullwidth punctuation range for bold spacing checks
_RE_CJK_PUNCT = re.compile(r'[\u4e00-\u9fff\u3000-\u303f\uff01-\uffef，。、；：！？（）【】「」《》""'']')
_RE_BOLD_PAIR = re.compile(r'\*\*(.+?)\*\*')


def _is_grid_border(line: str) -> bool:
    """Check if a line is a grid table border like +---+ or +:---+."""
    stripped = line.strip()
    return bool(_RE_GRID_BORDER.match(stripped))


def _is_nested_grid_border(line: str) -> bool:
    """Check if a line is a nested grid border like | +---+ |."""
    stripped = line.strip()
    return bool(_RE_NESTED_GRID_BORDER.match(stripped))


def _count_grid_columns(border_line: str) -> int:
    """Count columns in a grid table border line."""
    stripped = border_line.strip()
    if not stripped.startswith("+"):
        return 0
    # Count + separators minus 1 = number of columns
    return stripped.count("+") - 1



# Languages recognized as code block hints in pandoc dashed-line blocks
_KNOWN_CODE_LANGS = frozenset({
    "json", "bash", "shell", "python", "javascript", "js",
    "html", "css", "yaml", "xml", "sql", "plain text",
    "text", "plaintext", "typescript", "ts", "go", "rust",
    "java", "c", "cpp", "ruby", "php",
})


def _build_pipe_table(rows: list[list[str]]) -> list[str]:
    """Build a standard markdown pipe table from rows of cells."""
    if not rows:
        return []
    col_count = max(len(r) for r in rows)
    lines = [
        "| " + " | ".join([""] * col_count) + " |",
        "| " + " | ".join(["---"] * col_count) + " |",
    ]
    for row in rows:
        padded = row + [""] * (col_count - len(row))
        lines.append("| " + " | ".join(padded) + " |")
    return lines


def _collect_images(directory: Path) -> list[str]:
    """Collect image files from a directory (single glob pass)."""
    if not directory.exists():
        return []
    image_exts = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
    return sorted(
        str(p) for p in directory.rglob("*")
        if p.suffix.lower() in image_exts
    )


def _convert_grid_tables(text: str, stats: PostProcessStats) -> str:
    """Convert pandoc grid tables to standard markdown.

    Single-column grid tables (info boxes) -> blockquotes.
    Multi-column grid tables (side-by-side images) -> split into individual elements.
    Nested grid tables are flattened.
    """
    lines = text.split("\n")
    result = []
    i = 0

    while i < len(lines):
        line = lines[i]

        # Detect grid table start
        if _is_grid_border(line):
            # Collect the entire grid table
            table_lines = [line]
            i += 1
            while i < len(lines):
                table_lines.append(lines[i])
                if _is_grid_border(lines[i]) and len(table_lines) > 1:
                    i += 1
                    break
                i += 1
            else:
                # Reached end of file without closing border
                # Just output as-is
                result.extend(table_lines)
                continue

            stats.grid_tables_converted += 1
            num_cols = _count_grid_columns(table_lines[0])

            # Extract content lines (skip borders)
            content_lines = []
            for tl in table_lines:
                if _is_grid_border(tl) or _is_nested_grid_border(tl):
                    continue
                m = _RE_GRID_ROW.match(tl.strip())
                if m:
                    content_lines.append(m.group(1).strip())
                else:
                    # Non-standard line inside grid, keep content
                    stripped = tl.strip()
                    if stripped and stripped != "|":
                        content_lines.append(stripped)

            if num_cols <= 1:
                # Single column -> blockquote
                result.append("")
                for cl in content_lines:
                    # Strip outer pipes if present from nested grids
                    cleaned = cl.strip()
                    if cleaned.startswith("|") and cleaned.endswith("|"):
                        cleaned = cleaned[1:-1].strip()
                    # Skip nested grid borders
                    if _RE_GRID_BORDER.match(cleaned):
                        continue
                    if cleaned:
                        result.append(f"> {cleaned}")
                    else:
                        result.append(">")
                result.append("")
            else:
                # Multi-column -> convert to standard pipe table
                # Parse rows: each content_line is a row, split by | into cells
                table_rows = []
                for cl in content_lines:
                    cells = [c.strip() for c in cl.split("|") if c.strip() and not _RE_GRID_BORDER.match(c.strip())]
                    if cells:
                        table_rows.append(cells)

                if table_rows:
                    result.append("")
                    result.extend(_build_pipe_table(table_rows))
                    result.append("")
        else:
            result.append(line)
            i += 1

    return "\n".join(result)


def _fix_image_paths(text: str, assets_dir: Optional[Path], stats: PostProcessStats) -> str:
    """Fix pandoc's double media path and verify images exist.

    Pandoc extracts to <assets_dir>/media/<files> but references as
    <assets_dir>/media/media/<files>. Fix the references.
    Also flatten the actual directory if needed.
    """
    def fix_path(m: re.Match) -> str:
        alt = m.group(1)
        path = m.group(2)
        new_path = path

        # Fix double media/ path
        if "media/media/" in path:
            new_path = path.replace("media/media/", "media/")
            stats.image_paths_fixed += 1

        return f"![{alt}]({new_path})"

    text = re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", fix_path, text)

    # Flatten double media/ nesting if present (pandoc artifact)
    if assets_dir:
        double_media = assets_dir / "media" / "media"
        single_media = assets_dir / "media"
        try:
            for f in double_media.iterdir():
                dest = single_media / f.name
                if not dest.exists():
                    shutil.move(str(f), str(dest))
            double_media.rmdir()
        except (FileNotFoundError, OSError):
                pass

    return text


def _clean_pandoc_attributes(text: str, stats: PostProcessStats) -> str:
    """Remove pandoc attribute annotations from markdown.

    Removes: {width="..." height="..."}, {.underline}, {.mark}, etc.
    """
    count_before = len(text)

    # Remove width/height attributes on images
    text = _RE_PANDOC_ATTR.sub("", text)

    # Remove class attributes like {.underline}
    text = _RE_PANDOC_CLASS.sub("", text)

    if len(text) != count_before:
        # Rough count of removals
        stats.attributes_removed = count_before - len(text)

    return text


def _is_code_content(lines: list[str]) -> bool:
    """Heuristic: decide if content between dashed lines is code or a note/callout.

    Code indicators:
    - Has a language hint on the first line
    - Contains JSON/code-like syntax ({, }, =, ;, ->, //)
    - Contains URLs with protocols
    - Has backslash line continuations

    Note indicators:
    - Mostly CJK/prose text without code syntax
    - Short single-line content
    """
    text = "\n".join(lines)
    stripped = text.strip()

    if not stripped:
        return False

    # Code syntax indicators
    code_chars = set('{}[]();=<>/\\')
    code_char_count = sum(1 for c in stripped if c in code_chars)

    # If >5% of content is code syntax characters, treat as code
    if len(stripped) > 0 and code_char_count / len(stripped) > 0.05:
        return True

    # JSON-like structure
    if stripped.startswith("{") or stripped.startswith("["):
        return True

    # Command-like (starts with common command patterns)
    first_line = lines[0].strip() if lines else ""
    if re.match(r"^(curl|wget|npm|pip|brew|apt|docker|git|ssh|cd|ls|cat|echo|python|node|uv)\s", first_line):
        return True

    return False


def _fix_code_blocks(text: str, stats: PostProcessStats) -> str:
    """Convert pandoc's indented dashed-line blocks to fenced code blocks or blockquotes.

    Pandoc wraps both code and notes in:
      ------------------------------------------------------------------
      content here

      ------------------------------------------------------------------

    With language hint -> code block:
      ```json
      content here
      ```

    Without language hint + prose content -> blockquote:
      > content here

    Without language hint + code-like content -> code block:
      ```
      content here
      ```
    """
    lines = text.split("\n")
    result = []
    i = 0

    known_langs = _KNOWN_CODE_LANGS

    while i < len(lines):
        line = lines[i]

        # Detect indented dashed line (2+ leading spaces, 3+ dashes)
        if _RE_DASHED_LINE.match(line):
            # Check if this is a pandoc simple table (multiple dashed columns
            # on the same line, or content between dashes contains images)
            # Simple table pattern: "  ----  ----" (multiple dash groups separated by spaces)
            # Gap can be 1+ spaces (pandoc uses varying gaps)
            dash_parts = [p for p in line.strip().split() if p.strip()]
            is_simple_table_border = len(dash_parts) > 1 and all(
                re.match(r"^-+$", p.strip()) for p in dash_parts
            )

            if is_simple_table_border:
                # This is a pandoc simple table border - collect rows until
                # next simple table border, convert to pipe table
                table_rows = []
                j = i + 1
                while j < len(lines):
                    next_line = lines[j]
                    # Check for closing simple table border
                    next_parts = [p for p in next_line.strip().split() if p.strip()]
                    is_next_border = len(next_parts) > 1 and all(
                        re.match(r"^-+$", p.strip()) for p in next_parts
                    )
                    if is_next_border:
                        j += 1
                        break
                    if next_line.strip():
                        # Split by 2+ spaces to get columns (pandoc uses varying gaps)
                        cells = [c.strip() for c in re.split(r"\s{2,}", next_line.strip()) if c.strip()]
                        if cells:
                            table_rows.append(cells)
                    j += 1

                if table_rows:
                    stats.code_blocks_fixed += 1
                    result.append("")
                    result.extend(_build_pipe_table(table_rows))
                    result.append("")

                i = j
                continue

            # Not a simple table - look for content and closing dashed line
            block_content = []
            lang_hint = ""
            j = i + 1

            while j < len(lines):
                next_line = lines[j]

                if _RE_DASHED_LINE.match(next_line):
                    # Found closing dashed line
                    j += 1
                    break

                block_content.append(next_line)
                j += 1
            else:
                # No closing dashed line found - not a block, keep as-is
                result.append(line)
                i += 1
                continue

            # If content contains images, treat as simple table (single-column)
            has_images = any("![" in cl for cl in block_content)
            if has_images:
                result.append("")
                for cl in block_content:
                    cl = cl.strip()
                    if cl:
                        result.append(cl)
                result.append("")
                i = j
                continue

            # Check if first line is a language hint (e.g., "  JSON\", "  Plain Text\")
            has_lang_hint = False
            if block_content:
                first = block_content[0].strip()
                first_clean = first.rstrip("\\").strip()
                if first_clean.lower() in known_langs:
                    lang_hint = first_clean.lower()
                    if lang_hint in ("plain text", "text", "plaintext"):
                        lang_hint = ""  # No language tag for plain text
                    has_lang_hint = True
                    block_content = block_content[1:]

            # Clean content: remove leading 2-space indent, fix escaped quotes
            cleaned = []
            for cl in block_content:
                if cl.startswith("  "):
                    cl = cl[2:]
                cl = cl.replace('\\"', '"')
                if cl.endswith("\\"):
                    cl = cl[:-1]
                cleaned.append(cl)

            # Remove trailing/leading empty lines
            while cleaned and not cleaned[-1].strip():
                cleaned.pop()
            while cleaned and not cleaned[0].strip():
                cleaned.pop(0)

            if cleaned:
                stats.code_blocks_fixed += 1

                # Decide: code block vs blockquote
                if has_lang_hint or _is_code_content(cleaned):
                    # Code block — try to pretty-print JSON
                    code_lines = cleaned
                    if lang_hint == "json":
                        try:
                            raw = "\n".join(cleaned)
                            parsed = json.loads(raw)
                            code_lines = json.dumps(parsed, indent=2, ensure_ascii=False).split("\n")
                        except (json.JSONDecodeError, ValueError):
                            pass  # Keep original if not valid JSON

                    result.append("")
                    result.append(f"```{lang_hint}")
                    result.extend(code_lines)
                    result.append("```")
                    result.append("")
                else:
                    # Note/callout -> blockquote
                    result.append("")
                    for cl in cleaned:
                        if cl.strip():
                            result.append(f"> {cl}")
                        else:
                            result.append(">")
                    result.append("")

            i = j
        else:
            result.append(line)
            i += 1

    return "\n".join(result)


def _fix_escaped_brackets(text: str, stats: PostProcessStats) -> str:
    r"""Fix pandoc's escaped brackets: \[ -> [, \] -> ]."""
    count = len(_RE_ESCAPED_BRACKET.findall(text))
    if count:
        stats.escaped_brackets_fixed = count
        text = _RE_ESCAPED_BRACKET.sub(r"\1", text)
    return text


def _fix_double_bracket_links(text: str, stats: PostProcessStats) -> str:
    """Fix double-bracket links: [[text]{.underline}](url) -> [text](url)."""
    count = 0

    def _replace_link(m: re.Match) -> str:
        nonlocal count
        count += 1
        return f"[{m.group(1)}]({m.group(2)})"

    text = _RE_DOUBLE_BRACKET_ATTR_LINK.sub(_replace_link, text)
    text = _RE_DOUBLE_BRACKET_CLOSED.sub(_replace_link, text)
    text = _RE_DOUBLE_BRACKET_LINK.sub(_replace_link, text)

    stats.double_brackets_fixed = count
    return text


def _fix_cjk_bold_spacing(text: str) -> str:
    """Add space around **bold** spans that contain CJK characters.

    DOCX uses run-level styling for bold — no spaces between runs in CJK text.
    Markdown renderers need whitespace around ** to recognize bold boundaries.

    Rule: if a **content** span contains any CJK character, ensure both sides
    have a space (unless already spaced or at line boundary). This handles:
    - CJK directly touching **: 打开**飞书** → 打开 **飞书**
    - Emoji touching **: **密码】**➡️ → **密码】** ➡️
    - Already spaced: 已有 **粗体** → unchanged
    - English bold: English **bold** text → unchanged
    """
    result = []
    last_end = 0

    for m in _RE_BOLD_PAIR.finditer(text):
        start, end = m.start(), m.end()
        content = m.group(1)

        result.append(text[last_end:start])

        # Only add spaces for bold spans containing CJK
        if _RE_CJK_PUNCT.search(content):
            # Space before ** if previous char is not whitespace
            if start > 0 and text[start - 1] not in (' ', '\t', '\n'):
                result.append(' ')

            result.append(m.group(0))

            # Space after ** if next char is not whitespace
            if end < len(text) and text[end] not in (' ', '\t', '\n'):
                result.append(' ')
        else:
            result.append(m.group(0))

        last_end = end

    result.append(text[last_end:])
    return ''.join(result)


def _cleanup_excessive_blank_lines(text: str) -> str:
    """Collapse 3+ consecutive blank lines to 2."""
    return re.sub(r"\n{4,}", "\n\n\n", text)


def postprocess_docx_markdown(
    text: str,
    assets_dir: Optional[Path] = None,
) -> tuple[str, PostProcessStats]:
    """Apply all DOCX-specific post-processing to pandoc markdown output.

    Returns (cleaned_text, stats).
    """
    stats = PostProcessStats()

    # Order matters: grid tables first (they contain images with attributes)
    text = _convert_grid_tables(text, stats)
    text = _fix_image_paths(text, assets_dir, stats)
    text = _clean_pandoc_attributes(text, stats)
    text = _fix_code_blocks(text, stats)
    text = _fix_double_bracket_links(text, stats)
    text = _fix_escaped_brackets(text, stats)
    text = _fix_cjk_bold_spacing(text)
    text = _cleanup_excessive_blank_lines(text)

    return text, stats


# ── DOCX deep parsing (python-docx) ──────────────────────────────────────────

def convert_with_docx_deep(
    file_path: Path, assets_dir: Optional[Path] = None
) -> ConversionResult:
    """Convert DOCX using python-docx direct parsing (experimental).

    More precise than pandoc for:
    - Table structure preservation
    - Comment extraction
    - Image extraction with position info
    """
    try:
        from docx import Document
        from docx.opc.constants import RELATIONSHIP_TYPE as RT
    except ImportError:
        return ConversionResult(
            markdown="",
            tool="docx-deep",
            success=False,
            error="python-docx not installed. Run: pip install python-docx",
        )

    try:
        doc = Document(str(file_path))
        md_parts = []
        images = []
        image_counter = 0

        # Extract images from docx zip
        if assets_dir:
            assets_dir.mkdir(parents=True, exist_ok=True)
            media_dir = assets_dir / "media"
            media_dir.mkdir(exist_ok=True)

            with zipfile.ZipFile(str(file_path), "r") as zf:
                for name in zf.namelist():
                    if name.startswith("word/media/"):
                        img_name = Path(name).name
                        img_dest = media_dir / img_name
                        with zf.open(name) as src, open(img_dest, "wb") as dst:
                            dst.write(src.read())
                        images.append(str(img_dest))

        # Process paragraphs
        for para in doc.paragraphs:
            style_name = para.style.name if para.style else ""
            text = para.text.strip()

            if not text:
                md_parts.append("")
                continue

            # Headings
            if style_name.startswith("Heading"):
                try:
                    level = int(style_name.split()[-1])
                except (ValueError, IndexError):
                    level = 1
                md_parts.append(f"{'#' * level} {text}")
                md_parts.append("")
                continue

            # Check for bold-only paragraphs (often sub-headings in Chinese docs)
            all_bold = all(run.bold for run in para.runs if run.text.strip())
            if all_bold and para.runs and len(text) < 100:
                md_parts.append(f"**{text}**")
                md_parts.append("")
                continue

            # Regular paragraph
            md_parts.append(text)
            md_parts.append("")

        # Process tables
        for table in doc.tables:
            md_parts.append("")
            rows = table.rows
            if not rows:
                continue

            # Header row
            header_cells = [cell.text.strip() for cell in rows[0].cells]
            md_parts.append("| " + " | ".join(header_cells) + " |")
            md_parts.append("| " + " | ".join(["---"] * len(header_cells)) + " |")

            # Data rows
            for row in rows[1:]:
                cells = [cell.text.strip() for cell in row.cells]
                md_parts.append("| " + " | ".join(cells) + " |")
            md_parts.append("")

        markdown = "\n".join(md_parts)

        return ConversionResult(
            markdown=markdown,
            tool="docx-deep",
            images=images,
            success=True,
        )
    except Exception as e:
        return ConversionResult(
            markdown="", tool="docx-deep", success=False, error=str(e)
        )


# ── Existing tool converters ─────────────────────────────────────────────────

def check_tool_available(tool: str) -> bool:
    """Check if a conversion tool is available."""
    if tool == "pymupdf4llm":
        try:
            import pymupdf4llm
            return True
        except ImportError:
            return False
    elif tool == "markitdown":
        try:
            import markitdown
            return True
        except ImportError:
            return False
    elif tool == "pandoc":
        return shutil.which("pandoc") is not None
    elif tool == "docx-deep":
        try:
            from docx import Document
            return True
        except ImportError:
            return False
    return False


def select_tools(file_path: Path, mode: str) -> list[str]:
    """Select conversion tools based on file type and mode."""
    ext = file_path.suffix.lower()

    # Tool preferences by format
    tool_map = {
        ".pdf": {
            "quick": ["pymupdf4llm", "markitdown"],  # fallback order
            "heavy": ["pymupdf4llm", "markitdown"],
        },
        ".docx": {
            "quick": ["pandoc", "markitdown"],
            "heavy": ["pandoc", "markitdown"],
        },
        ".doc": {
            "quick": ["pandoc", "markitdown"],
            "heavy": ["pandoc", "markitdown"],
        },
        ".pptx": {
            "quick": ["markitdown", "pandoc"],
            "heavy": ["markitdown", "pandoc"],
        },
        ".xlsx": {
            "quick": ["markitdown"],
            "heavy": ["markitdown"],
        },
    }

    tools = tool_map.get(ext, {"quick": ["markitdown"], "heavy": ["markitdown"]})

    if mode == "quick":
        # Return first available tool
        for tool in tools["quick"]:
            if check_tool_available(tool):
                return [tool]
        return []
    else:  # heavy
        # Return all available tools
        return [t for t in tools["heavy"] if check_tool_available(t)]


def convert_with_pymupdf4llm(
    file_path: Path, assets_dir: Optional[Path] = None
) -> ConversionResult:
    """Convert using PyMuPDF4LLM (best for PDFs)."""
    try:
        import pymupdf4llm

        kwargs = {}
        images = []

        if assets_dir:
            assets_dir.mkdir(parents=True, exist_ok=True)
            kwargs["write_images"] = True
            kwargs["image_path"] = str(assets_dir)
            kwargs["dpi"] = 150

        # Use best table detection strategy
        kwargs["table_strategy"] = "lines_strict"

        md_text = pymupdf4llm.to_markdown(str(file_path), **kwargs)

        if assets_dir:
            images = _collect_images(assets_dir)

        return ConversionResult(
            markdown=md_text, tool="pymupdf4llm", images=images, success=True
        )
    except Exception as e:
        return ConversionResult(
            markdown="", tool="pymupdf4llm", success=False, error=str(e)
        )


def convert_with_markitdown(
    file_path: Path, assets_dir: Optional[Path] = None
) -> ConversionResult:
    """Convert using markitdown."""
    try:
        # markitdown CLI approach
        result = subprocess.run(
            ["markitdown", str(file_path)],
            capture_output=True,
            text=True,
            timeout=120,
        )

        if result.returncode != 0:
            return ConversionResult(
                markdown="",
                tool="markitdown",
                success=False,
                error=result.stderr,
            )

        return ConversionResult(
            markdown=result.stdout, tool="markitdown", success=True
        )
    except FileNotFoundError:
        # Try Python API
        try:
            from markitdown import MarkItDown

            md = MarkItDown()
            result = md.convert(str(file_path))
            return ConversionResult(
                markdown=result.text_content, tool="markitdown", success=True
            )
        except Exception as e:
            return ConversionResult(
                markdown="", tool="markitdown", success=False, error=str(e)
            )
    except Exception as e:
        return ConversionResult(
            markdown="", tool="markitdown", success=False, error=str(e)
        )


def convert_with_pandoc(
    file_path: Path, assets_dir: Optional[Path] = None
) -> ConversionResult:
    """Convert using pandoc.

    Pandoc's --extract-media=DIR creates a media/ subdirectory inside DIR.
    We point --extract-media at assets_dir's parent so pandoc's media/
    subdirectory lands exactly at assets_dir (when assets_dir ends with 'media'),
    or we use a temp dir and move files afterward.
    """
    try:
        cmd = ["pandoc", str(file_path), "-t", "markdown", "--wrap=none"]

        extract_dir = None
        if assets_dir:
            assets_dir.mkdir(parents=True, exist_ok=True)
            # Pandoc always creates a media/ subdirectory inside --extract-media.
            # Point it at the parent so media/ lands at assets_dir.
            if assets_dir.name == "media":
                extract_dir = assets_dir.parent
            else:
                extract_dir = assets_dir
            cmd.extend(["--extract-media", str(extract_dir)])

        result = subprocess.run(
            cmd, capture_output=True, text=True, timeout=120
        )

        if result.returncode != 0:
            return ConversionResult(
                markdown="", tool="pandoc", success=False, error=result.stderr
            )

        md = result.stdout

        # Convert absolute image paths to relative paths based on output location
        if extract_dir:
            abs_media = str(extract_dir / "media")
            # Replace absolute paths with relative 'media/' prefix
            md = md.replace(abs_media + "/", "media/")

        images = _collect_images(assets_dir) if assets_dir else []

        return ConversionResult(
            markdown=md, tool="pandoc", images=images, success=True
        )
    except Exception as e:
        return ConversionResult(
            markdown="", tool="pandoc", success=False, error=str(e)
        )


def convert_single(
    file_path: Path, tool: str, assets_dir: Optional[Path] = None
) -> ConversionResult:
    """Run a single conversion tool."""
    converters = {
        "pymupdf4llm": convert_with_pymupdf4llm,
        "markitdown": convert_with_markitdown,
        "pandoc": convert_with_pandoc,
        "docx-deep": convert_with_docx_deep,
    }

    converter = converters.get(tool)
    if not converter:
        return ConversionResult(
            markdown="", tool=tool, success=False, error=f"Unknown tool: {tool}"
        )

    return converter(file_path, assets_dir)


def merge_results(results: list[ConversionResult]) -> ConversionResult:
    """Merge results from multiple tools, selecting best segments."""
    if not results:
        return ConversionResult(markdown="", tool="none", success=False)

    # Filter successful results
    successful = [r for r in results if r.success and r.markdown.strip()]
    if not successful:
        # Return first error
        return results[0] if results else ConversionResult(
            markdown="", tool="none", success=False
        )

    if len(successful) == 1:
        return successful[0]

    # Multiple successful results - merge them
    # Strategy: Compare key metrics and select best
    best = successful[0]
    best_score = score_markdown(best.markdown)

    for result in successful[1:]:
        score = score_markdown(result.markdown)
        if score > best_score:
            best = result
            best_score = score

    # Merge images from all results
    all_images = []
    seen = set()
    for result in successful:
        for img in result.images:
            if img not in seen:
                all_images.append(img)
                seen.add(img)

    best.images = all_images
    best.tool = f"merged({','.join(r.tool for r in successful)})"

    return best


def score_markdown(md: str) -> float:
    """Score markdown quality for comparison."""
    score = 0.0

    # Length (more content is generally better)
    score += min(len(md) / 10000, 5.0)  # Cap at 5 points

    # Tables (proper markdown tables)
    table_count = md.count("|---|") + md.count("| ---")
    score += min(table_count * 0.5, 3.0)

    # Images (referenced images)
    image_count = md.count("![")
    score += min(image_count * 0.3, 2.0)

    # Headings (proper hierarchy)
    h1_count = md.count("\n# ")
    h2_count = md.count("\n## ")
    h3_count = md.count("\n### ")
    if h1_count > 0 and h2_count >= h1_count:
        score += 1.0  # Good hierarchy

    # Lists (structured content)
    list_count = md.count("\n- ") + md.count("\n* ") + md.count("\n1. ")
    score += min(list_count * 0.1, 2.0)

    # Penalize pandoc artifacts (grid tables, attributes)
    artifact_count = md.count("+:---") + md.count("+---+")
    artifact_count += md.count('{width="') + md.count("{.underline}")
    score -= artifact_count * 0.5

    return score


def main():
    parser = argparse.ArgumentParser(
        description="Convert documents to markdown with multi-tool orchestration",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    # Quick mode (default)
    python convert.py document.pdf -o output.md

    # Heavy mode (best quality)
    python convert.py document.pdf -o output.md --heavy

    # DOCX deep mode (python-docx parsing)
    python convert.py document.docx -o output.md --docx-deep

    # With custom assets directory
    python convert.py document.pdf -o output.md --assets-dir ./images
        """,
    )
    parser.add_argument("input", type=Path, nargs="?", help="Input document path")
    parser.add_argument(
        "-o", "--output", type=Path, help="Output markdown file"
    )
    parser.add_argument(
        "--heavy",
        action="store_true",
        help="Enable Heavy Mode (multi-tool, best quality)",
    )
    parser.add_argument(
        "--docx-deep",
        action="store_true",
        help="Use python-docx direct parsing (experimental, DOCX only)",
    )
    parser.add_argument(
        "--no-postprocess",
        action="store_true",
        help="Disable DOCX post-processing (keep raw pandoc output)",
    )
    parser.add_argument(
        "--assets-dir",
        type=Path,
        default=None,
        help="Directory for extracted images (default: <output>_assets/)",
    )
    parser.add_argument(
        "--tool",
        choices=["pymupdf4llm", "markitdown", "pandoc", "docx-deep"],
        help="Force specific tool (overrides auto-selection)",
    )
    parser.add_argument(
        "--list-tools",
        action="store_true",
        help="List available tools and exit",
    )

    args = parser.parse_args()

    # List tools mode
    if args.list_tools:
        tools = ["pymupdf4llm", "markitdown", "pandoc", "docx-deep"]
        print("Available conversion tools:")
        for tool in tools:
            status = "+" if check_tool_available(tool) else "-"
            print(f"  {status} {tool}")
        sys.exit(0)

    # Validate input
    if args.input is None:
        parser.error("the following arguments are required: input")
    if not args.input.exists():
        print(f"Error: Input file not found: {args.input}", file=sys.stderr)
        sys.exit(1)

    # Determine output path
    output_path = args.output or args.input.with_suffix(".md")

    # Determine assets directory
    assets_dir = args.assets_dir
    if assets_dir is None:
        assets_dir = output_path.parent / f"{output_path.stem}_assets"

    is_docx = args.input.suffix.lower() in (".docx", ".doc")

    # Handle --docx-deep mode
    if args.docx_deep:
        if not is_docx:
            print("Error: --docx-deep only works with DOCX files.", file=sys.stderr)
            sys.exit(1)
        tools = ["docx-deep"]
    elif args.tool:
        tools = [args.tool] if check_tool_available(args.tool) else []
    else:
        # Select tools
        mode = "heavy" if args.heavy else "quick"
        tools = select_tools(args.input, mode)

    mode = "docx-deep" if args.docx_deep else ("heavy" if args.heavy else "quick")

    if not tools:
        print("Error: No conversion tools available.", file=sys.stderr)
        print("Install with:", file=sys.stderr)
        print("  pip install pymupdf4llm", file=sys.stderr)
        print("  uv tool install markitdown[pdf]", file=sys.stderr)
        print("  brew install pandoc", file=sys.stderr)
        sys.exit(1)

    print(f"Converting: {args.input}")
    print(f"Mode: {mode.upper()}")
    print(f"Tools: {', '.join(tools)}")

    # Run conversions
    results = []
    for tool in tools:
        print(f"  Running {tool}...", end=" ", flush=True)

        # Use separate assets dirs for each tool in heavy mode
        tool_assets = None
        if assets_dir and mode == "heavy" and len(tools) > 1:
            tool_assets = assets_dir / tool
        elif assets_dir:
            tool_assets = assets_dir

        result = convert_single(args.input, tool, tool_assets)
        results.append(result)

        if result.success:
            print(f"ok ({len(result.markdown):,} chars, {len(result.images)} images)")
        else:
            print(f"FAIL ({result.error[:50]}...)")

    # Merge results if heavy mode
    if mode == "heavy" and len(results) > 1:
        print("  Merging results...", end=" ", flush=True)
        final = merge_results(results)
        print(f"ok (using {final.tool})")
    else:
        final = merge_results(results)

    if not final.success:
        print(f"Error: Conversion failed: {final.error}", file=sys.stderr)
        sys.exit(1)

    # Apply DOCX post-processing
    if is_docx and not args.no_postprocess and "pandoc" in final.tool:
        print("  Post-processing DOCX output...", end=" ", flush=True)
        final.markdown, pp_stats = postprocess_docx_markdown(
            final.markdown, assets_dir
        )
        print(f"ok ({pp_stats.summary()})")

    # Write output
    output_path.parent.mkdir(parents=True, exist_ok=True)
    output_path.write_text(final.markdown)

    print(f"\nOutput: {output_path}")
    print(f"  Size: {len(final.markdown):,} characters")
    if final.images:
        print(f"  Images: {len(final.images)} extracted")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Extract images from PDF files with metadata using PyMuPDF.

Features:
- Extracts all images with page and position metadata
- Generates JSON metadata file for each image
- Supports markdown reference generation
- Optional DPI control for quality

Usage:
    uv run --with pymupdf scripts/extract_pdf_images.py document.pdf
    uv run --with pymupdf scripts/extract_pdf_images.py document.pdf -o ./images
    uv run --with pymupdf scripts/extract_pdf_images.py document.pdf --markdown refs.md

Examples:
    # Basic extraction
    uv run --with pymupdf scripts/extract_pdf_images.py document.pdf

    # With custom output and markdown references
    uv run --with pymupdf scripts/extract_pdf_images.py doc.pdf -o assets --markdown images.md
"""

import argparse
import json
import sys
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Optional


@dataclass
class ImageMetadata:
    """Metadata for an extracted image."""
    filename: str
    page: int  # 1-indexed
    index: int  # Image index on page (1-indexed)
    width: int  # Original width in pixels
    height: int  # Original height in pixels
    x: float  # X position on page (points)
    y: float  # Y position on page (points)
    bbox_width: float  # Width on page (points)
    bbox_height: float  # Height on page (points)
    size_bytes: int
    format: str  # png, jpg, etc.
    colorspace: str  # RGB, CMYK, Gray
    bits_per_component: int


def extract_images(
    pdf_path: Path,
    output_dir: Path,
    markdown_file: Optional[Path] = None
) -> list[ImageMetadata]:
    """
    Extract all images from a PDF file with metadata.

    Args:
        pdf_path: Path to the PDF file
        output_dir: Directory to save extracted images
        markdown_file: Optional path to write markdown references

    Returns:
        List of ImageMetadata for each extracted image
    """
    try:
        import fitz  # PyMuPDF
    except ImportError:
        print("Error: PyMuPDF not installed. Run with:")
        print('  uv run --with pymupdf scripts/extract_pdf_images.py <pdf_path>')
        sys.exit(1)

    output_dir.mkdir(parents=True, exist_ok=True)

    doc = fitz.open(str(pdf_path))
    extracted: list[ImageMetadata] = []
    markdown_refs: list[str] = []

    for page_num in range(len(doc)):
        page = doc[page_num]
        image_list = page.get_images(full=True)

        for img_index, img_info in enumerate(image_list):
            xref = img_info[0]

            try:
                base_image = doc.extract_image(xref)
            except Exception as e:
                print(f"  Warning: Could not extract image xref={xref}: {e}")
                continue

            image_bytes = base_image["image"]
            image_ext = base_image["ext"]
            width = base_image.get("width", 0)
            height = base_image.get("height", 0)
            colorspace = base_image.get("colorspace", 0)
            bpc = base_image.get("bpc", 8)

            # Map colorspace number to name
            cs_names = {1: "Gray", 3: "RGB", 4: "CMYK"}
            cs_name = cs_names.get(colorspace, f"Unknown({colorspace})")

            # Get image position on page
            # img_info: (xref, smask, width, height, bpc, colorspace, alt, name, filter, referencer)
            # We need to find the image rect on page
            bbox_x, bbox_y, bbox_w, bbox_h = 0.0, 0.0, 0.0, 0.0

            # Search for image instances on page
            for img_block in page.get_images():
                if img_block[0] == xref:
                    # Found matching image, try to get its rect
                    rects = page.get_image_rects(img_block)
                    if rects:
                        rect = rects[0]  # Use first occurrence
                        bbox_x = rect.x0
                        bbox_y = rect.y0
                        bbox_w = rect.width
                        bbox_h = rect.height
                    break

            # Create descriptive filename
            img_filename = f"img_page{page_num + 1}_{img_index + 1}.{image_ext}"
            img_path = output_dir / img_filename

            # Save image
            with open(img_path, "wb") as f:
                f.write(image_bytes)

            # Create metadata
            metadata = ImageMetadata(
                filename=img_filename,
                page=page_num + 1,
                index=img_index + 1,
                width=width,
                height=height,
                x=round(bbox_x, 2),
                y=round(bbox_y, 2),
                bbox_width=round(bbox_w, 2),
                bbox_height=round(bbox_h, 2),
                size_bytes=len(image_bytes),
                format=image_ext,
                colorspace=cs_name,
                bits_per_component=bpc
            )
            extracted.append(metadata)

            # Generate markdown reference
            alt_text = f"Image from page {page_num + 1}"
            md_ref = f"![{alt_text}]({img_path.name})"
            markdown_refs.append(f"<!-- Page {page_num + 1}, Position: ({bbox_x:.0f}, {bbox_y:.0f}) -->\n{md_ref}")

            print(f"  ✓ {img_filename} ({width}x{height}, {len(image_bytes):,} bytes)")

    doc.close()

    # Write metadata JSON
    metadata_path = output_dir / "images_metadata.json"
    with open(metadata_path, "w") as f:
        json.dump(
            {
                "source": str(pdf_path),
                "image_count": len(extracted),
                "images": [asdict(m) for m in extracted]
            },
            f,
            indent=2
        )
    print(f"\n📋 Metadata: {metadata_path}")

    # Write markdown references if requested
    if markdown_file and markdown_refs:
        markdown_content = f"# Images from {pdf_path.name}\n\n"
        markdown_content += "\n\n".join(markdown_refs)
        markdown_file.parent.mkdir(parents=True, exist_ok=True)
        markdown_file.write_text(markdown_content)
        print(f"📝 Markdown refs: {markdown_file}")

    print(f"\n✅ Total: {len(extracted)} images extracted to {output_dir}/")
    return extracted


def main():
    parser = argparse.ArgumentParser(
        description="Extract images from PDF files with metadata",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
    # Basic extraction
    uv run --with pymupdf scripts/extract_pdf_images.py document.pdf

    # Custom output directory
    uv run --with pymupdf scripts/extract_pdf_images.py doc.pdf -o ./images

    # With markdown references
    uv run --with pymupdf scripts/extract_pdf_images.py doc.pdf --markdown refs.md

Output:
    Images are saved with descriptive names: img_page1_1.png, img_page2_1.jpg
    Metadata is saved to: images_metadata.json
        """
    )
    parser.add_argument(
        "pdf_path",
        type=Path,
        help="Path to the PDF file"
    )
    parser.add_argument(
        "-o", "--output",
        type=Path,
        default=Path("assets"),
        help="Directory to save images (default: ./assets)"
    )
    parser.add_argument(
        "--markdown",
        type=Path,
        help="Generate markdown file with image references"
    )
    parser.add_argument(
        "--json",
        action="store_true",
        help="Output metadata as JSON to stdout"
    )

    args = parser.parse_args()

    if not args.pdf_path.exists():
        print(f"Error: File not found: {args.pdf_path}", file=sys.stderr)
        sys.exit(1)

    print(f"📄 Extracting images from: {args.pdf_path}")

    extracted = extract_images(
        args.pdf_path,
        args.output,
        args.markdown
    )

    if args.json:
        print(json.dumps([asdict(m) for m in extracted], indent=2))


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Multi-tool markdown output merger with segment-level comparison.

Merges markdown outputs from multiple conversion tools by selecting
the best version of each segment (tables, images, headings, paragraphs).

Usage:
    python merge_outputs.py output1.md output2.md -o merged.md
    python merge_outputs.py --from-json results.json -o merged.md
"""

import argparse
import json
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional


@dataclass
class Segment:
    """A segment of markdown content."""
    type: str  # 'heading', 'table', 'image', 'list', 'paragraph', 'code'
    content: str
    level: int = 0  # For headings
    score: float = 0.0


@dataclass
class MergeResult:
    """Result from merging multiple markdown files."""
    markdown: str
    sources: list[str] = field(default_factory=list)
    segment_sources: dict = field(default_factory=dict)  # segment_idx -> source


def parse_segments(markdown: str) -> list[Segment]:
    """Parse markdown into typed segments."""
    segments = []
    lines = markdown.split('\n')
    current_segment = []
    current_type = 'paragraph'
    current_level = 0
    in_code_block = False
    in_table = False

    def flush_segment():
        nonlocal current_segment, current_type, current_level
        if current_segment:
            content = '\n'.join(current_segment).strip()
            if content:
                segments.append(Segment(
                    type=current_type,
                    content=content,
                    level=current_level
                ))
        current_segment = []
        current_type = 'paragraph'
        current_level = 0

    for line in lines:
        # Code block detection
        if line.startswith('```'):
            if in_code_block:
                current_segment.append(line)
                flush_segment()
                in_code_block = False
                continue
            else:
                flush_segment()
                in_code_block = True
                current_type = 'code'
                current_segment.append(line)
                continue

        if in_code_block:
            current_segment.append(line)
            continue

        # Heading detection
        heading_match = re.match(r'^(#{1,6})\s+(.+)$', line)
        if heading_match:
            flush_segment()
            current_type = 'heading'
            current_level = len(heading_match.group(1))
            current_segment.append(line)
            flush_segment()
            continue

        # Table detection
        if '|' in line and re.match(r'^\s*\|.*\|\s*$', line):
            if not in_table:
                flush_segment()
                in_table = True
                current_type = 'table'
            current_segment.append(line)
            continue
        elif in_table:
            flush_segment()
            in_table = False

        # Image detection
        if re.match(r'!\[.*\]\(.*\)', line):
            flush_segment()
            current_type = 'image'
            current_segment.append(line)
            flush_segment()
            continue

        # List detection
        if re.match(r'^[\s]*[-*+]\s+', line) or re.match(r'^[\s]*\d+\.\s+', line):
            if current_type != 'list':
                flush_segment()
                current_type = 'list'
            current_segment.append(line)
            continue
        elif current_type == 'list' and line.strip() == '':
            flush_segment()
            continue

        # Empty line - potential paragraph break
        if line.strip() == '':
            if current_type == 'paragraph' and current_segment:
                flush_segment()
            continue

        # Default: paragraph
        if current_type not in ['list']:
            current_type = 'paragraph'
        current_segment.append(line)

    flush_segment()
    return segments


def score_segment(segment: Segment) -> float:
    """Score a segment for quality comparison."""
    score = 0.0
    content = segment.content

    if segment.type == 'table':
        # Count rows and columns
        rows = [l for l in content.split('\n') if '|' in l]
        if rows:
            cols = rows[0].count('|') - 1
            score += len(rows) * 0.5  # More rows = better
            score += cols * 0.3  # More columns = better
            # Penalize separator-only tables
            if all(re.match(r'^[\s|:-]+$', r) for r in rows):
                score -= 5.0
            # Bonus for proper header separator
            if len(rows) > 1 and re.match(r'^[\s|:-]+$', rows[1]):
                score += 1.0

    elif segment.type == 'heading':
        # Prefer proper heading hierarchy
        score += 1.0
        # Penalize very long headings
        if len(content) > 100:
            score -= 0.5

    elif segment.type == 'image':
        # Prefer images with alt text
        if re.search(r'!\[.+\]', content):
            score += 1.0
        # Prefer local paths over base64
        if 'data:image' not in content:
            score += 0.5

    elif segment.type == 'list':
        items = re.findall(r'^[\s]*[-*+\d.]+\s+', content, re.MULTILINE)
        score += len(items) * 0.3
        # Bonus for nested lists
        if re.search(r'^\s{2,}[-*+]', content, re.MULTILINE):
            score += 0.5

    elif segment.type == 'code':
        lines = content.split('\n')
        score += min(len(lines) * 0.2, 3.0)
        # Bonus for language specification
        if re.match(r'^```\w+', content):
            score += 0.5

    else:  # paragraph
        words = len(content.split())
        score += min(words * 0.05, 2.0)
        # Penalize very short paragraphs
        if words < 5:
            score -= 0.5

    return score


def find_matching_segment(
    segment: Segment,
    candidates: list[Segment],
    used_indices: set
) -> Optional[int]:
    """Find a matching segment in candidates by type and similarity."""
    best_match = None
    best_similarity = 0.3  # Minimum threshold

    for i, candidate in enumerate(candidates):
        if i in used_indices:
            continue
        if candidate.type != segment.type:
            continue

        # Calculate similarity
        if segment.type == 'heading':
            # Compare heading text (ignore # symbols)
            s1 = re.sub(r'^#+\s*', '', segment.content).lower()
            s2 = re.sub(r'^#+\s*', '', candidate.content).lower()
            similarity = _text_similarity(s1, s2)
        elif segment.type == 'table':
            # Compare first row (header)
            h1 = segment.content.split('\n')[0] if segment.content else ''
            h2 = candidate.content.split('\n')[0] if candidate.content else ''
            similarity = _text_similarity(h1, h2)
        else:
            # Compare content directly
            similarity = _text_similarity(segment.content, candidate.content)

        if similarity > best_similarity:
            best_similarity = similarity
            best_match = i

    return best_match


def _text_similarity(s1: str, s2: str) -> float:
    """Calculate simple text similarity (Jaccard on words)."""
    if not s1 or not s2:
        return 0.0

    words1 = set(s1.lower().split())
    words2 = set(s2.lower().split())

    if not words1 or not words2:
        return 0.0

    intersection = len(words1 & words2)
    union = len(words1 | words2)

    return intersection / union if union > 0 else 0.0


def merge_markdown_files(
    files: list[Path],
    source_names: Optional[list[str]] = None
) -> MergeResult:
    """Merge multiple markdown files by selecting best segments."""
    if not files:
        return MergeResult(markdown="", sources=[])

    if source_names is None:
        source_names = [f.stem for f in files]

    # Parse all files into segments
    all_segments = []
    for i, file_path in enumerate(files):
        content = file_path.read_text()
        segments = parse_segments(content)
        # Score each segment
        for seg in segments:
            seg.score = score_segment(seg)
        all_segments.append((source_names[i], segments))

    if len(all_segments) == 1:
        return MergeResult(
            markdown=files[0].read_text(),
            sources=[source_names[0]]
        )

    # Use first file as base structure
    base_name, base_segments = all_segments[0]
    merged_segments = []
    segment_sources = {}

    for i, base_seg in enumerate(base_segments):
        best_segment = base_seg
        best_source = base_name

        # Find matching segments in other files
        for other_name, other_segments in all_segments[1:]:
            used = set()
            match_idx = find_matching_segment(base_seg, other_segments, used)

            if match_idx is not None:
                other_seg = other_segments[match_idx]
                if other_seg.score > best_segment.score:
                    best_segment = other_seg
                    best_source = other_name

        merged_segments.append(best_segment)
        segment_sources[i] = best_source

    # Check for segments in other files that weren't matched
    # (content that only appears in secondary sources)
    base_used = set(range(len(base_segments)))
    for other_name, other_segments in all_segments[1:]:
        for j, other_seg in enumerate(other_segments):
            match_idx = find_matching_segment(other_seg, base_segments, set())
            if match_idx is None and other_seg.score > 0.5:
                # This segment doesn't exist in base - consider adding
                merged_segments.append(other_seg)
                segment_sources[len(merged_segments) - 1] = other_name

    # Reconstruct markdown
    merged_md = '\n\n'.join(seg.content for seg in merged_segments)

    return MergeResult(
        markdown=merged_md,
        sources=source_names,
        segment_sources=segment_sources
    )


def merge_from_json(json_path: Path) -> MergeResult:
    """Merge from JSON results file (from convert.py)."""
    with open(json_path) as f:
        data = json.load(f)

    results = data.get('results', [])
    if not results:
        return MergeResult(markdown="", sources=[])

    # Filter successful results
    successful = [r for r in results if r.get('success') and r.get('markdown')]
    if not successful:
        return MergeResult(markdown="", sources=[])

    if len(successful) == 1:
        return MergeResult(
            markdown=successful[0]['markdown'],
            sources=[successful[0]['tool']]
        )

    # Parse and merge
    all_segments = []
    for result in successful:
        tool = result['tool']
        segments = parse_segments(result['markdown'])
        for seg in segments:
            seg.score = score_segment(seg)
        all_segments.append((tool, segments))

    # Same merge logic as merge_markdown_files
    base_name, base_segments = all_segments[0]
    merged_segments = []
    segment_sources = {}

    for i, base_seg in enumerate(base_segments):
        best_segment = base_seg
        best_source = base_name

        for other_name, other_segments in all_segments[1:]:
            match_idx = find_matching_segment(base_seg, other_segments, set())
            if match_idx is not None:
                other_seg = other_segments[match_idx]
                if other_seg.score > best_segment.score:
                    best_segment = other_seg
                    best_source = other_name

        merged_segments.append(best_segment)
        segment_sources[i] = best_source

    merged_md = '\n\n'.join(seg.content for seg in merged_segments)

    return MergeResult(
        markdown=merged_md,
        sources=[r['tool'] for r in successful],
        segment_sources=segment_sources
    )


def main():
    parser = argparse.ArgumentParser(
        description="Merge markdown outputs from multiple conversion tools"
    )
    parser.add_argument(
        "inputs",
        nargs="*",
        type=Path,
        help="Input markdown files to merge"
    )
    parser.add_argument(
        "-o", "--output",
        type=Path,
        help="Output merged markdown file"
    )
    parser.add_argument(
        "--from-json",
        type=Path,
        help="Merge from JSON results file (from convert.py)"
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        help="Show segment source attribution"
    )

    args = parser.parse_args()

    if args.from_json:
        result = merge_from_json(args.from_json)
    elif args.inputs:
        # Validate inputs
        for f in args.inputs:
            if not f.exists():
                print(f"Error: File not found: {f}", file=sys.stderr)
                sys.exit(1)
        result = merge_markdown_files(args.inputs)
    else:
        parser.error("Either input files or --from-json is required")

    if not result.markdown:
        print("Error: No content to merge", file=sys.stderr)
        sys.exit(1)

    # Output
    if args.output:
        args.output.parent.mkdir(parents=True, exist_ok=True)
        args.output.write_text(result.markdown)
        print(f"Merged output: {args.output}")
        print(f"Sources: {', '.join(result.sources)}")
    else:
        print(result.markdown)

    if args.verbose and result.segment_sources:
        print("\n--- Segment Attribution ---", file=sys.stderr)
        for idx, source in result.segment_sources.items():
            print(f"  Segment {idx}: {source}", file=sys.stderr)


if __name__ == "__main__":
    main()