Pdf To Html

Security scan passed
Scanned at: 2026-06-07T01:06:19.687678
Tool: gitleaks + pattern-based validation
Content hash: b742b75949f08f3accd8d4962e81347bd14708150ac6bb89857a92c6ba89f43f

#!/usr/bin/env python3
"""Rebuild structure.json into one self-contained, readable HTML file.

Data-driven, not template-per-document: heading levels are inferred from font
size (the most common size is body text; larger sizes step up to h3/h2/h1), so
this works on an arbitrary PDF without hand-coding its sections. Images are
compressed and inlined as base64 -> a single portable .html you can double-click.

Optional overlays (both produced by the translation workflow):
  --translation units.json   {"p1_t0": "translated text", ...}
                             key = p{page}_t{Nth-text-block-on-that-page}.
                             A block with a translation renders its translation;
                             others keep the original text.
  --captions caps.json       {"img-p5-1.png": {"title": "...", "caption": "..."}}
                             attaches a heading bar + caption under that figure
                             (used to explain a chart whose insides stay original).

Text overlay convention (so the renderer can lay out translated prose well):
  blank line = paragraph break · "- " line = list item · "## " line = sub-heading.
  Original (untranslated) text has none of these, so it just flows as paragraphs.

Usage:
  uv run --with Pillow python build_html.py build/structure.json --out out.html
  uv run --with Pillow python build_html.py build/structure.json --out out.html \\
      --translation build/units.json --captions build/caps.json --title "..." --lang zh-CN
"""
import os
import io
import re
import sys
import json
import html
import base64
import argparse
from collections import Counter
from PIL import Image

# Inline a content image up to this width. Bigger than any reading viewport, small
# enough to keep the single file manageable. Charts/line art stay PNG (crisp text);
# wide photos go JPEG (much smaller). Threshold below splits the two.
MAX_IMG_WIDTH = 1400
PHOTO_WIDTH_THRESHOLD = 1000   # rendered width above which we prefer JPEG
JPEG_QUALITY = 82


def load_json(path):
    if not path:
        return {}
    if not os.path.isfile(path):
        sys.exit(f"error: no such file: {path}")
    with open(path) as f:
        return json.load(f)


def data_uri(img_path):
    """Compress + base64 a content image. JPEG for wide photos, PNG otherwise."""
    im = Image.open(img_path)
    if im.width > MAX_IMG_WIDTH:
        h = round(im.height * MAX_IMG_WIDTH / im.width)
        im = im.resize((MAX_IMG_WIDTH, h), Image.LANCZOS)
    buf = io.BytesIO()
    if im.width >= PHOTO_WIDTH_THRESHOLD and im.mode in ("RGB", "RGBA", "P"):
        im.convert("RGB").save(buf, "JPEG", quality=JPEG_QUALITY)
        mime = "jpeg"
    else:
        im.save(buf, "PNG")
        mime = "png"
    return f"data:image/{mime};base64," + base64.b64encode(buf.getvalue()).decode()


def is_page_number(text):
    """A standalone short numeric block is a page number — drop it."""
    return text.strip().isdigit() and len(text.strip()) <= 4


def md(text):
    """Render the lightweight text-overlay convention to HTML.

    Safe on original (non-translated) text too: with no '## ', '- ' or blank
    lines it simply becomes paragraphs.
    """
    out = []
    for blk in re.split(r"\n\s*\n", (text or "").strip()):
        lines = [l for l in blk.split("\n") if l.strip()]
        if not lines:
            continue
        if all(l.strip().startswith("- ") for l in lines):
            items = "".join(f"<li>{html.escape(l.strip()[2:].strip())}</li>" for l in lines)
            out.append(f"<ul>{items}</ul>")
        elif lines[0].strip().startswith("## "):
            out.append(f"<h3>{html.escape(lines[0].strip()[3:].strip())}</h3>")
            rest = " ".join(l.strip() for l in lines[1:])
            if rest:
                out.append(f"<p>{html.escape(rest)}</p>")
        else:
            out.append(f"<p>{html.escape(' '.join(l.strip() for l in lines))}</p>")
    return "\n".join(out)


def build(structure_path, out_path, translation, captions, title, lang, drop_decorative):
    struct = load_json(structure_path)
    tr = load_json(translation)
    caps = load_json(captions)
    img_dir = os.path.join(os.path.dirname(structure_path), "images")

    pages = struct["pages"]
    page_width = struct.get("meta", {}).get("page_width", 612)
    # Body text size = the most common max-size among real text blocks.
    sizes = [round(b["size"]) for pg in pages for b in pg["blocks"]
             if b["type"] == "text" and not is_page_number(b["text"]) and b["size"]]
    body_size = Counter(sizes).most_common(1)[0][0] if sizes else 11
    # Heading levels come from the document's ACTUAL distinct sizes above body,
    # not a fixed multiplier — otherwise a 44pt title and a 16pt sub-heading both
    # land in h1. Largest distinct size -> h1, next -> h2, next and smaller -> h3.
    big_sizes = sorted({s for s in sizes if s > body_size}, reverse=True)
    tier = {s: ("h1", "h2", "h3")[min(i, 2)] for i, s in enumerate(big_sizes)}

    def tag_of(size):
        return tier.get(round(size), "p")

    parts = []
    for pg in pages:
        p = pg["page"]
        ti = 0
        for b in pg["blocks"]:
            if b["type"] == "text":
                raw = b["text"]
                if is_page_number(raw):
                    continue
                key = f"p{p}_t{ti}"
                ti += 1
                content = tr.get(key, raw)
                tag = tag_of(b["size"])
                if tag == "p":
                    parts.append(md(content))          # paragraphs / lists / sub-heads
                else:
                    parts.append(f"<{tag}>{html.escape(content.replace(chr(10), ' ').strip())}</{tag}>")
            elif b["type"] == "image":
                if drop_decorative and b.get("decorative"):
                    continue
                src = os.path.join(img_dir, b["file"])
                if not os.path.isfile(src):
                    continue
                # Display size from how big the image is ON THE PAGE (its bbox),
                # so a small inline icon doesn't blow up to full column width.
                ratio = (b["bbox"][2] - b["bbox"][0]) / page_width if page_width else 1
                szcls = "wide" if ratio >= 0.5 else ("mid" if ratio >= 0.25 else "small")
                cap = caps.get(b["file"])
                if cap:
                    parts.append(
                        f'<figure class="cap">'
                        f'<div class="cap-head">{html.escape(cap.get("title", ""))}</div>'
                        f'<img src="{data_uri(src)}" alt="{html.escape(cap.get("title",""))}">'
                        f'<figcaption>{html.escape(cap.get("caption", ""))}</figcaption></figure>')
                else:
                    parts.append(f'<figure class="{szcls}"><img src="{data_uri(src)}" alt=""></figure>')

    doc_title = title or struct.get("meta", {}).get("title") or "Document"
    body_html = "\n".join(parts)
    page_html = HTML_TEMPLATE.format(lang=html.escape(lang),
                                     title=html.escape(doc_title),
                                     body=body_html)
    with open(out_path, "w") as f:
        f.write(page_html)
    kb = os.path.getsize(out_path) // 1024
    print(f"wrote {out_path} ({kb} KB) — body size={body_size}pt, "
          f"{len(parts)} blocks, {len(tr)} translated, {len(caps)} captioned")
    print("NEXT: verify visually — render with verify_render.py and Read the segments.")


# Neutral, light, professional reading layout. Accent is a calm slate-blue, not a
# brand color, so it suits an arbitrary document. 760px column + 1.85 line-height
# reads well for both Latin and CJK; responsive break at 680px stacks figures.
HTML_TEMPLATE = """<!DOCTYPE html>
<html lang="{lang}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>{title}</title>
<style>
  :root {{ --ink:#1d1d1f; --muted:#6b6b70; --line:#e4e4e8; --soft:#f6f6f8; --accent:#3b5b8c; }}
  * {{ box-sizing:border-box; }}
  body {{ margin:0; background:#fff; color:var(--ink);
    font-family:-apple-system,BlinkMacSystemFont,"PingFang SC","Hiragino Sans GB",
    "Microsoft YaHei","Segoe UI",Roboto,sans-serif;
    font-size:17px; line-height:1.85; -webkit-font-smoothing:antialiased; }}
  .wrap {{ max-width:760px; margin:0 auto; padding:48px 24px 96px; }}
  p {{ margin:0 0 1.1em; }}
  h1 {{ font-size:34px; font-weight:700; line-height:1.3; margin:8px 0 22px; }}
  h2 {{ font-size:24px; font-weight:700; line-height:1.35; margin:42px 0 16px; }}
  h3 {{ font-size:19px; font-weight:700; margin:30px 0 10px; }}
  ul {{ margin:0 0 1.2em; padding-left:1.3em; }}
  li {{ margin:0 0 .55em; }}
  figure {{ margin:30px 0; }}
  figure img {{ width:100%; border:1px solid var(--line); border-radius:10px; display:block; }}
  figure.small {{ text-align:center; }}
  figure.small img {{ width:auto; max-width:140px; display:inline-block; }}
  figure.mid img {{ max-width:62%; margin:0 auto; }}
  figure.cap {{ border:1px solid var(--line); border-radius:12px; overflow:hidden; }}
  figure.cap img {{ border:none; border-radius:0; padding:14px 14px 6px; }}
  .cap-head {{ background:var(--accent); color:#fff; font-weight:700; font-size:15px; padding:11px 18px; }}
  figcaption {{ font-size:13.5px; color:var(--muted); padding:6px 18px 16px; line-height:1.7; }}
  @media (max-width:680px) {{ .wrap {{ padding:28px 18px 60px; }} h1 {{ font-size:27px; }} body {{ font-size:16px; }} }}
</style>
</head>
<body><div class="wrap">
{body}
</div></body>
</html>"""


if __name__ == "__main__":
    ap = argparse.ArgumentParser(description="Build single-file HTML from structure.json.")
    ap.add_argument("structure", help="path to structure.json from extract_pdf.py")
    ap.add_argument("--out", required=True, help="output .html path")
    ap.add_argument("--translation", default=None, help="optional units.json overlay")
    ap.add_argument("--captions", default=None, help="optional figure-caption json")
    ap.add_argument("--title", default=None, help="override document title")
    ap.add_argument("--lang", default="en", help="html lang attribute (e.g. zh-CN)")
    ap.add_argument("--keep-decorative", action="store_true",
                    help="keep images flagged decorative (default: drop them)")
    args = ap.parse_args()
    build(args.structure, args.out, args.translation, args.captions,
          args.title, args.lang, drop_decorative=not args.keep_decorative)

#!/usr/bin/env python3
"""Extract a PDF's structure so it can be rebuilt as faithful, readable HTML.

The point of a separate extraction step is a *verifiable intermediate output*:
structure.json is the plan. Inspect it (and the rendered page PNGs) before
building, instead of going PDF -> HTML in one opaque jump.

Outputs (under --outdir, default "<pdf-stem>-build/"):
  structure.json  per-page blocks in reading order. Text blocks carry their
                  bbox + max font size (font size is what build_html.py uses to
                  infer heading levels). Image blocks carry bbox, pixel size,
                  byte size, and a `decorative` flag.
  images/         every embedded raster at original resolution.
  pages/          one rendered PNG per page — so Claude can SEE the layout and
                  read figures, not just the text stream. Text-correct is not
                  layout-correct; always look at these.

Reading order: PyMuPDF's get_text("dict") already returns blocks in reading
order, so block order is preserved as-is — this is what lets an image sit in the
right place between paragraphs.

Usage:
  uv run --with pymupdf python extract_pdf.py input.pdf
  uv run --with pymupdf python extract_pdf.py input.pdf --outdir build --dpi 150
"""
import sys
import os
import json
import argparse
import fitz  # PyMuPDF

# A raster under this many bytes is almost always a rule / spacer / bullet glyph,
# not content worth inlining. Real figures in Office/Google-Docs exports are tens
# of KB and up; decorative separators are well under 3 KB. Marked, not deleted —
# build_html.py decides whether to drop it, and you can override per document.
DECORATIVE_MAX_BYTES = 3000


def extract(pdf_path, outdir, dpi):
    if not os.path.isfile(pdf_path):
        sys.exit(f"error: no such file: {pdf_path}")
    try:
        doc = fitz.open(pdf_path)
    except Exception as e:
        sys.exit(f"error: cannot open PDF ({e}). If it's a scanned image PDF, "
                 f"OCR it first (e.g. ocrmypdf) — this skill needs real text.")

    os.makedirs(f"{outdir}/images", exist_ok=True)
    os.makedirs(f"{outdir}/pages", exist_ok=True)

    npages = len(doc)
    bbox_counts = {}   # bucketed image bbox -> how many pages it appears on
    raw_pages = []

    for pno in range(npages):
        page = doc.load_page(pno)
        # Render the page so Claude can look at the real layout. dpi 120 is a
        # readable default; raise for tiny print.
        page.get_pixmap(dpi=dpi).save(f"{outdir}/pages/page-{pno+1:02d}.png")

        blocks = []
        nimg = 0
        for b in page.get_text("dict")["blocks"]:
            if b["type"] == 0:  # text
                text, sizes = "", []
                for line in b["lines"]:
                    for span in line["spans"]:
                        text += span["text"]
                        sizes.append(round(span["size"], 1))
                    text += "\n"
                text = text.strip()
                if text:
                    blocks.append({
                        "type": "text",
                        "bbox": [round(x) for x in b["bbox"]],
                        "text": text,
                        "size": max(sizes) if sizes else 0,
                    })
            elif b["type"] == 1:  # image
                nimg += 1
                ext = b.get("ext", "png")
                fn = f"img-p{pno+1}-{nimg}.{ext}"
                data = b["image"]
                with open(f"{outdir}/images/{fn}", "wb") as f:
                    f.write(data)
                # Bucket the bbox so near-identical positions across pages collapse
                # to one key — that is how we detect repeating headers/footers.
                key = tuple(round(x / 5) * 5 for x in b["bbox"])
                bbox_counts[key] = bbox_counts.get(key, 0) + 1
                blocks.append({
                    "type": "image",
                    "bbox": [round(x) for x in b["bbox"]],
                    "file": fn,
                    "w": b.get("width"),
                    "h": b.get("height"),
                    "bytes": len(data),
                    "_bbox_key": list(key),
                })
        raw_pages.append({"page": pno + 1, "blocks": blocks})

    # Mark decorative images: tiny byte size, OR the same bbox repeating on more
    # than half the pages (a running header/footer logo). max(2, ...) so short
    # documents don't false-positive a 2-page coincidence.
    repeat_threshold = max(2, npages // 2)
    for pg in raw_pages:
        for blk in pg["blocks"]:
            if blk["type"] == "image":
                repeated = bbox_counts.get(tuple(blk["_bbox_key"]), 0) > repeat_threshold
                blk["decorative"] = bool(blk["bytes"] < DECORATIVE_MAX_BYTES or repeated)
                del blk["_bbox_key"]

    meta = {
        "source": os.path.basename(pdf_path),
        "pages": npages,
        "page_width": round(doc[0].rect.width) if npages else 612,
        "title": doc.metadata.get("title") or "",
        "render_dpi": dpi,
    }
    out = {"meta": meta, "pages": raw_pages}
    with open(f"{outdir}/structure.json", "w") as f:
        json.dump(out, f, ensure_ascii=False, indent=1)

    # Console summary so a successful run is self-evident (and easy to sanity-check).
    print(f"source: {meta['source']}  pages: {npages}  title: {meta['title'] or '(none)'}")
    for pg in raw_pages:
        ntext = sum(1 for b in pg["blocks"] if b["type"] == "text")
        imgs = [b for b in pg["blocks"] if b["type"] == "image"]
        content_imgs = sum(1 for b in imgs if not b["decorative"])
        print(f"  page {pg['page']:>2}: {ntext} text blocks, "
              f"{content_imgs} content image(s), {len(imgs)-content_imgs} decorative")
    print(f"\nwrote {outdir}/structure.json + images/ + pages/")
    print("NEXT: Read the pages/*.png to see the real layout before building.")


if __name__ == "__main__":
    ap = argparse.ArgumentParser(description="Extract PDF structure for HTML rebuild.")
    ap.add_argument("pdf")
    ap.add_argument("--outdir", default=None, help="default: <pdf-stem>-build/")
    ap.add_argument("--dpi", type=int, default=120, help="page render DPI (default 120)")
    args = ap.parse_args()
    outdir = args.outdir or os.path.splitext(os.path.basename(args.pdf))[0] + "-build"
    extract(args.pdf, outdir, args.dpi)

#!/usr/bin/env python3
"""Render the built HTML with headless Chrome and slice it into readable segments.

Why this exists: text-correct is not render-correct. Fonts can fall back, tables
can overflow, a translated heading can wrap badly — none of which show up unless
you LOOK. After running this, Read each seg-*.png and check the layout.

Two real gotchas this script handles for you:
  1. Chrome's headless screenshot caps height around 16384 physical px. A 2x shot
     of a long page silently truncates. So we first probe the real content height
     at 1x, then pick the largest device-scale-factor that keeps the full page
     under the cap (crisp when it fits, still complete when it doesn't).
  2. A full-page shot is one tall image; thumbnailed, the text is unreadable. So
     we slice into ~2600px-tall segments — each one is legible when Read.

Usage:
  uv run --with Pillow --with numpy python verify_render.py out.html
  uv run --with Pillow --with numpy python verify_render.py out.html --outdir shots --width 840 --scale 2
"""
import os
import sys
import shutil
import argparse
import subprocess
from PIL import Image
import numpy as np

# Stay comfortably under Chrome's ~16384px headless screenshot ceiling.
MAX_PHYSICAL = 15000
SEGMENT_PHYSICAL = 2600   # tall enough to be efficient, short enough to read


def find_chrome():
    candidates = [
        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
        "/Applications/Chromium.app/Contents/MacOS/Chromium",
    ]
    for c in candidates:
        if os.path.isfile(c):
            return c
    for name in ("google-chrome", "chromium", "chromium-browser", "chrome"):
        p = shutil.which(name)
        if p:
            return p
    sys.exit("error: Chrome/Chromium not found — it's required for visual "
             "verification. Install Google Chrome, or pass a different verifier.")


def shoot(chrome, html_path, out_png, width, height, scale):
    subprocess.run([
        chrome, "--headless", "--disable-gpu", "--no-sandbox", "--hide-scrollbars",
        "--no-proxy-server",                      # local file:// must not route via a proxy
        f"--force-device-scale-factor={scale}",
        "--virtual-time-budget=10000",            # let base64 images + fonts settle
        f"--window-size={width},{height}",
        f"--screenshot={out_png}", f"file://{html_path}",
    ], check=False, capture_output=True)
    if not os.path.isfile(out_png):
        sys.exit(f"error: Chrome produced no screenshot ({out_png}). "
                 f"Check the HTML path and that Chrome runs headless on this machine.")


def content_height(png):
    """Bottom of the actual content (trim the blank tail below the page)."""
    a = np.array(Image.open(png).convert("L"))
    nonwhite = np.where((a < 250).any(axis=1))[0]
    return int(nonwhite.max()) + 1 if len(nonwhite) else a.shape[0]


def verify(html_path, outdir, width, desired_scale):
    if not os.path.isfile(html_path):
        sys.exit(f"error: no such file: {html_path}")
    chrome = find_chrome()
    os.makedirs(outdir, exist_ok=True)

    # 1) Probe true content height at 1x (1 CSS px == 1 device px here).
    probe = os.path.join(outdir, "_probe.png")
    shoot(chrome, html_path, probe, width, 16000, 1)
    css_height = content_height(probe)

    # 2) Largest scale that keeps the whole page under the physical cap.
    # Largest scale that keeps the whole page under the cap. Don't round up —
    # that can nudge scale*height back over the cap and force an unwanted 1x.
    scale = max(1.0, min(desired_scale, MAX_PHYSICAL / max(css_height, 1)))

    if scale * css_height <= MAX_PHYSICAL:
        final = os.path.join(outdir, "_full.png")
        shoot(chrome, html_path, final, width, css_height + 40, scale)
        note = f"scale {scale}x"
    else:
        # Page taller than the cap even at 1x — keep the complete 1x probe, trimmed.
        final = probe
        scale = 1.0
        note = "scale 1x (page exceeds cap; rendered complete but not magnified)"

    # 3) Slice into readable segments.
    im = Image.open(final)
    full = im.crop((0, 0, im.width, min(im.height, round((css_height + 40) * scale))))
    n = (full.height + SEGMENT_PHYSICAL - 1) // SEGMENT_PHYSICAL
    paths = []
    for i in range(n):
        top, bot = i * SEGMENT_PHYSICAL, min(full.height, (i + 1) * SEGMENT_PHYSICAL)
        seg = os.path.join(outdir, f"seg-{i+1:02d}.png")
        full.crop((0, top, full.width, bot)).save(seg)
        paths.append(seg)

    if os.path.exists(probe) and final != probe:
        os.remove(probe)

    print(f"rendered {html_path} at {note} -> {n} segment(s) in {outdir}/")
    for p in paths:
        print(f"  {p}")
    print("\nNEXT: Read every seg-*.png and check: fonts render (no tofu boxes), "
          "tables/figures aren't clipped, headings/lists look right, images present.")


if __name__ == "__main__":
    ap = argparse.ArgumentParser(description="Headless-render HTML and slice into readable PNGs.")
    ap.add_argument("html")
    ap.add_argument("--outdir", default="render-check", help="where to write segments")
    ap.add_argument("--width", type=int, default=840, help="viewport CSS width (default 840)")
    ap.add_argument("--scale", type=float, default=2.0, help="desired device scale (default 2)")
    args = ap.parse_args()
    verify(args.html, args.outdir, args.width, args.scale)