Literature Review Agent

#!/usr/bin/env python3
"""
bibtex_format.py — Generate refs.bib from a verified citation pool.

Reads citation_pool.json (output of dedupe_by_id.py) and emits a BibTeX file
with deterministic citation keys derived from the first author + year +
first significant title word.

Never invents fields. Only writes fields that are actually present in the
S2 metadata. Writes one of:
    @article{ ... }      — when venue looks like a journal
    @inproceedings{ ... }— when venue looks like a conference
    @misc{ ... }         — fallback (e.g., arXiv-only papers)

Usage:
    python bibtex_format.py --pool citation_pool.json --out refs.bib
"""
import argparse
import json
import re
import sys

CONFERENCE_HINTS = {
    "neurips", "nips", "icml", "iclr", "cvpr", "iccv", "eccv", "aaai",
    "ijcai", "acl", "emnlp", "naacl", "kdd", "www", "sigir", "uai", "siggraph",
    "interspeech", "icassp", "miccai", "wacv", "bmvc", "coling", "conll",
}
STOPWORDS = {
    "a", "an", "and", "the", "of", "for", "to", "with", "on", "in", "by",
    "from", "as", "is", "are", "be", "via", "into", "their", "our", "we",
    "this", "that", "using", "use", "about", "at", "or", "if",
}


def normalize(s: str) -> str:
    return re.sub(r"[^a-z]", "", s.lower())


def first_significant_word(title: str) -> str:
    for w in re.findall(r"[A-Za-z][A-Za-z\-]*", title):
        wn = w.lower()
        if wn not in STOPWORDS and len(wn) > 2:
            return normalize(wn)
    return "paper"


def first_author_lastname(authors: list[dict]) -> str:
    if not authors:
        return "anon"
    name = authors[0].get("name", "").strip()
    if not name:
        return "anon"
    parts = name.replace(",", "").split()
    return normalize(parts[-1]) or "anon"


def make_key(paper: dict) -> str:
    last = first_author_lastname(paper.get("authors") or [])
    year = paper.get("year") or "0000"
    word = first_significant_word(paper.get("title", ""))
    return f"{last}{year}{word}"


def is_conference(venue: str) -> bool:
    if not venue:
        return False
    v = venue.lower()
    return any(h in v for h in CONFERENCE_HINTS)


def escape_bibtex(s: str) -> str:
    if not s:
        return ""
    return s.replace("{", "\\{").replace("}", "\\}").replace("&", "\\&")


def author_field(authors: list[dict]) -> str:
    names = [a.get("name", "").strip() for a in authors if a.get("name")]
    return " and ".join(escape_bibtex(n) for n in names)


def format_entry(paper: dict, key: str) -> str:
    venue = paper.get("venue") or ""
    if is_conference(venue):
        kind = "inproceedings"
        venue_key = "booktitle"
    elif venue:
        kind = "article"
        venue_key = "journal"
    else:
        kind = "misc"
        venue_key = None

    lines = [f"@{kind}{{{key},"]
    if title := paper.get("title"):
        lines.append(f'  title    = {{{escape_bibtex(title)}}},')
    if authors := paper.get("authors"):
        lines.append(f'  author   = {{{author_field(authors)}}},')
    if year := paper.get("year"):
        lines.append(f'  year     = {{{year}}},')
    if venue and venue_key:
        lines.append(f'  {venue_key:8s} = {{{escape_bibtex(venue)}}},')
    ext = paper.get("externalIds") or {}
    if doi := ext.get("DOI"):
        lines.append(f'  doi      = {{{doi}}},')
    if arxiv := ext.get("ArXiv"):
        lines.append(f'  eprint   = {{{arxiv}}},')
        lines.append(f'  archivePrefix = {{arXiv}},')
    # Strip trailing comma on last field
    if lines[-1].endswith(","):
        lines[-1] = lines[-1].rstrip(",")
    lines.append("}")
    return "\n".join(lines)


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--pool", required=True, help="citation_pool.json")
    p.add_argument("--out", required=True, help="output refs.bib")
    args = p.parse_args()

    with open(args.pool) as f:
        pool = json.load(f)
    papers = pool.get("papers", [])
    if not papers:
        print("ERROR: pool contains no papers", file=sys.stderr)
        return 1

    keys_used: dict[str, int] = {}
    entries: list[str] = []
    paper_keys: list[str] = []

    for paper in papers:
        base_key = make_key(paper)
        # Disambiguate collisions with letter suffix
        if base_key in keys_used:
            keys_used[base_key] += 1
            suffix = chr(ord("a") + keys_used[base_key] - 1)
            key = base_key + suffix
        else:
            keys_used[base_key] = 1
            key = base_key
        paper["bibtex_key"] = key
        paper_keys.append(key)
        entries.append(format_entry(paper, key))

    with open(args.out, "w") as f:
        f.write("% Generated by paper-orchestra literature-review-agent/bibtex_format.py\n")
        f.write(f"% {len(entries)} entries from citation_pool.json\n\n")
        f.write("\n\n".join(entries))
        f.write("\n")

    # Write the keys back into the pool so the writing step has the
    # citation_checklist mapping. (Idempotent — overwrites with same data.)
    with open(args.pool, "w") as f:
        json.dump(pool, f, indent=2, ensure_ascii=False)

    print(f"OK: {len(entries)} BibTeX entries → {args.out}")
    print(f"    keys: {', '.join(paper_keys[:5])}{'...' if len(paper_keys) > 5 else ''}")
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
check_cutoff.py — Strict temporal cutoff check for citation verification.

Implements the paper's Rule 3 (App. D.3): a paper passes only if its
publication date strictly predates the research cutoff. When only the year
is known, assume the worst case (Dec 31). When year + month are known,
assume day-1 of that month (per the paper's "first day of that month"
default).

Exit codes:
    0  paper strictly predates cutoff (PASS)
    1  paper does not strictly predate cutoff (FAIL)
    2  argument error

Usage:
    python check_cutoff.py --paper-year 2024 --paper-month 9 --cutoff 2024-10-01
    python check_cutoff.py --paper-year 2024 --cutoff 2024-10-01
    python check_cutoff.py --paper-date 2024-09-15 --cutoff 2024-10-01
"""
import argparse
import datetime as dt
import sys


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--paper-year", type=int, help="Paper publication year")
    p.add_argument("--paper-month", type=int, help="Paper publication month (1-12), optional")
    p.add_argument("--paper-date", help="Full paper date YYYY-MM-DD, overrides year/month")
    p.add_argument("--cutoff", required=True, help="Research cutoff date YYYY-MM-DD")
    args = p.parse_args()

    try:
        cutoff = dt.date.fromisoformat(args.cutoff)
    except ValueError:
        print(f"ERROR: --cutoff must be YYYY-MM-DD, got {args.cutoff}", file=sys.stderr)
        return 2

    if args.paper_date:
        try:
            paper_date = dt.date.fromisoformat(args.paper_date)
        except ValueError:
            print(f"ERROR: --paper-date must be YYYY-MM-DD, got {args.paper_date}",
                  file=sys.stderr)
            return 2
    elif args.paper_year:
        if args.paper_month:
            paper_date = dt.date(args.paper_year, args.paper_month, 1)
        else:
            paper_date = dt.date(args.paper_year, 12, 31)
    else:
        print("ERROR: must provide --paper-date OR --paper-year", file=sys.stderr)
        return 2

    if paper_date < cutoff:
        print(f"PASS  paper={paper_date}  <  cutoff={cutoff}")
        return 0
    print(f"FAIL  paper={paper_date}  not strictly before cutoff={cutoff}")
    return 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
citation_coverage.py — Enforce the paper's ≥90% citation integration rule
(App. D.3).

Greps a generated .tex file for all citation commands, counts the unique
keys actually cited, and compares against the verified citation pool.
Exits non-zero if coverage < 90%.

Usage:
    python citation_coverage.py --tex intro_relwork.tex --pool citation_pool.json
    python citation_coverage.py --tex intro_relwork.tex --pool citation_pool.json --threshold 0.85
"""
import argparse
import json
import re
import sys

CITE_RE = re.compile(
    r"\\(?:cite|citep|citet|citeauthor|citeyear|autocite|parencite|textcite)"
    r"(?:\[[^\]]*\])?"
    r"\{([^}]+)\}"
)


def extract_cited_keys(tex: str) -> set[str]:
    keys = set()
    for m in CITE_RE.finditer(tex):
        for k in m.group(1).split(","):
            k = k.strip()
            if k:
                keys.add(k)
    return keys


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--tex", required=True, help="LaTeX file to inspect")
    p.add_argument("--pool", required=True, help="citation_pool.json")
    p.add_argument("--threshold", type=float, default=0.90,
                   help="Minimum integration ratio (default 0.90 per paper)")
    args = p.parse_args()

    with open(args.tex) as f:
        tex = f.read()
    with open(args.pool) as f:
        pool = json.load(f)

    pool_papers = pool.get("papers", [])
    pool_keys = {p.get("bibtex_key") for p in pool_papers if p.get("bibtex_key")}
    if not pool_keys:
        print("ERROR: pool has no bibtex_keys. Run bibtex_format.py first.",
              file=sys.stderr)
        return 1

    cited = extract_cited_keys(tex)
    cited_in_pool = cited & pool_keys
    n_pool = len(pool_keys)
    n_cited = len(cited_in_pool)
    ratio = n_cited / n_pool if n_pool else 0.0
    threshold_n = int(args.threshold * n_pool)

    print(f"Coverage: {n_cited}/{n_pool} = {ratio*100:.1f}% "
          f"(threshold {args.threshold*100:.0f}% = {threshold_n})")

    # report keys cited but NOT in pool — those are forbidden by the prompt
    foreign = cited - pool_keys
    if foreign:
        print(f"\nWARNING: {len(foreign)} cited keys NOT in citation pool "
              f"(violates 'cite ONLY collected_papers' rule):")
        for k in sorted(foreign):
            print(f"  - {k}")

    if n_cited < threshold_n:
        uncited = pool_keys - cited
        print(f"\nFAIL: missing {len(uncited)} pool papers from .tex:")
        # show with title for actionable re-prompting
        title_by_key = {p.get("bibtex_key"): p.get("title", "")
                        for p in pool_papers if p.get("bibtex_key")}
        discovered_by_key = {p.get("bibtex_key"): p.get("discovered_for", [])
                             for p in pool_papers if p.get("bibtex_key")}
        for k in sorted(uncited):
            tag = ",".join(discovered_by_key.get(k, [])) or "?"
            t = title_by_key.get(k, "")
            print(f"  - {k:40s}  [{tag}]  {t[:60]}")
        return 1

    print("OK: citation coverage meets threshold")
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
cross_verify.py — Cross-index corroboration for a verified citation pool.

Semantic Scholar verification (Levenshtein title match + cutoff + dedup) is the
pipeline's first gate. This script adds a second: every paper that S2 accepted
is re-checked against two *independent* scholarly indices — Crossref and
OpenAlex. A genuine paper turns up in all three with matching metadata; a
hallucinated or mis-attributed record typically does not. This is the practical
defense against the well-documented problem of fabricated citations leaking
into AI-assisted writing.

The script is a WARN gate, not a hard gate (mirrors validate_consistency.py):
it annotates the pool and writes a report, but exits non-zero only to draw the
host agent's attention to flagged entries — it does not block the pipeline.
The host agent reviews flagged citations and decides whether to drop them.

Confidence tiers written onto each paper's `cross_verification` field:
    high      corroborated by >=1 external index, no metadata conflicts
    medium    corroborated, but publication year disagrees beyond tolerance
    low       NOT found in Crossref or OpenAlex — phantom/hallucination risk
    conflict  a DOI present in the pool disagrees with the external index's DOI

Network etiquette: queries are throttled (--sleep, default 1.0s between papers)
and identify via a polite-pool email if PAPER_ORCHESTRA_MAILTO (or the
per-service CROSSREF_MAILTO / OPENALEX_MAILTO) is set. If an index is
unreachable the script degrades gracefully — it disables that index, notes it
in the report, and continues with whatever indices remain.

Usage:
    python cross_verify.py --pool workspace/citation_pool.json
    python cross_verify.py --pool workspace/citation_pool.json --inplace
    python cross_verify.py --pool workspace/citation_pool.json \\
        --out workspace/cross_verification_report.json \\
        --indices crossref,openalex --threshold 70 --year-tolerance 1 --sleep 1.0

Exit codes:
    0  every paper corroborated (all high/medium), no flags
    1  one or more papers flagged low/conflict, OR an index was unavailable (WARN)
    2  usage error / pool unreadable
"""
import argparse
import json
import os
import re
import sys
import time

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import crossref_client  # noqa: E402
import openalex_client  # noqa: E402

try:
    import Levenshtein  # noqa: E402

    def _ratio(a: str, b: str) -> int:
        return int(round(Levenshtein.ratio(a, b) * 100))
except ImportError:  # graceful fallback — stdlib only
    from difflib import SequenceMatcher

    def _ratio(a: str, b: str) -> int:
        return int(round(SequenceMatcher(None, a, b).ratio() * 100))


def _normalize(s: str) -> str:
    s = (s or "").lower().strip()
    s = re.sub(r"[^a-z0-9\s]", " ", s)
    return re.sub(r"\s+", " ", s).strip()


def title_ratio(a: str, b: str) -> int:
    return _ratio(_normalize(a), _normalize(b))


def _bare_doi(doi: str | None) -> str:
    if not doi:
        return ""
    d = doi.strip().lower()
    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
        if d.startswith(prefix):
            d = d[len(prefix):]
    return d


def pool_doi(paper: dict) -> str:
    """Extract a DOI from a pool record, tolerating several shapes."""
    ext = paper.get("externalIds") or {}
    for k in ("DOI", "doi", "Doi"):
        if ext.get(k):
            return _bare_doi(ext[k])
    if paper.get("doi"):
        return _bare_doi(paper["doi"])
    return ""


def best_match(title: str, hits: list[dict]) -> dict | None:
    """Pick the external hit whose title best matches `title`."""
    best, best_r = None, -1
    for h in hits:
        r = title_ratio(title, h.get("title", ""))
        if r > best_r:
            best, best_r = h, r
    if best is None:
        return None
    out = dict(best)
    out["title_ratio"] = best_r
    return out


def _safe(fn, *args):
    """Call a client function, converting its SystemExit (network/HTTP error)
    into a None so cross_verify can degrade instead of dying."""
    try:
        return fn(*args)
    except SystemExit:
        return None


def check_index(module, paper: dict, threshold: int, strong_threshold: int, limit: int) -> dict:
    """Look one paper up in a single external index. Returns a per-index dict,
    or {"available": False} if the index could not be reached.

    Two match strengths are recorded:
      found  — title_ratio > threshold (lenient): "something like this exists"
      strong — title_ratio >= strong_threshold, or an exact DOI hit: "this is
               confidently the same paper". Only `strong` matches are trusted
               for year/DOI conflict downgrades, so a noisy near-title hit
               (e.g. a different paper with a similar name) cannot pollute the
               metadata-conflict checks.
    """
    title = paper.get("title", "")
    doi = pool_doi(paper)

    result = None
    via = "title"
    if doi:
        r = _safe(module.lookup_doi, doi)
        if r is None:
            return {"available": False}
        if r["data"]:
            result = r["data"][0]
            result["title_ratio"] = title_ratio(title, result.get("title", ""))
            via = "doi"
    if result is None:
        r = _safe(module.search, title, limit)
        if r is None:
            return {"available": False}
        result = best_match(title, r["data"])

    if result is None:
        return {"available": True, "found": False, "strong": False}

    tr = result.get("title_ratio", 0)
    found = tr > threshold or via == "doi"
    strong = tr >= strong_threshold or via == "doi"
    ext_doi = _bare_doi(result.get("doi"))
    doi_state = "n/a"
    if doi and ext_doi:
        doi_state = "agree" if doi == ext_doi else "conflict"
    return {
        "available": True,
        "found": bool(found),
        "strong": bool(strong),
        "via": via,
        "matched_title": result.get("title", ""),
        "title_ratio": tr,
        "matched_year": result.get("year"),
        "matched_doi": ext_doi,
        "doi_state": doi_state,
    }


def classify(paper: dict, per_index: dict, year_tolerance: int) -> dict:
    found_in = [name for name, r in per_index.items() if r.get("found")]
    # Only confident same-paper matches can trigger a metadata-conflict downgrade.
    doi_conflict = any(
        r.get("doi_state") == "conflict" and r.get("strong") for r in per_index.values()
    )

    year = paper.get("year")
    year_conflict = False
    if year is not None:
        for r in per_index.values():
            my = r.get("matched_year")
            if r.get("strong") and my is not None and abs(int(my) - int(year)) > year_tolerance:
                year_conflict = True

    notes = []
    if doi_conflict:
        confidence = "conflict"
        notes.append("DOI in pool disagrees with external index — verify this record")
    elif not found_in:
        confidence = "low"
        notes.append("not found in Crossref or OpenAlex — possible phantom citation")
    elif year_conflict:
        confidence = "medium"
        notes.append("corroborated, but publication year disagrees beyond tolerance")
    else:
        confidence = "high"

    return {
        "confidence": confidence,
        "indices_found": found_in,
        "doi_conflict": doi_conflict,
        "year_conflict": year_conflict,
        "per_index": per_index,
        "notes": notes,
    }


def main() -> int:
    p = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    p.add_argument("--pool", required=True, help="citation_pool.json path")
    p.add_argument("--out", default=None,
                   help="Report path (default: <pool dir>/cross_verification_report.json)")
    p.add_argument("--inplace", action="store_true",
                   help="Annotate each paper in the pool with its cross_verification result")
    p.add_argument("--indices", default="crossref,openalex",
                   help="Comma-separated external indices to check (default both)")
    p.add_argument("--threshold", type=int, default=70,
                   help="Levenshtein title-match threshold for corroboration "
                        "(default 70, matches S2 gate)")
    p.add_argument("--strong-threshold", type=int, default=90,
                   help="Stricter title-match threshold before an external record's "
                        "year/DOI is trusted for a conflict downgrade (default 90)")
    p.add_argument("--year-tolerance", type=int, default=1,
                   help="Allowed |year| difference before flagging (default 1)")
    p.add_argument("--limit", type=int, default=5, help="Max hits per title search")
    p.add_argument("--sleep", type=float, default=1.0,
                   help="Seconds to pause between papers (politeness, default 1.0)")
    args = p.parse_args()

    try:
        with open(args.pool) as f:
            pool = json.load(f)
    except (OSError, json.JSONDecodeError) as exc:
        print(f"ERROR: cannot read pool: {exc}", file=sys.stderr)
        return 2

    papers = pool.get("papers", [])
    if not papers:
        print("ERROR: pool['papers'] is empty or missing", file=sys.stderr)
        return 2

    modules = {"crossref": crossref_client, "openalex": openalex_client}
    requested = [x.strip() for x in args.indices.split(",") if x.strip()]
    unknown = [x for x in requested if x not in modules]
    if unknown:
        print(f"ERROR: unknown index/indices: {', '.join(unknown)}", file=sys.stderr)
        return 2

    disabled: set[str] = set()
    summary = {"high": 0, "medium": 0, "low": 0, "conflict": 0}
    flagged = []

    for i, paper in enumerate(papers):
        per_index = {}
        for name in requested:
            if name in disabled:
                continue
            res = check_index(modules[name], paper, args.threshold,
                              args.strong_threshold, args.limit)
            if res.get("available") is False:
                disabled.add(name)
                print(f"WARN: index '{name}' unreachable — disabling for the rest of this run.",
                      file=sys.stderr)
                continue
            per_index[name] = res

        verdict = classify(paper, per_index, args.year_tolerance)
        summary[verdict["confidence"]] += 1
        if verdict["confidence"] in ("low", "conflict"):
            flagged.append({
                "bibtex_key": paper.get("bibtex_key") or paper.get("key"),
                "title": paper.get("title"),
                "confidence": verdict["confidence"],
                "notes": verdict["notes"],
            })
        if args.inplace:
            paper["cross_verification"] = verdict

        # Throttle between papers (skip after the last one).
        if i < len(papers) - 1 and args.sleep > 0 and requested != list(disabled):
            time.sleep(args.sleep)

    report = {
        "total_papers": len(papers),
        "indices_checked": [x for x in requested if x not in disabled],
        "indices_unavailable": sorted(disabled),
        "summary": summary,
        "flagged": flagged,
    }

    out_path = args.out or os.path.join(os.path.dirname(os.path.abspath(args.pool)),
                                        "cross_verification_report.json")
    with open(out_path, "w") as f:
        json.dump(report, f, indent=2, ensure_ascii=False)

    if args.inplace:
        with open(args.pool, "w") as f:
            json.dump(pool, f, indent=2, ensure_ascii=False)

    # Human-readable summary to stdout.
    print(f"Cross-index verification ({', '.join(report['indices_checked']) or 'none'}):")
    print(f"  {len(papers)} papers  |  high={summary['high']} "
          f"medium={summary['medium']} low={summary['low']} conflict={summary['conflict']}")
    print(f"  report → {out_path}")
    if flagged:
        print(f"\nWARN: {len(flagged)} citation(s) need review:")
        for f_ in flagged:
            print(f"  [{f_['confidence'].upper()}] {f_['bibtex_key']}: {f_['title']}")
            for note in f_["notes"]:
                print(f"      → {note}")
        print("\nReview these before building refs.bib. Drop any you cannot corroborate.")

    if disabled:
        print(f"\nNote: {', '.join(sorted(disabled))} unavailable this run — "
              "corroboration is partial.", file=sys.stderr)

    return 1 if (flagged or disabled) else 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
crossref_client.py — Crossref REST API title/DOI lookup for cross-index
citation corroboration.

Used by cross_verify.py as a second opinion alongside Semantic Scholar:
if a paper that S2 "verified" cannot be found in Crossref *or* OpenAlex, it is
flagged as a potential phantom record (hallucination risk). See
references/cross-index-verification.md for the rationale.

No API key is required. Crossref offers a "polite pool" with better
reliability when you identify yourself via an email address. Set one of:
    export CROSSREF_MAILTO="you@example.com"
    export PAPER_ORCHESTRA_MAILTO="you@example.com"   # shared fallback
The email is sent only as a `mailto` query parameter / User-Agent, per
Crossref's etiquette guidelines. The repo never commits an address.

Usage:
    # title search
    python crossref_client.py --query "Attention is All You Need"

    # direct DOI lookup (exact)
    python crossref_client.py --doi 10.5555/3295222.3295349

    # raw Crossref JSON
    python crossref_client.py --query "BERT pre-training" --raw

Output (normalized): {"total": N, "data": [{title, year, doi, venue, authors}, ...]}

Exit codes:
    0  at least one result returned
    1  HTTP error, network error, or zero results
    2  usage error (bad arguments)
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

CROSSREF_BASE = "https://api.crossref.org/works"
DEFAULT_LIMIT = 5
MAX_LIMIT = 20
_RETRY_SLEEP = 5  # seconds to wait after a 429 before retrying
SELECT_FIELDS = "DOI,title,author,published,published-print,published-online,issued,container-title,type"


def _mailto() -> str:
    return (
        os.environ.get("CROSSREF_MAILTO", "").strip()
        or os.environ.get("PAPER_ORCHESTRA_MAILTO", "").strip()
    )


def _build_request(url: str) -> urllib.request.Request:
    mailto = _mailto()
    ua = "paper-orchestra/1.0 (https://github.com/Ar9av/paper-orchestra)"
    if mailto:
        ua = f"paper-orchestra/1.0 (mailto:{mailto})"
    return urllib.request.Request(
        url, headers={"Accept": "application/json", "User-Agent": ua}, method="GET"
    )


def _get(url: str, retries: int = 3) -> dict:
    for attempt in range(1, retries + 1):
        try:
            with urllib.request.urlopen(_build_request(url), timeout=30) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except urllib.error.HTTPError as exc:
            if exc.code == 404:
                return {"message": {"items": []}}
            if exc.code == 429 and attempt < retries:
                print(f"WARN: Crossref rate-limited (429). Sleeping {_RETRY_SLEEP}s "
                      f"before retry {attempt + 1}/{retries}.", file=sys.stderr)
                time.sleep(_RETRY_SLEEP)
                continue
            if exc.code in (500, 502, 503) and attempt < retries:
                print(f"WARN: Crossref server error ({exc.code}). Retrying.", file=sys.stderr)
                time.sleep(10)
                continue
            print(f"ERROR: Crossref HTTP {exc.code}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as exc:
            print(f"ERROR: Network error reaching Crossref: {exc.reason}", file=sys.stderr)
            sys.exit(1)
    sys.exit(1)


def _year_from(work: dict) -> int | None:
    for key in ("published", "published-print", "published-online", "issued"):
        dp = (work.get(key) or {}).get("date-parts")
        if dp and dp[0] and dp[0][0]:
            return int(dp[0][0])
    return None


def _normalize_work(work: dict) -> dict:
    titles = work.get("title") or []
    venues = work.get("container-title") or []
    authors = []
    for a in work.get("author", []) or []:
        name = " ".join(p for p in [a.get("given"), a.get("family")] if p).strip()
        if name:
            authors.append(name)
    doi = (work.get("DOI") or "").lower().strip()
    return {
        "title": titles[0] if titles else "",
        "year": _year_from(work),
        "doi": doi,
        "venue": venues[0] if venues else "",
        "authors": authors,
        "type": work.get("type", ""),
    }


def search(query: str, limit: int) -> dict:
    params = {"query.bibliographic": query, "rows": limit, "select": SELECT_FIELDS}
    mailto = _mailto()
    if mailto:
        params["mailto"] = mailto
    url = f"{CROSSREF_BASE}?{urllib.parse.urlencode(params)}"
    resp = _get(url)
    items = (resp.get("message") or {}).get("items") or []
    return {"raw": resp, "data": [_normalize_work(w) for w in items]}


def lookup_doi(doi: str) -> dict:
    url = f"{CROSSREF_BASE}/{urllib.parse.quote(doi)}"
    mailto = _mailto()
    if mailto:
        url += f"?mailto={urllib.parse.quote(mailto)}"
    resp = _get(url)
    msg = resp.get("message")
    data = [_normalize_work(msg)] if msg else []
    return {"raw": resp, "data": data}


def main() -> int:
    p = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    p.add_argument("--query", help="Paper title (bibliographic search)")
    p.add_argument("--doi", help="Look up an exact DOI instead of a title search")
    p.add_argument("--limit", type=int, default=DEFAULT_LIMIT,
                   help=f"Max hits (default {DEFAULT_LIMIT}, max {MAX_LIMIT})")
    p.add_argument("--raw", action="store_true", help="Print full Crossref JSON")
    args = p.parse_args()

    if not args.query and not args.doi:
        print("ERROR: provide --query or --doi", file=sys.stderr)
        return 2

    if args.doi:
        result = lookup_doi(args.doi.lower())
    else:
        result = search(args.query, max(1, min(MAX_LIMIT, args.limit)))

    if args.raw:
        json.dump(result["raw"], sys.stdout, indent=2, ensure_ascii=False)
        sys.stdout.write("\n")
        return 0 if result["data"] else 1

    data = result["data"]
    json.dump({"total": len(data), "data": data}, sys.stdout, indent=2, ensure_ascii=False)
    sys.stdout.write("\n")
    return 0 if data else 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
dedupe_by_id.py — Deduplicate a verified citation pool using Semantic Scholar
unique paperId, with DOI / ArXiv / normalized-title fallbacks.

Implements the paper's Rule 4 (App. D.3): "gathered citations are
deduplicated using unique paper ID keys".

Also computes `min_cite_paper_count = floor(0.9 * len(papers))` for the
≥90% citation integration rule.

Usage:
    python dedupe_by_id.py --in raw_pool.json --out citation_pool.json [--cutoff 2024-10-01]
"""
import argparse
import json
import math
import re
import sys


def norm_title(t: str) -> str:
    return re.sub(r"[^a-z0-9]", "", t.lower())


def make_key(paper: dict) -> str:
    if paper.get("paperId"):
        return f"s2:{paper['paperId']}"
    ext = paper.get("externalIds") or {}
    if ext.get("DOI"):
        return f"doi:{ext['DOI'].lower()}"
    if ext.get("ArXiv"):
        # strip version suffix if any
        a = ext["ArXiv"].split("v")[0] if "v" in ext["ArXiv"][-3:] else ext["ArXiv"]
        return f"arxiv:{a.lower()}"
    title = paper.get("title", "")
    return f"title:{norm_title(title)}"


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--in", dest="inp", required=True, help="Raw verified pool JSON")
    p.add_argument("--out", required=True, help="Deduped citation_pool.json")
    p.add_argument("--cutoff", help="Cutoff date YYYY-MM-DD (recorded in output)")
    args = p.parse_args()

    with open(args.inp) as f:
        raw = json.load(f)

    candidates = raw.get("papers") or raw.get("candidates") or []
    if not candidates:
        print("ERROR: input has neither 'papers' nor 'candidates' key", file=sys.stderr)
        return 1

    by_key: dict[str, dict] = {}
    collisions: list[tuple[str, str]] = []
    for c in candidates:
        key = make_key(c)
        if key in by_key:
            existing = by_key[key]
            score_new = c.get("match_score", 0)
            score_old = existing.get("match_score", 0)
            if score_new > score_old:
                # merge discovered_for
                merged = existing.get("discovered_for", []) + c.get("discovered_for", [])
                c["discovered_for"] = list(dict.fromkeys(merged))  # preserve order, dedupe
                by_key[key] = c
            else:
                merged = existing.get("discovered_for", []) + c.get("discovered_for", [])
                existing["discovered_for"] = list(dict.fromkeys(merged))
            collisions.append((key, c.get("title", "")))
        else:
            by_key[key] = c

    deduped = list(by_key.values())
    n = len(deduped)
    min_cite = math.floor(0.9 * n)

    out = {
        "papers": deduped,
        "min_cite_paper_count": min_cite,
        "n_total": n,
        "n_collisions_merged": len(collisions),
    }
    if args.cutoff:
        out["cutoff_date"] = args.cutoff

    with open(args.out, "w") as f:
        json.dump(out, f, indent=2, ensure_ascii=False)

    print(f"OK: {len(candidates)} candidates → {n} unique papers")
    print(f"    {len(collisions)} duplicates merged")
    print(f"    min_cite_paper_count (≥90%): {min_cite}")
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
exa_search.py — Optional Exa (https://exa.ai) backend for the literature
review agent's Phase 1 (parallel candidate discovery) step.

Exa is a search engine optimized for finding academic papers and other
high-quality content. It is OPTIONAL — the literature-review-agent works
fine with any host coding agent's native web search tool. Use Exa only if:

  - Your host has no built-in web search (e.g., Aider, OpenCode, generic
    CLI agents).
  - You want a research-paper-focused search backend with better
    signal-to-noise than general web search.
  - You're running the pipeline in batch / non-interactive mode and want
    a deterministic, scriptable backend.

This helper reads EXA_API_KEY from the environment. The key is YOUR
responsibility to provide; this repo never commits one. Get a key at
https://dashboard.exa.ai/.

Usage:
    export EXA_API_KEY="your-key-here"
    python exa_search.py --query "Sparse attention long context" --num-results 15
    python exa_search.py --query "..." --raw                       # full JSON
    python exa_search.py --query "..." --discovered-for "related_work[2.1]"

Default output: JSON candidates in the literature-review-agent format, ready
to be merged into raw_candidates.json before Phase 2 verification.

Exit codes:
    0  query succeeded
    1  EXA_API_KEY missing, HTTP error, network error, or empty results
"""
import argparse
import json
import os
import sys
import urllib.error
import urllib.request

EXA_ENDPOINT = "https://api.exa.ai/search"
DEFAULT_NUM = 10
MAX_NUM = 20      # the user explicitly asked for a 10-20 range
SNIPPET_CAP = 1500


def search(query: str, num_results: int, category: str | None,
           highlight_max_chars: int) -> dict:
    api_key = os.environ.get("EXA_API_KEY")
    if not api_key:
        print(
            "ERROR: EXA_API_KEY environment variable not set.\n"
            "Get a key at https://dashboard.exa.ai/ and run:\n"
            '  export EXA_API_KEY="your-key-here"\n'
            "Then retry. The literature-review-agent also works without\n"
            "Exa — see references/discovery-pipeline.md for the default\n"
            "host-native web search path.",
            file=sys.stderr,
        )
        sys.exit(1)

    body: dict = {
        "query":      query,
        "numResults": num_results,
        "type":       "auto",
        "contents":   {"highlights": {"maxCharacters": highlight_max_chars}},
    }
    if category:
        body["category"] = category

    req = urllib.request.Request(
        EXA_ENDPOINT,
        data=json.dumps(body).encode("utf-8"),
        headers={
            "content-type": "application/json",
            "x-api-key":    api_key,
        },
        method="POST",
    )
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read().decode("utf-8"))
    except urllib.error.HTTPError as e:
        body_text = e.read().decode("utf-8", errors="replace")[:500]
        print(f"ERROR: Exa HTTP {e.code}: {body_text}", file=sys.stderr)
        sys.exit(1)
    except urllib.error.URLError as e:
        print(f"ERROR: Exa network error: {e.reason}", file=sys.stderr)
        sys.exit(1)


def normalize(exa_response: dict, discovered_for: list[str]) -> list[dict]:
    """Convert Exa results into the literature-review-agent candidate format."""
    candidates: list[dict] = []
    for r in exa_response.get("results", []):
        title = (r.get("title") or "").strip()
        url = r.get("url") or r.get("id") or ""
        highlights = r.get("highlights") or []
        snippet = " ".join(h.strip() for h in highlights)[:SNIPPET_CAP]
        candidates.append({
            "title":          title,
            "snippet":        snippet,
            "source_url":     url,
            "discovered_for": list(discovered_for),
            "_exa_id":             r.get("id"),
            "_exa_published_date": r.get("publishedDate"),
        })
    return candidates


def main() -> int:
    p = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )
    p.add_argument("--query", required=True, help="Search query")
    p.add_argument("--num-results", type=int, default=DEFAULT_NUM,
                   help=f"Number of results to fetch "
                        f"(default {DEFAULT_NUM}, clamped to [1, {MAX_NUM}])")
    p.add_argument("--category", default="research paper",
                   help='Exa category filter (default "research paper"; '
                        'pass an empty string to disable)')
    p.add_argument("--highlight-chars", type=int, default=4000,
                   help="Max characters per highlight (default 4000)")
    p.add_argument("--discovered-for", default="intro",
                   help='Tag to attach to each candidate '
                        '(default "intro"). Use "related_work[2.1]" or '
                        'similar for cluster-specific queries so the '
                        'downstream citation_coverage gate can attribute '
                        'the citation to the right section.')
    p.add_argument("--raw", action="store_true",
                   help="Print the full Exa response JSON unmodified "
                        "instead of normalized candidates")
    args = p.parse_args()

    n = max(1, min(MAX_NUM, args.num_results))
    category = args.category or None

    response = search(args.query, n, category, args.highlight_chars)
    if not response.get("results"):
        print(f"WARN: Exa returned 0 results for query: {args.query!r}",
              file=sys.stderr)
        return 1

    if args.raw:
        json.dump(response, sys.stdout, indent=2, ensure_ascii=False)
    else:
        candidates = normalize(response, [args.discovered_for])
        json.dump({"candidates": candidates}, sys.stdout, indent=2,
                  ensure_ascii=False)
    sys.stdout.write("\n")
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
levenshtein_match.py — Fuzzy title match for citation verification.

Implements the paper's Rule 1 (App. D.3): a candidate paper passes only if
its title's Levenshtein ratio against the Semantic Scholar hit's title is
strictly greater than 70.

Includes a substring-bypass safety net for short candidate titles (the
Linformer false-negative case): if the candidate is < 4 words and is
contained as a substring in the S2 hit's title, return 100.

Exit code is always 0; the integer ratio is printed to stdout. The caller
parses it and decides whether to discard.

Usage:
    python levenshtein_match.py --candidate "..." --found "..."
    python levenshtein_match.py --candidate "..." --found "..." --substring-bypass
"""
import argparse
import re
import sys

try:
    import Levenshtein
except ImportError:
    print("ERROR: python-Levenshtein required. Install with: pip install python-Levenshtein",
          file=sys.stderr)
    sys.exit(2)


def normalize(s: str) -> str:
    s = s.lower().strip()
    s = re.sub(r"[^a-z0-9\s]", " ", s)
    s = re.sub(r"\s+", " ", s)
    return s


def ratio(a: str, b: str, substring_bypass: bool = False) -> int:
    na, nb = normalize(a), normalize(b)
    r = int(round(Levenshtein.ratio(na, nb) * 100))
    if substring_bypass and len(na.split()) < 4:
        if na in nb:
            return max(r, 95)
    return r


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--candidate", required=True,
                   help="The original candidate title (from web search)")
    p.add_argument("--found", required=True,
                   help="The title returned by Semantic Scholar")
    p.add_argument("--substring-bypass", action="store_true",
                   help="Bump short-candidate substring matches to 95")
    p.add_argument("--threshold", type=int, default=70,
                   help="Print PASS/FAIL alongside the ratio (default 70)")
    args = p.parse_args()

    r = ratio(args.candidate, args.found, args.substring_bypass)
    verdict = "PASS" if r > args.threshold else "FAIL"
    print(f"{r} {verdict}")
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
openalex_client.py — OpenAlex API title/DOI lookup for cross-index citation
corroboration.

The third index (alongside Semantic Scholar and Crossref) used by
cross_verify.py. Triangulating across three independent scholarly indices is
the practical defense against hallucinated citations: a fabricated paper may
slip past one index but is unlikely to appear in all three with matching
metadata. See references/cross-index-verification.md.

No API key is required. OpenAlex offers a faster "polite pool" when you
identify yourself via email. Set one of:
    export OPENALEX_MAILTO="you@example.com"
    export PAPER_ORCHESTRA_MAILTO="you@example.com"   # shared fallback
The email is sent only as the `mailto` query parameter, per OpenAlex docs.

Usage:
    python openalex_client.py --query "Attention is All You Need"
    python openalex_client.py --doi 10.5555/3295222.3295349
    python openalex_client.py --query "BERT pre-training" --raw

Output (normalized): {"total": N, "data": [{title, year, doi, venue, authors}, ...]}

Exit codes:
    0  at least one result returned
    1  HTTP error, network error, or zero results
    2  usage error (bad arguments)
"""
import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

OPENALEX_BASE = "https://api.openalex.org/works"
DEFAULT_LIMIT = 5
MAX_LIMIT = 25
_RETRY_SLEEP = 5


def _mailto() -> str:
    return (
        os.environ.get("OPENALEX_MAILTO", "").strip()
        or os.environ.get("PAPER_ORCHESTRA_MAILTO", "").strip()
    )


def _build_request(url: str) -> urllib.request.Request:
    return urllib.request.Request(
        url,
        headers={
            "Accept": "application/json",
            "User-Agent": "paper-orchestra/1.0 (https://github.com/Ar9av/paper-orchestra)",
        },
        method="GET",
    )


def _get(url: str, retries: int = 3) -> dict:
    for attempt in range(1, retries + 1):
        try:
            with urllib.request.urlopen(_build_request(url), timeout=30) as resp:
                return json.loads(resp.read().decode("utf-8"))
        except urllib.error.HTTPError as exc:
            if exc.code == 404:
                return {"results": []}
            if exc.code == 429 and attempt < retries:
                print(f"WARN: OpenAlex rate-limited (429). Sleeping {_RETRY_SLEEP}s "
                      f"before retry {attempt + 1}/{retries}.", file=sys.stderr)
                time.sleep(_RETRY_SLEEP)
                continue
            if exc.code in (500, 502, 503) and attempt < retries:
                print(f"WARN: OpenAlex server error ({exc.code}). Retrying.", file=sys.stderr)
                time.sleep(10)
                continue
            print(f"ERROR: OpenAlex HTTP {exc.code}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as exc:
            print(f"ERROR: Network error reaching OpenAlex: {exc.reason}", file=sys.stderr)
            sys.exit(1)
    sys.exit(1)


def _bare_doi(doi_url: str | None) -> str:
    if not doi_url:
        return ""
    d = doi_url.strip().lower()
    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
        if d.startswith(prefix):
            d = d[len(prefix):]
    return d


def _normalize_work(work: dict) -> dict:
    venue = ""
    loc = work.get("primary_location") or {}
    src = loc.get("source") or {}
    if src.get("display_name"):
        venue = src["display_name"]
    authors = []
    for a in work.get("authorships", []) or []:
        name = (a.get("author") or {}).get("display_name")
        if name:
            authors.append(name)
    return {
        "title": work.get("title") or work.get("display_name") or "",
        "year": work.get("publication_year"),
        "doi": _bare_doi(work.get("doi")),
        "venue": venue,
        "authors": authors,
        "type": work.get("type", ""),
    }


def _with_mailto(params: dict) -> dict:
    mailto = _mailto()
    if mailto:
        params["mailto"] = mailto
    return params


def search(query: str, limit: int) -> dict:
    params = _with_mailto({"search": query, "per-page": limit})
    url = f"{OPENALEX_BASE}?{urllib.parse.urlencode(params)}"
    resp = _get(url)
    results = resp.get("results") or []
    return {"raw": resp, "data": [_normalize_work(w) for w in results]}


def lookup_doi(doi: str) -> dict:
    # OpenAlex resolves a single work via the /works/doi:<doi> path.
    path = f"{OPENALEX_BASE}/doi:{urllib.parse.quote(doi)}"
    mailto = _mailto()
    if mailto:
        path += f"?mailto={urllib.parse.quote(mailto)}"
    resp = _get(path)
    # A single-work response is the work object itself, not a results list.
    if "results" in resp:
        works = resp["results"]
    elif resp.get("id"):
        works = [resp]
    else:
        works = []
    return {"raw": resp, "data": [_normalize_work(w) for w in works]}


def main() -> int:
    p = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    p.add_argument("--query", help="Paper title (full-text search)")
    p.add_argument("--doi", help="Look up an exact DOI instead of a title search")
    p.add_argument("--limit", type=int, default=DEFAULT_LIMIT,
                   help=f"Max hits (default {DEFAULT_LIMIT}, max {MAX_LIMIT})")
    p.add_argument("--raw", action="store_true", help="Print full OpenAlex JSON")
    args = p.parse_args()

    if not args.query and not args.doi:
        print("ERROR: provide --query or --doi", file=sys.stderr)
        return 2

    if args.doi:
        result = lookup_doi(args.doi.lower())
    else:
        result = search(args.query, max(1, min(MAX_LIMIT, args.limit)))

    if args.raw:
        json.dump(result["raw"], sys.stdout, indent=2, ensure_ascii=False)
        sys.stdout.write("\n")
        return 0 if result["data"] else 1

    data = result["data"]
    json.dump({"total": len(data), "data": data}, sys.stdout, indent=2, ensure_ascii=False)
    sys.stdout.write("\n")
    return 0 if data else 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""
pre_dedup_candidates.py — Deduplicate Phase 1 raw candidates by normalized
title before Phase 2 Semantic Scholar verification.

Multiple search queries in Phase 1 often return the same papers. Verifying
duplicates wastes S2 quota (1 QPS hard cap) and adds 30-40% unnecessary
wall-time. This script removes obvious duplicates — same paper found via
multiple queries — before the sequential verification loop begins.

Dedup strategy (in order of preference):
1. Exact arXiv ID match extracted from source URL or snippet.
2. Levenshtein ratio >= 92 on normalized titles (high threshold to avoid
   false collisions between similarly-named papers).

When two candidates are considered the same, we keep the one that appeared
earlier in the list and merge their `discovered_for` attribution tags so
the surviving entry is credited to all originating queries.

Usage:
    python pre_dedup_candidates.py \\
        --in workspace/raw_candidates.json \\
        --out workspace/deduped_candidates.json

Input JSON shape:
    {"candidates": [{"title": "...", "url": "...", "snippet": "...",
                     "discovered_for": ["intro.1"]}, ...]}
    OR a bare list.
"""
import argparse
import json
import re
import sys

ARXIV_RE = re.compile(r"arxiv\.org/(?:abs|pdf)/(\d{4}\.\d{4,5})", re.IGNORECASE)


def norm_title(t: str) -> str:
    t = re.sub(r"[^a-z0-9 ]", " ", t.lower())
    return " ".join(t.split())


def levenshtein_ratio(a: str, b: str) -> float:
    if not a and not b:
        return 100.0
    if not a or not b:
        return 0.0
    la, lb = len(a), len(b)
    if la < lb:
        a, b = b, a
        la, lb = lb, la
    prev = list(range(lb + 1))
    for i, ca in enumerate(a):
        curr = [i + 1]
        for j, cb in enumerate(b):
            cost = 0 if ca == cb else 1
            curr.append(min(prev[j + 1] + 1, curr[j] + 1, prev[j] + cost))
        prev = curr
    dist = prev[lb]
    return (1.0 - dist / max(la, lb)) * 100.0


def extract_arxiv_id(candidate: dict) -> str | None:
    for text in (candidate.get("url", ""), candidate.get("snippet", "")):
        m = ARXIV_RE.search(text)
        if m:
            return m.group(1)
    return None


def make_exact_key(candidate: dict) -> str:
    """Canonical key: arXiv ID if extractable, else normalized title."""
    aid = extract_arxiv_id(candidate)
    if aid:
        return f"arxiv:{aid}"
    return f"title:{norm_title(candidate.get('title', ''))}"


def merge_discovered_for(a: dict, b: dict) -> list:
    df_a = a.get("discovered_for") or []
    df_b = b.get("discovered_for") or []
    return list(dict.fromkeys(df_a + df_b))


def dedup(candidates: list[dict], title_ratio_threshold: float = 92.0) -> list[dict]:
    # Pass 1: exact key dedup (arXiv ID or identical normalized title)
    by_key: dict[str, dict] = {}
    for c in candidates:
        key = make_exact_key(c)
        if key in by_key:
            by_key[key]["discovered_for"] = merge_discovered_for(by_key[key], c)
        else:
            by_key[key] = dict(c)

    deduped = list(by_key.values())

    # Pass 2: fuzzy title dedup — O(n²) but n is ~50-100 candidates max
    normed = [norm_title(c.get("title", "")) for c in deduped]
    drop: set[int] = set()
    for i in range(len(deduped)):
        if i in drop:
            continue
        for j in range(i + 1, len(deduped)):
            if j in drop:
                continue
            if levenshtein_ratio(normed[i], normed[j]) >= title_ratio_threshold:
                deduped[i]["discovered_for"] = merge_discovered_for(deduped[i], deduped[j])
                drop.add(j)

    return [c for idx, c in enumerate(deduped) if idx not in drop]


def main() -> int:
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument("--in", dest="inp", required=True, help="Raw Phase 1 candidates JSON")
    p.add_argument("--out", required=True, help="Deduped candidates JSON")
    p.add_argument("--title-ratio", type=float, default=92.0,
                   help="Levenshtein ratio threshold for fuzzy title match (default: 92)")
    args = p.parse_args()

    with open(args.inp) as f:
        raw = json.load(f)

    if isinstance(raw, list):
        candidates = raw
    else:
        candidates = raw.get("candidates") or raw.get("papers") or []

    if not isinstance(candidates, list):
        print("ERROR: input must be a JSON array or object with 'candidates' key",
              file=sys.stderr)
        return 1

    before = len(candidates)
    result = dedup(candidates, title_ratio_threshold=args.title_ratio)
    after = len(result)
    removed = before - after

    out_obj = {
        "candidates": result,
        "n_before_dedup": before,
        "n_after_dedup": after,
        "n_removed": removed,
    }
    with open(args.out, "w") as f:
        json.dump(out_obj, f, indent=2, ensure_ascii=False)

    print(f"OK: {before} candidates → {after} unique ({removed} duplicates removed)")
    return 0


if __name__ == "__main__":
    sys.exit(main())