Search Lit

#!/usr/bin/env python3
"""
Parse PubMed E-utilities responses into structured data.

Usage:
    # Parse esearch JSON → list of PMIDs
    echo '<json>' | python3 parse_pubmed.py esearch

    # Parse esummary JSON → markdown table
    echo '<json>' | python3 parse_pubmed.py esummary

    # Parse efetch XML → detailed metadata (for BibTeX generation)
    echo '<xml>' | python3 parse_pubmed.py efetch

    # Parse efetch XML → BibTeX entries
    echo '<xml>' | python3 parse_pubmed.py bibtex
"""

import sys
import json
import re
import xml.etree.ElementTree as ET
from datetime import date
from textwrap import shorten


# Heuristic for East Asian name reverse-encoding in PubMed XML.
# Cases observed: <LastName>Qiaoling</LastName><ForeName>Fu</ForeName> where
# Fu is the actual family name. Pattern: LastName looks like a long given
# name (≥3 alpha chars, no spaces) AND ForeName looks like a short surname
# fragment (1-2 chars, no period). The naive test catches the common reverse
# encoding without flagging legitimate short-surname authors.
_EAST_ASIAN_REVERSE_THRESHOLD = 3  # LastName length lower bound for suspicion


def _looks_east_asian_reversed(last: str, fore: str) -> bool:
    """Return True if (LastName, ForeName) look swapped per PubMed encoding bug."""
    if not last or not fore:
        return False
    # ForeName should look like a surname (1-2 chars, no spaces, no period)
    # AND LastName should look like a multi-char given name.
    return (
        1 <= len(fore) <= 2
        and fore.isalpha()
        and "." not in fore
        and len(last) >= _EAST_ASIAN_REVERSE_THRESHOLD
        and last.isalpha()
    )


def _extract_authors(author_list_el):
    """Walk an <AuthorList> element. Return (bib_authors, display_authors,
    first_author_last, suspicions, has_collective_only).

    - bib_authors: list of "Family, Given" strings for BibTeX `author = {...}`.
      Corporate (<CollectiveName>) authors are double-braced.
    - display_authors: list of "Last First" strings for human-readable output.
    - first_author_last: surname of the first listed author (used for cite key).
    - suspicions: list of human-readable warning strings (East Asian reverse
      encoding, missing LastName, etc.).
    - has_collective_only: True if AuthorList contains only <CollectiveName>
      entries (no individual <LastName>). Caller should consider emitting
      `@misc` instead of `@article` (guideline / consortium pattern).
    """
    bib_authors: list[str] = []
    display_authors: list[str] = []
    first_author_last = ""
    suspicions: list[str] = []
    individual_count = 0
    collective_count = 0

    if author_list_el is None:
        return bib_authors, display_authors, "", suspicions, False

    for au in author_list_el.findall("Author"):
        last = au.findtext("LastName", "") or ""
        fore = au.findtext("ForeName", "") or ""
        collective = au.findtext("CollectiveName", "") or ""

        if collective:
            collective_count += 1
            # Double-brace to prevent BibTeX from splitting on the comma /
            # spaces inside the corporate name.
            bib_authors.append("{" + collective + "}")
            display_authors.append(collective)
            if not first_author_last:
                first_author_last = re.sub(r"[^A-Za-z]+", "", collective.split()[0]) or "Group"
            continue

        if last:
            individual_count += 1
            if _looks_east_asian_reversed(last, fore):
                suspicions.append(
                    f"East Asian name order suspected for '{last} {fore}' — "
                    "PubMed XML may have LastName/ForeName swapped"
                )
            bib_authors.append(f"{last}, {fore}")
            display_authors.append(f"{last} {fore}".strip())
            if not first_author_last:
                first_author_last = last
            continue

        # Author element with neither <LastName> nor <CollectiveName>: rare
        # but possible. Record as suspicion, otherwise skip.
        suspicions.append("Author element with no LastName and no CollectiveName")

    has_collective_only = collective_count > 0 and individual_count == 0
    return bib_authors, display_authors, first_author_last, suspicions, has_collective_only


def parse_esearch(data: str) -> None:
    """Parse esearch JSON response, print PMIDs and count."""
    result = json.loads(data)
    esearch = result.get("esearchresult", {})
    count = esearch.get("count", "0")
    ids = esearch.get("idlist", [])
    print(f"Total results: {count}")
    print(f"Returned: {len(ids)}")
    print(f"PMIDs: {','.join(ids)}")


def parse_esummary(data: str) -> None:
    """Parse esummary JSON response into a markdown table."""
    result = json.loads(data)
    docs = result.get("result", {})
    uids = docs.get("uids", [])

    if not uids:
        print("No results found.")
        return

    print("| # | PMID | Year | Journal | Title | Authors |")
    print("|---|------|------|---------|-------|---------|")

    for i, uid in enumerate(uids, 1):
        doc = docs.get(uid, {})
        title = shorten(doc.get("title", "N/A"), width=80, placeholder="...")
        authors_raw = doc.get("authors", [])
        if authors_raw:
            first = authors_raw[0].get("name", "")
            last = authors_raw[-1].get("name", "") if len(authors_raw) > 1 else ""
            authors = f"{first}, ... {last}" if last and last != first else first
        else:
            authors = "N/A"
        journal = shorten(doc.get("fulljournalname", doc.get("source", "N/A")),
                          width=40, placeholder="...")
        pubdate = doc.get("pubdate", "N/A")
        year = pubdate[:4] if pubdate else "N/A"
        doi_list = doc.get("articleids", [])
        doi = next((d["value"] for d in doi_list if d.get("idtype") == "doi"), "")

        print(f"| {i} | {uid} | {year} | {journal} | {title} | {authors} |")

    print(f"\n*{len(uids)} articles retrieved*")


def parse_efetch(data: str) -> None:
    """Parse efetch XML response into structured metadata."""
    root = ET.fromstring(data)
    articles = root.findall(".//PubmedArticle")

    for article in articles:
        medline = article.find("MedlineCitation")
        if medline is None:
            continue

        pmid = medline.findtext("PMID", "N/A")
        art = medline.find("Article")
        if art is None:
            continue

        title = art.findtext("ArticleTitle", "N/A")
        journal_el = art.find("Journal")
        journal = journal_el.findtext("Title", "N/A") if journal_el is not None else "N/A"
        journal_abbrev = journal_el.findtext("ISOAbbreviation", "") if journal_el is not None else ""

        # Year
        ji = journal_el.find("JournalIssue") if journal_el is not None else None
        pd = ji.find("PubDate") if ji is not None else None
        year = pd.findtext("Year", "") if pd is not None else ""
        if not year:
            medline_date = pd.findtext("MedlineDate", "") if pd is not None else ""
            year = medline_date[:4] if medline_date else "N/A"

        volume = ji.findtext("Volume", "") if ji is not None else ""
        issue = ji.findtext("Issue", "") if ji is not None else ""

        # Pages
        pages = art.findtext("Pagination/MedlinePgn", "")

        # Authors (handles East Asian reverse encoding + CollectiveName)
        author_list = art.find("AuthorList")
        _, authors, _, suspicions, _ = _extract_authors(author_list)

        # DOI
        doi = ""
        for aid in art.findall("ELocationID"):
            if aid.get("EIdType") == "doi":
                doi = aid.text or ""

        # Abstract
        abstract_el = art.find("Abstract")
        abstract = ""
        if abstract_el is not None:
            parts = abstract_el.findall("AbstractText")
            abstract = " ".join(
                (p.get("Label", "") + ": " if p.get("Label") else "") + (p.text or "")
                for p in parts
            )

        print(f"## PMID: {pmid}")
        print(f"**Title**: {title}")
        print(f"**Authors**: {'; '.join(authors)}")
        print(f"**Journal**: {journal} ({journal_abbrev})")
        print(f"**Year**: {year}  **Volume**: {volume}  **Issue**: {issue}  **Pages**: {pages}")
        print(f"**DOI**: {doi}")
        if abstract:
            print(f"**Abstract**: {shorten(abstract, width=500, placeholder='...')}")
        for note in suspicions:
            print(f"> ⚠ {note}")
        print()


def generate_bibtex(data: str) -> None:
    """Parse efetch XML and generate BibTeX entries."""
    root = ET.fromstring(data)
    articles = root.findall(".//PubmedArticle")

    for article in articles:
        medline = article.find("MedlineCitation")
        if medline is None:
            continue

        pmid = medline.findtext("PMID", "")
        art = medline.find("Article")
        if art is None:
            continue

        title = art.findtext("ArticleTitle", "")
        journal_el = art.find("Journal")
        journal_abbrev = journal_el.findtext("ISOAbbreviation", "") if journal_el is not None else ""
        journal_full = journal_el.findtext("Title", "") if journal_el is not None else ""

        ji = journal_el.find("JournalIssue") if journal_el is not None else None
        pd = ji.find("PubDate") if ji is not None else None
        year = pd.findtext("Year", "") if pd is not None else ""
        if not year:
            md = pd.findtext("MedlineDate", "") if pd is not None else ""
            year = md[:4] if md else ""

        volume = ji.findtext("Volume", "") if ji is not None else ""
        issue = ji.findtext("Issue", "") if ji is not None else ""
        pages = art.findtext("Pagination/MedlinePgn", "")

        doi = ""
        for aid in art.findall("ELocationID"):
            if aid.get("EIdType") == "doi":
                doi = aid.text or ""

        author_list = art.find("AuthorList")
        bib_authors, _, first_author_last, suspicions, has_collective_only = \
            _extract_authors(author_list)

        # Generate citation key
        key = f"{first_author_last}_{year}_{pmid}" if first_author_last else f"PMID_{pmid}"

        # Corporate / consortium guideline (e.g., KDIGO, AHA/ACC) — emit as
        # @misc so BibTeX styles render the body of the entry without trying
        # to format a personal author. Vancouver / AMA CSL handle both
        # @article and @misc with author = {{Organization Name}}.
        entry_type = "misc" if has_collective_only else "article"

        # Prepend suspicion comments so they survive .bib copy/paste audits.
        for note in suspicions:
            print(f"% [VERIFY] {note}")

        print(f"@{entry_type}{{{key},")
        print(f"  author    = {{{' and '.join(bib_authors)}}},")
        print(f"  title     = {{{title}}},")
        print(f"  journal   = {{{journal_full}}},")
        print(f"  year      = {{{year}}},")
        if volume:
            print(f"  volume    = {{{volume}}},")
        if issue:
            print(f"  number    = {{{issue}}},")
        if pages:
            print(f"  pages     = {{{pages}}},")
        if doi:
            print(f"  doi       = {{{doi}}},")
        print(f"  pmid      = {{{pmid}}},")

        # Anti-hallucination verification flag. Entries emitted by this script
        # originate from PubMed efetch XML, so a non-empty PMID is proof of
        # API provenance (verified=true). Missing PMID → verified=false and
        # downstream tooling (/verify-refs) will flag for manual check.
        verified = bool(pmid)
        verified_by = "pubmed+crossref" if (pmid and doi) else ("pubmed" if pmid else "")
        print(f"  verified  = {{{'true' if verified else 'false'}}},")
        if verified_by:
            print(f"  verified_by = {{{verified_by}}},")
            print(f"  verified_on = {{{date.today().isoformat()}}},")
        print("}")
        print()


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    mode = sys.argv[1]
    data = sys.stdin.read()

    dispatch = {
        "esearch": parse_esearch,
        "esummary": parse_esummary,
        "efetch": parse_efetch,
        "bibtex": generate_bibtex,
    }

    func = dispatch.get(mode)
    if func is None:
        print(f"Unknown mode: {mode}. Use: {', '.join(dispatch.keys())}")
        sys.exit(1)

    func(data)

#!/bin/bash
# PubMed E-utilities CLI wrapper
# Fallback when PubMed MCP server is unavailable
# Usage: bash pubmed_eutils.sh <command> <args...>
#
# Commands:
#   search <query> [retmax]       -- Search PubMed, return PMIDs
#   fetch <pmid1,pmid2,...>       -- Fetch article metadata (XML)
#   fetch_json <pmid1,pmid2,...>  -- Fetch article summary (JSON, DocSum)
#   related <pmid> [retmax]       -- Find related articles
#   cite_lookup <title>           -- Search by exact title to verify citation
#
# Environment:
#   NCBI_API_KEY  -- Optional. Increases rate limit from 3/sec to 10/sec.
#                    Register at https://www.ncbi.nlm.nih.gov/account/settings/
#
# Rate limiting: 3 requests/second without API key, 10/sec with key.
# This script sleeps 350ms between calls for safety.

set -euo pipefail

BASE="https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
TOOL="claude-code-search-lit"
EMAIL="noreply@example.com"
DB="pubmed"
SLEEP=0.35

# Build API key param if available
API_KEY_PARAM=""
if [ -n "${NCBI_API_KEY:-}" ]; then
  API_KEY_PARAM="&api_key=${NCBI_API_KEY}"
  SLEEP=0.1
fi

_sleep() { sleep "$SLEEP"; }

_curl() {
  local http_code body
  body=$(curl -sS -w '\n%{http_code}' -A "Mozilla/5.0 (${TOOL})" "$@")
  http_code=$(echo "$body" | tail -n1)
  body=$(echo "$body" | sed '$d')
  if [ "$http_code" -ge 400 ] 2>/dev/null; then
    echo "{\"error\": \"HTTP ${http_code}\", \"url\": \"$1\"}" >&2
    return 1
  fi
  echo "$body"
}

cmd_search() {
  local query="${1:?Usage: search <query> [retmax]}"
  local retmax="${2:-20}"
  local url="${BASE}/esearch.fcgi?db=${DB}&term=$(python3 -c "import urllib.parse; print(urllib.parse.quote('${query}'))")&retmax=${retmax}&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_fetch() {
  local ids="${1:?Usage: fetch <pmid1,pmid2,...>}"
  local url="${BASE}/efetch.fcgi?db=${DB}&id=${ids}&rettype=xml&retmode=xml&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_fetch_json() {
  local ids="${1:?Usage: fetch_json <pmid1,pmid2,...>}"
  local url="${BASE}/esummary.fcgi?db=${DB}&id=${ids}&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_related() {
  local pmid="${1:?Usage: related <pmid> [retmax]}"
  local retmax="${2:-10}"
  local url="${BASE}/elink.fcgi?dbfrom=${DB}&db=${DB}&id=${pmid}&cmd=neighbor_score&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  local result
  result=$(_curl "$url")
  # Extract linked PMIDs and fetch their summaries
  local linked_ids
  linked_ids=$(echo "$result" | python3 -c "
import sys, json
data = json.load(sys.stdin)
links = data.get('linksets', [{}])[0].get('linksetdbs', [{}])
for db in links:
    if db.get('linkname') == 'pubmed_pubmed':
        ids = [str(l['id']) for l in db.get('links', [])[:${retmax}]]
        print(','.join(ids))
        break
" 2>/dev/null || echo "")
  if [ -n "$linked_ids" ]; then
    _sleep
    cmd_fetch_json "$linked_ids"
  else
    echo '{"error": "No related articles found"}'
  fi
}

cmd_cite_lookup() {
  local title="${1:?Usage: cite_lookup <title>}"
  # Search by title field for exact verification
  cmd_search "${title}[Title]" 5
}

# Dispatch
case "${1:-help}" in
  search)      shift; cmd_search "$@" ;;
  fetch)       shift; cmd_fetch "$@" ;;
  fetch_json)  shift; cmd_fetch_json "$@" ;;
  related)     shift; cmd_related "$@" ;;
  cite_lookup) shift; cmd_cite_lookup "$@" ;;
  help|*)
    echo "Usage: bash pubmed_eutils.sh <command> <args...>"
    echo "Commands: search, fetch, fetch_json, related, cite_lookup"
    ;;
esac

@article{Researcher_2019_Synthetic,
  author    = {Researcher, Ada and Coauthor, Ben},
  title     = {Synthetic Backward Paper One},
  journal   = {Journal of Synthetic Radiology},
  year      = {2019},
  doi       = {10.0/back-one},
  pmid      = {30000001},
  verified  = {false},
  verified_by = {semantic_scholar},
  verified_on = {2026-06-14},
  source    = {citation_snowball},
  snowball_direction = {backward},
}

@article{Writer_2022_Synthetic,
  author    = {Writer, Dan},
  title     = {Synthetic Forward Paper Alpha},
  journal   = {AI in Synthetic Medicine},
  year      = {2022},
  doi       = {10.0/fwd-alpha},
  pmid      = {30000003},
  verified  = {false},
  verified_by = {semantic_scholar},
  verified_on = {2026-06-14},
  source    = {citation_snowball},
  snowball_direction = {forward},
}

@article{Scholar_2023_Synthetic,
  author    = {Scholar, Eve and Helper, Finn},
  title     = {Synthetic Forward Paper Beta},
  journal   = {Synthetic Cardiology},
  year      = {2023},
  doi       = {10.0/fwd-beta},
  verified  = {false},
  verified_by = {semantic_scholar},
  verified_on = {2026-06-14},
  source    = {citation_snowball},
  snowball_direction = {forward},
}

@article{Expert_2021_Synthetic,
  author    = {Expert, Gail},
  title     = {Synthetic Similar Recommendation},
  journal   = {Synthetic Methods},
  year      = {2021},
  doi       = {10.0/sim-one},
  pmid      = {30000005},
  verified  = {false},
  verified_by = {semantic_scholar},
  verified_on = {2026-06-14},
  source    = {citation_snowball},
  snowball_direction = {similar},
}

{
  "data": [
    {
      "citedPaper": {
        "title": "Synthetic Backward Paper One",
        "year": 2019,
        "venue": "Journal of Synthetic Radiology",
        "externalIds": {"DOI": "10.0/back-one", "PubMed": "30000001"},
        "authors": [{"name": "Ada Researcher"}, {"name": "Ben Coauthor"}]
      }
    },
    {
      "citedPaper": {
        "title": "Synthetic Duplicate In Pool",
        "year": 2018,
        "venue": "Synthetic Imaging",
        "externalIds": {"DOI": "10.0/dup-backward", "PubMed": "30000002"},
        "authors": [{"name": "Cara Author"}]
      }
    }
  ]
}

{
  "data": [
    {
      "citingPaper": {
        "title": "Synthetic Forward Paper Alpha",
        "year": 2022,
        "venue": "AI in Synthetic Medicine",
        "externalIds": {"DOI": "10.0/fwd-alpha", "PubMed": "30000003"},
        "authors": [{"name": "Dan Writer"}]
      }
    },
    {
      "citingPaper": {
        "title": "Synthetic Forward Paper Beta",
        "year": 2023,
        "venue": "Synthetic Cardiology",
        "externalIds": {"DOI": "10.0/fwd-beta"},
        "authors": [{"name": "Eve Scholar"}, {"name": "Finn Helper"}]
      }
    }
  ]
}

{
  "recommendedPapers": [
    {
      "title": "Synthetic Similar Recommendation",
      "year": 2021,
      "venue": "Synthetic Methods",
      "externalIds": {"DOI": "10.0/sim-one", "PubMed": "30000005"},
      "authors": [{"name": "Gail Expert"}]
    }
  ]
}

@article{Cara_2018_Duplicate,
  author    = {Author, Cara},
  title     = {Synthetic Duplicate In Pool},
  journal   = {Synthetic Imaging},
  year      = {2018},
  doi       = {10.0/dup-backward},
  verified  = {true},
  verified_by = {pubmed},
  verified_on = {2026-06-01},
}

#!/usr/bin/env bash
# Deterministic verifier for the citation-snowballing challenge card.
# No network: reads recorded Semantic Scholar responses from fixture/.
# Exit 0 = output matches expected/snowball.bib ; non-zero = regression.
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"
SNOWBALL="$HERE/../snowball.py"

actual="$(python3 "$SNOWBALL" \
  --seed DOI:10.0/seed1 \
  --direction all \
  --offline-fixture "$HERE/fixture" \
  --pool "$HERE/fixture/library.bib" \
  --as-of 2026-06-14 \
  --stdout 2>/dev/null)"

if diff -u "$HERE/expected/snowball.bib" <(printf '%s\n' "$actual"); then
  echo "PASS: snowball output matches expected (4 new candidates; 1 backward dup removed)."
else
  echo "FAIL: snowball output drifted from expected/snowball.bib" >&2
  exit 1
fi

#!/usr/bin/env python3
"""
Citation snowballing for search-lit (Phase 2.5 Citation Searching).

Expands a seed set of papers along the citation graph and emits API-verified
BibTeX candidates, deduplicated against an existing candidate pool.

Directions
----------
    backward : references the seed papers cite (cited-by-the-seed)
    forward  : papers that cite the seed (citing-the-seed)
    similar  : Semantic Scholar recommendations for the seed
    all      : backward + forward + similar (default)

Data source: Semantic Scholar Graph API (deterministic, no model memory).
    references     GET /graph/v1/paper/{id}/references
    citations      GET /graph/v1/paper/{id}/citations
    recommendations GET /recommendations/v1/papers/forpaper/{id}

Anti-hallucination contract
---------------------------
Snowball candidates carry `verified=false` + `verified_by=semantic_scholar`.
They are NOT cross-checked against PubMed/CrossRef here; downstream
`/verify-refs` confirms each entry and upgrades the flag. Nothing is ever
generated from memory.

Output contract (matches search-lit BibTeX section)
---------------------------------------------------
- BibTeX is APPENDED to the candidate pool (default references/library.bib).
- NEVER writes to manuscript/_src/refs.bib (that is /lit-sync's sole path).
- A PRISMA "records identified through citation searching" line is printed.

Usage
-----
    # Live (network) — expand one DOI in all directions, dedup against pool
    python3 snowball.py --seed DOI:10.1148/radiol.2024123 \
        --pool references/library.bib --out references/library.bib

    # Multiple seeds from a file (one id per line), backward only
    python3 snowball.py --seed @seeds.txt --direction backward

    # Deterministic / offline (challenge-card verifier): read recorded JSON
    python3 snowball.py --seed DOI:10.0/seed1 --direction all \
        --offline-fixture fixture --as-of 2026-06-14 --stdout

Seed id formats accepted: `DOI:10.x/...`, `PMID:123456`, bare `10.x/...`
(treated as DOI), bare digits (treated as PMID), or a raw S2 paper id.
"""

import argparse
import json
import re
import sys
import urllib.parse
import urllib.request
from datetime import date
from pathlib import Path

S2_GRAPH = "https://api.semanticscholar.org/graph/v1/paper"
S2_REC = "https://api.semanticscholar.org/recommendations/v1/papers/forpaper"
S2_FIELDS = "title,year,venue,externalIds,authors.name"

DIRECTIONS = ("backward", "forward", "similar")


# --------------------------------------------------------------------------- #
# Seed id normalization
# --------------------------------------------------------------------------- #
def normalize_seed_id(raw: str) -> str:
    """Return an S2-acceptable paper id token (DOI:/PMID:/raw)."""
    s = raw.strip()
    if not s:
        return ""
    up = s.upper()
    if up.startswith("DOI:") or up.startswith("PMID:") or up.startswith("ARXIV:"):
        return s
    if s.startswith("10.") or "doi.org/" in s.lower():
        return "DOI:" + s.split("doi.org/")[-1]
    if s.isdigit():
        return "PMID:" + s
    return s  # assume raw S2 id


def fixture_slug(seed_id: str) -> str:
    """Filesystem-safe slug for an offline fixture filename."""
    return re.sub(r"[^A-Za-z0-9]+", "_", seed_id).strip("_")


# --------------------------------------------------------------------------- #
# Fetch (network or offline fixture)
# --------------------------------------------------------------------------- #
def _http_get_json(url: str) -> dict:
    req = urllib.request.Request(url, headers={"User-Agent": "medsci-skills/snowball"})
    with urllib.request.urlopen(req, timeout=30) as resp:  # noqa: S310 (trusted host)
        return json.loads(resp.read().decode("utf-8"))


def fetch_direction(seed_id: str, direction: str, limit: int,
                    fixture_dir: Path | None) -> list[dict]:
    """Return a list of paper dicts for one seed+direction.

    Each returned dict has at least: title, year, venue, externalIds, authors.
    """
    if fixture_dir is not None:
        fpath = fixture_dir / f"{fixture_slug(seed_id)}.{direction}.json"
        if not fpath.exists():
            return []
        payload = json.loads(fpath.read_text())
    else:
        enc = urllib.parse.quote(seed_id, safe="")
        if direction == "backward":
            url = f"{S2_GRAPH}/{enc}/references?fields={S2_FIELDS}&limit={limit}"
        elif direction == "forward":
            url = f"{S2_GRAPH}/{enc}/citations?fields={S2_FIELDS}&limit={limit}"
        elif direction == "similar":
            url = f"{S2_REC}/{enc}?fields={S2_FIELDS}&limit={limit}"
        else:
            raise ValueError(f"unknown direction: {direction}")
        try:
            payload = _http_get_json(url)
        except Exception as exc:  # noqa: BLE001 — network failure is non-fatal
            sys.stderr.write(f"[snowball] {direction} fetch failed for {seed_id}: {exc}\n")
            return []

    # Normalize payload shapes:
    #   references/citations: {"data": [{"citedPaper"|"citingPaper": {...}}]}
    #   recommendations:      {"recommendedPapers": [{...}]} or {"data": [{...}]}
    rows = payload.get("data") or payload.get("recommendedPapers") or []
    papers = []
    for row in rows:
        paper = row.get("citedPaper") or row.get("citingPaper") or row
        if isinstance(paper, dict) and paper.get("title"):
            papers.append(paper)
    return papers


# --------------------------------------------------------------------------- #
# Dedup
# --------------------------------------------------------------------------- #
def norm_doi(doi: str | None) -> str:
    if not doi:
        return ""
    return doi.strip().lower().replace("https://doi.org/", "")


def norm_title(title: str | None) -> str:
    if not title:
        return ""
    return re.sub(r"[^a-z0-9]+", "", title.lower())


def parse_pool_keys(pool_path: Path | None) -> tuple[set[str], set[str]]:
    """Extract existing DOIs and normalized titles from a BibTeX pool."""
    dois: set[str] = set()
    titles: set[str] = set()
    if pool_path is None or not pool_path.exists():
        return dois, titles
    text = pool_path.read_text(errors="ignore")
    for m in re.finditer(r"doi\s*=\s*[{\"]([^}\"]+)[}\"]", text, re.I):
        dois.add(norm_doi(m.group(1)))
    for m in re.finditer(r"title\s*=\s*[{\"](.+?)[}\"]\s*,?\s*\n", text, re.I | re.S):
        titles.add(norm_title(m.group(1)))
    return dois, titles


# --------------------------------------------------------------------------- #
# BibTeX emit
# --------------------------------------------------------------------------- #
def _author_last(name: str) -> str:
    parts = name.strip().split()
    return parts[-1] if parts else "Anon"


def bibtex_key(paper: dict) -> str:
    authors = paper.get("authors") or []
    last = _author_last(authors[0]["name"]) if authors and authors[0].get("name") else "Anon"
    last = re.sub(r"[^A-Za-z]", "", last) or "Anon"
    year = str(paper.get("year") or "ND")
    title_word = ""
    for w in re.findall(r"[A-Za-z]{4,}", paper.get("title") or ""):
        if w.lower() not in {"with", "from", "using", "study", "analysis", "based"}:
            title_word = w.capitalize()
            break
    return f"{last}_{year}_{title_word or 'Snowball'}"


def to_bibtex(paper: dict, direction: str, as_of: str, key: str) -> str:
    ext = paper.get("externalIds") or {}
    doi = ext.get("DOI", "")
    pmid = ext.get("PubMed", "")
    authors = paper.get("authors") or []
    author_str = " and ".join(
        f"{_author_last(a['name'])}, {' '.join(a['name'].split()[:-1])}".strip().rstrip(",")
        for a in authors if a.get("name")
    ) or "Unknown"
    lines = [
        f"@article{{{key},",
        f"  author    = {{{author_str}}},",
        f"  title     = {{{paper.get('title', '').strip()}}},",
        f"  journal   = {{{paper.get('venue', '') or ''}}},",
        f"  year      = {{{paper.get('year') or ''}}},",
    ]
    if doi:
        lines.append(f"  doi       = {{{doi}}},")
    if pmid:
        lines.append(f"  pmid      = {{{pmid}}},")
    lines += [
        "  verified  = {false},",
        "  verified_by = {semantic_scholar},",
        f"  verified_on = {{{as_of}}},",
        "  source    = {citation_snowball},",
        f"  snowball_direction = {{{direction}}},",
        "}",
    ]
    return "\n".join(lines)


# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def load_seeds(raw: str) -> list[str]:
    if raw.startswith("@"):
        lines = Path(raw[1:]).read_text().splitlines()
        items = [ln for ln in lines if ln.strip() and not ln.strip().startswith("#")]
    else:
        items = re.split(r"[,\s]+", raw)
    return [normalize_seed_id(x) for x in items if x.strip()]


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Citation snowballing for search-lit.")
    ap.add_argument("--seed", required=True,
                    help="comma/space-separated ids, or @file with one id per line")
    ap.add_argument("--direction", default="all",
                    choices=("all", *DIRECTIONS))
    ap.add_argument("--pool", default=None,
                    help="existing library.bib to dedup against")
    ap.add_argument("--out", default="references/library.bib",
                    help="BibTeX append target (NEVER manuscript/_src/refs.bib)")
    ap.add_argument("--limit", type=int, default=50, help="max per seed per direction")
    ap.add_argument("--offline-fixture", default=None,
                    help="dir of recorded JSON (<slug>.<direction>.json) for deterministic runs")
    ap.add_argument("--as-of", default=date.today().isoformat(),
                    help="verified_on date stamp (default today; set for reproducible output)")
    ap.add_argument("--stdout", action="store_true",
                    help="print BibTeX to stdout instead of appending to --out")
    args = ap.parse_args(argv)

    out_path = Path(args.out)
    if out_path.name == "refs.bib" and "_src" in str(out_path):
        ap.error("refusing to write manuscript/_src/refs.bib — that is /lit-sync's sole path")

    fixture_dir = Path(args.offline_fixture) if args.offline_fixture else None
    pool_dois, pool_titles = parse_pool_keys(Path(args.pool) if args.pool else None)

    seeds = load_seeds(args.seed)
    directions = list(DIRECTIONS) if args.direction == "all" else [args.direction]

    seen_doi = set(pool_dois)
    seen_title = set(pool_titles)
    seen_key: set[str] = set()
    counts = {d: 0 for d in directions}
    entries: list[str] = []
    raw_found = 0

    for seed in seeds:
        for direction in directions:
            papers = fetch_direction(seed, direction, args.limit, fixture_dir)
            for paper in papers:
                raw_found += 1
                ext = paper.get("externalIds") or {}
                d = norm_doi(ext.get("DOI"))
                t = norm_title(paper.get("title"))
                if (d and d in seen_doi) or (t and t in seen_title):
                    continue
                if d:
                    seen_doi.add(d)
                if t:
                    seen_title.add(t)
                key = bibtex_key(paper)
                base_key, n = key, 1
                while key in seen_key:
                    n += 1
                    key = f"{base_key}{chr(96 + n)}"  # _b, _c, ...
                seen_key.add(key)
                entries.append(to_bibtex(paper, direction, args.as_of, key))
                counts[direction] += 1

    new_total = len(entries)
    bibtex_blob = ("\n\n".join(entries) + "\n") if entries else ""

    if args.stdout or not entries:
        sys.stdout.write(bibtex_blob)
    else:
        out_path.parent.mkdir(parents=True, exist_ok=True)
        with out_path.open("a", encoding="utf-8") as fh:
            if out_path.exists() and out_path.stat().st_size > 0:
                fh.write("\n")
            fh.write(bibtex_blob)

    # PRISMA citation-searching line (stderr so --stdout BibTeX stays clean)
    breakdown = ", ".join(f"{d}={counts[d]}" for d in directions)
    pool_n = len(pool_dois) + len(pool_titles)
    sys.stderr.write(
        f"Records identified through citation searching (snowballing): "
        f"{raw_found} raw ({breakdown}); after dedup against existing pool: "
        f"{new_total} new candidates.\n"
    )
    sys.stderr.write(
        f"[snowball] seeds={len(seeds)} directions={'+'.join(directions)} "
        f"pool_keys={pool_n} -> {new_total} appended"
        f"{' (stdout)' if args.stdout else f' to {out_path}'}\n"
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

schema_version: 2
name: search-lit
layer: A
owner_domain: literature_discovery
maturity: official
when_to_use:
  - User asks to find papers, related work, citations, or background literature on a topic
  - Generating a verified candidate citation pool (PubMed / Semantic Scholar / bioRxiv / medRxiv)
  - Pre-/lit-sync candidate sourcing (search-lit produces candidates → /lit-sync syncs to Zotero + refs.bib)
  - Building a BibTeX library for a topic without yet committing to inclusion
when_NOT_to_use:
  - Verifying citations already in a manuscript (use /verify-refs)
  - Syncing to Zotero or writing manuscript/_src/refs.bib (use /lit-sync — sole writer)
  - Generating references from model memory (forbidden — every entry must be API-verified)
inputs:
  - literature_query
outputs:
  - references/library.bib  # search-result candidate pool; NOT the manuscript SSOT bib
  - references/search_results.tsv
deterministic_scripts:
  - references/pubmed_eutils.sh
  - references/parse_pubmed.py
  - references/snowball.py  # Phase 2.5 citation snowballing (S2 Graph API); challenge card in references/snowball_challenge/
side_effects:
  - may_call_external_literature_apis
downstream_consumers:
  - verify-refs
  - lit-sync  # confirmed candidates flow through Zotero, then lit-sync refreshes manuscript/_src/refs.bib
  - write-paper
ssot_boundary:
  - manuscript/_src/refs.bib is OWNED by /lit-sync (Better BibTeX auto-export). search-lit MUST NOT write to that path.
forbidden_actions:
  - generate_references_from_memory
  - silently_include_unverified_references
  - write_to_manuscript_refs_bib  # SSOT owner is /lit-sync

# v2.1 quality card
purpose: "Search PubMed, Semantic Scholar, and bioRxiv/medRxiv and generate API-verified BibTeX (anti-hallucination: every reference verified before inclusion)."
safety_boundaries:
  - "Never generates references from memory; unverified references are not silently included."
  - "Does not write to the manuscript refs.bib (that SSOT belongs to lit-sync)."
known_limitations:
  - "Depends on PubMed/Semantic Scholar availability; rate limits/outages reduce recall."
  - "Verification confirms existence/metadata, not topical relevance."
validation_commands:
  - "bash references/pubmed_eutils.sh <query>"
  - "bash references/snowball_challenge/verify.sh  # deterministic, network-free"
  - "/verify-refs --strict"
evidence_surface: bundled_script