Literature Search

#!/usr/bin/env python3
"""Download arXiv paper source by title, extract .tex content.

Searches the arXiv API by title, downloads the source tarball,
and extracts .tex files into a local directory.

Self-contained: uses only stdlib (urllib + xml.etree instead of feedparser).

Usage:
    python download_arxiv_source.py --title "Attention Is All You Need" --output-dir arxiv_papers/
    python download_arxiv_source.py --title "BERT" --max-results 3 --output-dir arxiv_papers/
    python download_arxiv_source.py --arxiv-id 1706.03762 --output-dir arxiv_papers/
"""

import argparse
import json
import os
import re
import sys
import tarfile
import tempfile
import time
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET

ARXIV_API = "http://export.arxiv.org/api/query"
ARXIV_NS = {"atom": "http://www.w3.org/2005/Atom"}


def search_arxiv(query: str, max_results: int = 5, search_field: str = "ti") -> list[dict]:
    """Search arXiv API and return paper metadata."""
    params = urllib.parse.urlencode({
        "search_query": f"{search_field}:{urllib.parse.quote(query)}",
        "start": 0,
        "max_results": max_results,
        "sortBy": "relevance",
        "sortOrder": "descending",
    })
    url = f"{ARXIV_API}?{params}"

    try:
        with urllib.request.urlopen(url, timeout=30) as resp:
            xml_data = resp.read()
    except Exception as e:
        print(f"arXiv API error: {e}", file=sys.stderr)
        return []

    root = ET.fromstring(xml_data)
    papers = []
    for entry in root.findall("atom:entry", ARXIV_NS):
        title_el = entry.find("atom:title", ARXIV_NS)
        title = title_el.text.strip().replace("\n", " ") if title_el is not None else ""

        summary_el = entry.find("atom:summary", ARXIV_NS)
        summary = summary_el.text.strip() if summary_el is not None else ""

        authors = []
        for author in entry.findall("atom:author", ARXIV_NS):
            name_el = author.find("atom:name", ARXIV_NS)
            if name_el is not None:
                authors.append(name_el.text)

        published_el = entry.find("atom:published", ARXIV_NS)
        published = published_el.text if published_el is not None else ""

        abs_link = ""
        pdf_link = ""
        for link in entry.findall("atom:link", ARXIV_NS):
            href = link.get("href", "")
            link_type = link.get("type", "")
            rel = link.get("rel", "")
            if link_type == "application/pdf":
                pdf_link = href
            elif rel == "alternate":
                abs_link = href

        arxiv_id = ""
        id_el = entry.find("atom:id", ARXIV_NS)
        if id_el is not None:
            m = re.search(r"abs/(.+)", id_el.text)
            if m:
                arxiv_id = m.group(1)

        papers.append({
            "title": title,
            "authors": authors,
            "published": published,
            "summary": summary,
            "abs_link": abs_link,
            "pdf_link": pdf_link,
            "arxiv_id": arxiv_id,
        })

    return papers


def download_source(arxiv_id: str, output_dir: str) -> str | None:
    """Download arXiv source tarball and extract .tex files.

    Returns the path to the extracted content or None on failure.
    """
    # Strip version suffix for source download
    base_id = re.sub(r"v\d+$", "", arxiv_id)
    source_url = f"https://arxiv.org/src/{base_id}"

    safe_name = re.sub(r"[^a-zA-Z0-9._-]", "_", arxiv_id)
    os.makedirs(output_dir, exist_ok=True)

    try:
        req = urllib.request.Request(source_url, headers={"User-Agent": "SkillScript/1.0"})
        with urllib.request.urlopen(req, timeout=60) as resp:
            tar_data = resp.read()
    except Exception as e:
        print(f"Download failed for {arxiv_id}: {e}", file=sys.stderr)
        return None

    # Save tarball to temp file, then extract
    with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as tmp:
        tmp.write(tar_data)
        tmp_path = tmp.name

    tex_contents = []
    try:
        with tarfile.open(tmp_path, "r:gz") as tar:
            tex_files = [m for m in tar.getmembers() if m.name.endswith(".tex")]
            for tex_file in tex_files:
                f = tar.extractfile(tex_file)
                if f is not None:
                    try:
                        content = f.read().decode("utf-8")
                    except UnicodeDecodeError:
                        f.seek(0)
                        content = f.read().decode("latin-1")
                    tex_contents.append((tex_file.name, content))
    except (tarfile.TarError, Exception) as e:
        print(f"Extraction failed for {arxiv_id}: {e}", file=sys.stderr)
        return None
    finally:
        os.unlink(tmp_path)

    if not tex_contents:
        print(f"No .tex files found in source for {arxiv_id}", file=sys.stderr)
        return None

    # Find main tex file (contains \documentclass or largest file)
    main_file = None
    for name, content in tex_contents:
        if r"\documentclass" in content:
            main_file = (name, content)
            break
    if main_file is None:
        main_file = max(tex_contents, key=lambda x: len(x[1]))

    # Write all tex files
    out_subdir = os.path.join(output_dir, safe_name)
    os.makedirs(out_subdir, exist_ok=True)
    for name, content in tex_contents:
        safe_tex = re.sub(r"[/\\]", "_", name)
        out_path = os.path.join(out_subdir, safe_tex)
        with open(out_path, "w", encoding="utf-8") as f:
            f.write(content)

    # Also write combined file
    combined_path = os.path.join(output_dir, f"{safe_name}.tex")
    with open(combined_path, "w", encoding="utf-8") as f:
        for name, content in tex_contents:
            f.write(f"\n{'=' * 50}\n% File: {name}\n{'=' * 50}\n")
            f.write(content)
            f.write("\n\n")

    return combined_path


def main():
    parser = argparse.ArgumentParser(description="Download arXiv paper source by title")
    parser.add_argument("--title", help="Paper title to search for")
    parser.add_argument("--arxiv-id", help="Direct arXiv ID (e.g., 1706.03762)")
    parser.add_argument("--max-results", type=int, default=5, help="Max search results (default: 5)")
    parser.add_argument("--output-dir", default="arxiv_papers", help="Output directory (default: arxiv_papers/)")
    parser.add_argument("--metadata", action="store_true", help="Also output metadata JSON")
    args = parser.parse_args()

    if not args.title and not args.arxiv_id:
        print("Error: must specify --title or --arxiv-id", file=sys.stderr)
        sys.exit(1)

    if args.arxiv_id:
        print(f"Downloading source for arXiv:{args.arxiv_id}", file=sys.stderr)
        result = download_source(args.arxiv_id, args.output_dir)
        if result:
            print(f"Saved to: {result}")
        else:
            sys.exit(1)
        return

    # Search by title
    print(f"Searching arXiv for: {args.title}", file=sys.stderr)
    papers = search_arxiv(args.title, max_results=args.max_results)
    if not papers:
        print("No results found.", file=sys.stderr)
        sys.exit(1)

    print(f"Found {len(papers)} results:", file=sys.stderr)
    for i, p in enumerate(papers):
        print(f"  [{i+1}] {p['title'][:80]} ({p['arxiv_id']})", file=sys.stderr)

    if args.metadata:
        meta_path = os.path.join(args.output_dir, "metadata.json")
        os.makedirs(args.output_dir, exist_ok=True)
        with open(meta_path, "w", encoding="utf-8") as f:
            json.dump(papers, f, indent=2, ensure_ascii=False)
        print(f"Metadata saved to: {meta_path}", file=sys.stderr)

    # Download first result
    paper = papers[0]
    if not paper["arxiv_id"]:
        print("No arXiv ID found for top result.", file=sys.stderr)
        sys.exit(1)

    print(f"\nDownloading source for: {paper['title'][:60]}...", file=sys.stderr)
    time.sleep(1)  # Be polite to arXiv
    result = download_source(paper["arxiv_id"], args.output_dir)
    if result:
        print(f"Saved to: {result}")
    else:
        print("Download failed.", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Search CrossRef API for academic papers and generate BibTeX entries.

Queries the CrossRef works API, maps document types to BibTeX types,
and generates clean BibTeX entries with proper key formatting.

Self-contained: uses only stdlib. Replaces unidecode with unicodedata.normalize().

Usage:
    python search_crossref.py --query "attention is all you need" --rows 5
    python search_crossref.py --query "transformer architecture" --rows 10 --output results.jsonl
    python search_crossref.py --query "diffusion models" --rows 3 --bibtex --output refs.bib
"""

import argparse
import json
import re
import sys
import unicodedata
import urllib.error
import urllib.parse
import urllib.request
import time

CROSSREF_URL = "https://api.crossref.org/works"
HEADERS = {"User-Agent": "SkillScript/1.0 (mailto:user@example.com)"}

TYPE_MAPPING = {
    "journal-article": "article",
    "proceedings-article": "inproceedings",
    "book-chapter": "incollection",
    "book": "book",
    "posted-content": "misc",
    "report": "techreport",
    "dissertation": "phdthesis",
    "dataset": "misc",
    "monograph": "book",
}

BIBTEX_FIELDS = {
    "article": ["author", "title", "journal", "year", "volume", "number", "pages", "doi"],
    "inproceedings": ["author", "title", "booktitle", "year", "pages", "doi"],
    "incollection": ["author", "title", "booktitle", "year", "pages", "doi"],
    "book": ["author", "title", "publisher", "year", "doi"],
    "misc": ["author", "title", "year", "doi", "howpublished"],
    "techreport": ["author", "title", "institution", "year"],
    "phdthesis": ["author", "title", "school", "year"],
}

COMMON_WORDS = {
    "a", "an", "the", "of", "in", "on", "at", "to", "for", "and", "or",
    "is", "are", "was", "were", "be", "been", "with", "from", "by", "as",
    "its", "it", "this", "that", "via", "using", "through", "between",
}

FORBIDDEN_IDS = {
    "none", "introduction", "references", "abstract", "conclusion",
    "method", "methods", "results", "discussion", "appendix",
}


def normalize_text(text: str) -> str:
    """Remove diacritics and normalize unicode to ASCII-safe text."""
    nfkd = unicodedata.normalize("NFKD", text)
    return "".join(c for c in nfkd if not unicodedata.combining(c))


def clean_bibtex_id(text: str) -> str:
    """Remove special characters from a BibTeX ID component."""
    return re.sub(r"[^a-zA-Z0-9]", "", normalize_text(text))


def make_bibtex_key(item: dict) -> str | None:
    """Generate a BibTeX key like 'vaswani2017attention' from CrossRef item."""
    authors = item.get("author", [])
    title = item.get("title", [""])[0] if item.get("title") else ""
    year = ""
    if item.get("published-print"):
        parts = item["published-print"].get("date-parts", [[]])
        if parts and parts[0]:
            year = str(parts[0][0])
    elif item.get("published-online"):
        parts = item["published-online"].get("date-parts", [[]])
        if parts and parts[0]:
            year = str(parts[0][0])
    elif item.get("created"):
        parts = item["created"].get("date-parts", [[]])
        if parts and parts[0]:
            year = str(parts[0][0])

    if not title:
        return None

    family = ""
    if authors:
        family = clean_bibtex_id(authors[0].get("family", ""))

    title_words = re.findall(r"[A-Za-z]+", normalize_text(title))
    content_words = [w.lower() for w in title_words if w.lower() not in COMMON_WORDS]
    title_part = content_words[0] if content_words else ""

    key = family + year + title_part
    if not key or key.lower() in FORBIDDEN_IDS:
        doi = item.get("DOI", "")
        key = "ref" + clean_bibtex_id(doi)[-12:] if doi else None

    return key


def format_authors(authors: list[dict]) -> str:
    """Format CrossRef author list as BibTeX author string."""
    parts = []
    for a in authors:
        given = a.get("given", "")
        family = a.get("family", "")
        if family and given:
            parts.append(f"{normalize_text(family)}, {normalize_text(given)}")
        elif family:
            parts.append(normalize_text(family))
    return " and ".join(parts)


def extract_year(item: dict) -> str:
    """Extract publication year from CrossRef item."""
    for field in ["published-print", "published-online", "created"]:
        if item.get(field):
            parts = item[field].get("date-parts", [[]])
            if parts and parts[0]:
                return str(parts[0][0])
    return ""


def item_to_record(item: dict) -> dict:
    """Convert a CrossRef API item to a flat record dict."""
    title = item.get("title", [""])[0] if item.get("title") else ""
    authors = format_authors(item.get("author", []))
    year = extract_year(item)
    journal = ""
    for cn in item.get("container-title", []):
        journal = cn
        break
    doi = item.get("DOI", "")
    abstract = item.get("abstract", "")
    # Strip HTML tags from abstract
    abstract = re.sub(r"<[^>]+>", "", abstract)
    score = item.get("score", 0)
    cited_by = item.get("is-referenced-by-count", 0)
    bib_type = TYPE_MAPPING.get(item.get("type", ""), "misc")

    return {
        "title": title,
        "authors": authors,
        "year": year,
        "journal": journal,
        "doi": doi,
        "abstract": abstract,
        "type": bib_type,
        "score": score,
        "cited_by": cited_by,
        "volume": item.get("volume", ""),
        "issue": item.get("issue", ""),
        "pages": item.get("page", ""),
        "publisher": item.get("publisher", ""),
        "booktitle": journal,
    }


def record_to_bibtex(record: dict, key: str) -> str:
    """Format a record dict as a BibTeX entry string."""
    bib_type = record.get("type", "misc")
    fields = BIBTEX_FIELDS.get(bib_type, BIBTEX_FIELDS["misc"])
    lines = [f"@{bib_type}{{{key},"]
    for field in fields:
        val = record.get(field, "")
        if field == "journal":
            val = record.get("journal", "")
        elif field == "number":
            val = record.get("issue", "")
        if val:
            lines.append(f"  {field} = {{{val}}},")
    lines.append("}")
    return "\n".join(lines)


def query_crossref(query: str, rows: int = 10, timeout: int = 30) -> list[dict]:
    """Query CrossRef API and return raw items."""
    params = urllib.parse.urlencode({"query": query, "rows": rows})
    url = f"{CROSSREF_URL}?{params}"
    req = urllib.request.Request(url, headers=HEADERS)

    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                data = json.loads(resp.read().decode("utf-8"))
            return data.get("message", {}).get("items", [])
        except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError) as e:
            if attempt < 2:
                time.sleep(2 ** attempt)
            else:
                print(f"CrossRef API error after 3 attempts: {e}", file=sys.stderr)
                return []


def main():
    parser = argparse.ArgumentParser(description="Search CrossRef for academic papers")
    parser.add_argument("--query", required=True, help="Search query")
    parser.add_argument("--rows", type=int, default=10, help="Number of results (default: 10)")
    parser.add_argument("--output", "-o", help="Output file (.jsonl or .bib)")
    parser.add_argument("--bibtex", action="store_true", help="Output BibTeX format")
    parser.add_argument("--timeout", type=int, default=30, help="Request timeout in seconds")
    args = parser.parse_args()

    items = query_crossref(args.query, rows=args.rows, timeout=args.timeout)
    if not items:
        print("No results found.", file=sys.stderr)
        sys.exit(1)

    records = []
    used_keys = set()
    for item in items:
        record = item_to_record(item)
        key = make_bibtex_key(item)
        if key is None:
            continue
        # Deduplicate keys
        orig_key = key
        suffix_idx = 0
        while key in used_keys:
            suffix_idx += 1
            key = orig_key + chr(ord("a") + suffix_idx - 1)
        used_keys.add(key)
        record["bibtex_key"] = key
        records.append(record)

    is_bibtex = args.bibtex or (args.output and args.output.endswith(".bib"))

    output_lines = []
    if is_bibtex:
        for r in records:
            output_lines.append(record_to_bibtex(r, r["bibtex_key"]))
            output_lines.append("")
    else:
        for r in records:
            output_lines.append(json.dumps(r, ensure_ascii=False))

    text = "\n".join(output_lines) + "\n" if output_lines else ""

    if args.output:
        with open(args.output, "w", encoding="utf-8") as f:
            f.write(text)
        print(f"Wrote {len(records)} results to {args.output}", file=sys.stderr)
    else:
        sys.stdout.write(text)

    print(f"Found {len(records)} results for: {args.query}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Search OpenAlex API and output JSONL paper metadata.

Self-contained: uses only stdlib (urllib, json).
OpenAlex is free, no API key needed, broadest coverage of academic literature.

Usage:
    python search_openalex.py --query "attention mechanism transformers" --max-results 50
    python search_openalex.py --query "graph neural networks" --min-citations 10 --year-range 2022-2026
    python search_openalex.py --query "diffusion models" --type article --sort cited_by_count:desc
"""

import argparse
import json
import sys
import time
import urllib.parse
import urllib.request


OPENALEX_API = "https://api.openalex.org"


def openalex_request(url: str) -> dict:
    """Make a request to OpenAlex API with retry logic."""
    headers = {
        "User-Agent": "research-engine/1.0 (mailto:research@example.com)",
        "Accept": "application/json",
    }
    req = urllib.request.Request(url, headers=headers)

    for attempt in range(3):
        try:
            with urllib.request.urlopen(req, timeout=30) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code == 429:
                wait = 2 ** (attempt + 1)
                print(f"Rate limited, waiting {wait}s...", file=sys.stderr)
                time.sleep(wait)
                continue
            raise
        except Exception:
            if attempt < 2:
                time.sleep(1)
                continue
            raise
    return {}


def parse_work(work: dict) -> dict | None:
    """Parse an OpenAlex work into our standard record format."""
    if not work or not work.get("title"):
        return None

    # Authors
    authors = []
    for authorship in work.get("authorships", []):
        author = authorship.get("author", {})
        name = author.get("display_name", "")
        if name:
            authors.append(name)

    # Venue
    primary_location = work.get("primary_location", {}) or {}
    source = primary_location.get("source", {}) or {}
    venue = source.get("display_name", "")

    # ArXiv ID from locations
    arxiv_id = ""
    for loc in work.get("locations", []):
        landing = loc.get("landing_page_url", "") or ""
        if "arxiv.org" in landing:
            parts = landing.rstrip("/").split("/")
            arxiv_id = parts[-1] if parts else ""
            break

    # DOI
    doi = work.get("doi", "") or ""
    if doi.startswith("https://doi.org/"):
        doi = doi[16:]

    # PDF URL
    pdf_url = ""
    oa = work.get("open_access", {}) or {}
    pdf_url = oa.get("oa_url", "") or ""
    if not pdf_url and arxiv_id:
        pdf_url = f"https://arxiv.org/pdf/{arxiv_id}"

    # Abstract (OpenAlex uses inverted index)
    abstract = ""
    abstract_inv = work.get("abstract_inverted_index", {})
    if abstract_inv:
        # Reconstruct from inverted index
        word_positions = []
        for word, positions in abstract_inv.items():
            for pos in positions:
                word_positions.append((pos, word))
        word_positions.sort()
        abstract = " ".join(w for _, w in word_positions)

    # Peer reviewed heuristic
    work_type = work.get("type", "")
    peer_reviewed = work_type in ("article", "proceedings-article", "book-chapter")

    return {
        "openalex_id": work.get("id", ""),
        "doi": doi,
        "arxiv_id": arxiv_id,
        "title": work["title"],
        "authors": authors,
        "abstract": abstract[:1000],
        "year": work.get("publication_year"),
        "venue": venue,
        "venue_normalized": venue,
        "peer_reviewed": peer_reviewed,
        "citationCount": work.get("cited_by_count", 0),
        "url": work.get("id", ""),
        "publicationDate": work.get("publication_date", ""),
        "pdf_url": pdf_url,
        "source": "openalex",
    }


def search_works(
    query: str,
    max_results: int = 50,
    year_range: str | None = None,
    min_citations: int = 0,
    work_type: str | None = None,
    sort: str = "cited_by_count:desc",
) -> list[dict]:
    """Search OpenAlex works and return parsed results."""
    all_papers = []

    filters = []
    if year_range:
        parts = year_range.split("-")
        if len(parts) == 2:
            filters.append(f"publication_year:{parts[0]}-{parts[1]}")
    if min_citations > 0:
        filters.append(f"cited_by_count:>{min_citations}")
    if work_type:
        filters.append(f"type:{work_type}")

    page = 1
    per_page = min(50, max_results)

    while len(all_papers) < max_results:
        params = {
            "search": query,
            "per_page": per_page,
            "page": page,
            "sort": sort,
        }
        if filters:
            params["filter"] = ",".join(filters)

        url = f"{OPENALEX_API}/works?{urllib.parse.urlencode(params)}"

        try:
            resp = openalex_request(url)
        except Exception as e:
            print(f"Warning: search failed at page {page}: {e}", file=sys.stderr)
            break

        results = resp.get("results", [])
        if not results:
            break

        for work in results:
            if len(all_papers) >= max_results:
                break
            paper = parse_work(work)
            if paper:
                all_papers.append(paper)

        meta = resp.get("meta", {})
        total = meta.get("count", 0)
        if page * per_page >= total:
            break

        page += 1
        time.sleep(0.2)  # Be polite

    return all_papers


def main():
    parser = argparse.ArgumentParser(description="Search OpenAlex and output JSONL")
    parser.add_argument("--query", required=True, help="Search keywords")
    parser.add_argument("--max-results", type=int, default=50, help="Max papers to return")
    parser.add_argument("--min-citations", type=int, default=0, help="Minimum citation count")
    parser.add_argument("--year-range", help="Year range filter (e.g. 2020-2026)")
    parser.add_argument("--type", help="Work type filter (e.g. article, proceedings-article)")
    parser.add_argument("--sort", default="cited_by_count:desc", help="Sort order")
    parser.add_argument("--output", "-o", help="Output file (default: stdout)")
    args = parser.parse_args()

    papers = search_works(
        query=args.query,
        max_results=args.max_results,
        year_range=args.year_range,
        min_citations=args.min_citations,
        work_type=args.type,
        sort=args.sort,
    )

    out = open(args.output, "w") if args.output else sys.stdout
    try:
        for paper in papers:
            out.write(json.dumps(paper, ensure_ascii=False) + "\n")
    finally:
        if args.output:
            out.close()

    print(f"Found {len(papers)} papers", file=sys.stderr)


if __name__ == "__main__":
    main()