Literature Search Arxiv

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Downloads the source (tar.gz) of a paper from arXiv given its ID.

This script allows downloading the LaTeX source files of arXiv papers
and saving them to a specified output file path.
"""

# /// script
# requires-python = ">=3.10"
# dependencies = [
#   "scienceskillscommon",
# ]
# [tool.uv.sources]
# scienceskillscommon = { path = "../../scienceskillscommon" }
# ///

import argparse
import os
import sys
import urllib.error

from science_skills.skills.scienceskillscommon import http_client

_CLIENT = http_client.HttpClient("https://export.arxiv.org/", qps=1.0 / 3.0)


def parse_args() -> argparse.Namespace:
  """Parses command-line arguments for the download script.

  Returns:
    argparse.Namespace: An object containing the parsed arguments.
  """
  parser = argparse.ArgumentParser(
      description="Download paper source (tar.gz) from arXiv"
  )
  parser.add_argument(
      "--id", type=str, required=True, help="arXiv ID (e.g., 2010.11645)"
  )
  parser.add_argument(
      "--output",
      type=str,
      required=True,
      help="Output file path for the tar.gz file",
  )
  return parser.parse_args()


def download_source(args: argparse.Namespace):
  """Downloads the source of a paper from arXiv based on the provided arguments.

  This function fetches the source (tar.gz) from arXiv using the
  specified ID, saving it to the given output path. It includes
  error handling for common issues like 404 Not Found and network errors,
  and enforces a rate limit after each download attempt.

  Args:
    args: An argparse.Namespace object containing: - id (str): The arXiv ID of
      the paper. - output (str): The file path where the tar.gz will be saved.
  """
  # Ensure ID is clean
  paper_id = args.id.strip()

  url = f"https://export.arxiv.org/e-print/{paper_id}"
  print(f"Attempting to download source from {url}...")

  try:
    content = _CLIENT.fetch_bytes(url)
    out_dir = os.path.dirname(args.output)
    if out_dir:
      os.makedirs(out_dir, exist_ok=True)
    with open(args.output, "wb") as f:
      f.write(content)
    print(f"Success! Saved to {args.output}")

  except urllib.error.HTTPError as e:
    if e.code == 404:
      print(
          f"Error 404: Source not found (ID: {paper_id}). Not all papers have"
          " source available.",
          file=sys.stderr,
      )
    else:
      raise


if __name__ == "__main__":
  main_args = parse_args()
  download_source(main_args)

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Downloads a paper from arXiv given its ID.

This script allows downloading arXiv papers in either PDF or HTML format
and saving them to a specified output file path.
"""

# /// script
# requires-python = ">=3.10"
# dependencies = [
#   "scienceskillscommon",
# ]
# [tool.uv.sources]
# scienceskillscommon = { path = "../../scienceskillscommon" }
# ///

import argparse
import os
import sys
import urllib.error

from science_skills.skills.scienceskillscommon import http_client

_CLIENT = http_client.HttpClient("https://arxiv.org/", qps=1.0 / 3.0)


def parse_args() -> argparse.Namespace:
  """Parses command-line arguments for the download script.

  Returns:
    argparse.Namespace: An object containing the parsed arguments.
  """
  parser = argparse.ArgumentParser(
      description="Download paper from arXiv (PDF or HTML)"
  )
  parser.add_argument(
      "--id", type=str, required=True, help="arXiv ID (e.g., 2305.10601)"
  )
  parser.add_argument(
      "--format",
      type=str,
      choices=["pdf", "html"],
      required=True,
      help="Download format",
  )
  parser.add_argument(
      "--output", type=str, required=True, help="Output file path"
  )
  return parser.parse_args()


def download_paper(args: argparse.Namespace):
  """Downloads a paper from arXiv based on the provided arguments.

  This function fetches a paper (either PDF or HTML) from arXiv using the
  specified ID and format, saving it to the given output path. It includes
  error handling for common issues like 404 Not Found and network errors,
  and enforces a rate limit after each download attempt.

  Args:
    args: An argparse.Namespace object containing: id (str) -- The arXiv ID of
      the paper; format (str) -- The desired format ("pdf" or "html"); output
      (str) -- The file path where the paper will be saved.
  """
  # Ensure ID is clean
  paper_id = args.id.strip()

  if args.format == "pdf":
    url = f"https://arxiv.org/pdf/{paper_id}.pdf"
  elif args.format == "html":
    url = f"https://arxiv.org/html/{paper_id}"
  else:
    raise ValueError(f"Unsupported format: {args.format}")

  print(f"Attempting to download {args.format.upper()} from {url}...")

  try:
    content = _CLIENT.fetch_bytes(url)
    out_dir = os.path.dirname(args.output)
    if out_dir:
      os.makedirs(out_dir, exist_ok=True)
    with open(args.output, "wb") as f:
      f.write(content)
    print(f"Success! Saved to {args.output}")

  except urllib.error.HTTPError as e:
    if e.code == 404:
      if args.format == "html":
        print(
            "Error 404: HTML format is not available for this paper (ID:"
            f" {paper_id}). Older papers may only have PDFs. Try downloading"
            " with --format pdf",
            file=sys.stderr,
        )
      else:
        print(
            f"Error 404: Paper not found (ID: {paper_id}). Check the ID.",
            file=sys.stderr,
        )
    else:
      raise


if __name__ == "__main__":
  main_args = parse_args()
  download_paper(main_args)

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Searches the arXiv API and returns results in a clean JSON format.

This script allows querying the arXiv API using either a search query string
or a list of arXiv IDs. It parses the XML response and outputs a JSON object
containing the search results.
"""

# /// script
# requires-python = ">=3.10"
# dependencies = [
#   "scienceskillscommon",
# ]
# [tool.uv.sources]
# scienceskillscommon = { path = "../../scienceskillscommon" }
# ///

import argparse
import json
import sys
import urllib.parse
import xml.etree.ElementTree as ET

from science_skills.skills.scienceskillscommon import http_client

_BASE_URL = "http://export.arxiv.org/api/query?"
_CLIENT = http_client.HttpClient(_BASE_URL, qps=1.0 / 3.0)


def parse_args() -> argparse.Namespace:
  """Parses command-line arguments for the arXiv search script.

  Returns:
    argparse.Namespace: An object containing the parsed command-line arguments.
  """
  parser = argparse.ArgumentParser(
      description="Search arXiv API and return clean JSON"
  )
  parser.add_argument(
      "--query",
      type=str,
      help="Search query string (e.g., 'au:einstein AND ti:relativity')",
  )
  parser.add_argument(
      "--id_list", type=str, help="Comma-separated list of arXiv IDs"
  )
  parser.add_argument("--start", type=int, default=0, help="Pagination offset")
  parser.add_argument(
      "--max_results", type=int, default=10, help="Number of results to return"
  )
  parser.add_argument(
      "--sort_by",
      type=str,
      choices=["relevance", "lastUpdatedDate", "submittedDate"],
      help="Sort by",
  )
  parser.add_argument(
      "--sort_order",
      type=str,
      choices=["ascending", "descending"],
      help="Sort order",
  )
  return parser.parse_args()


def strip_namespace(tag: str) -> str:
  if tag.startswith("{"):
    return tag.split("}", 1)[1]
  return tag


def search_arxiv(args: argparse.Namespace):
  """Searches the arXiv API and prints results as a JSON string.

  Constructs a query to the arXiv API based on the provided arguments,
  fetches the XML response, parses it, and prints a JSON object containing
  the extracted paper information.

  Args:
    args: An argparse.Namespace object containing the search parameters (query,
      id_list, start, max_results, sort_by, sort_order).
  """
  params = {"start": args.start, "max_results": args.max_results}
  if args.query:
    params["search_query"] = args.query
  if args.id_list:
    params["id_list"] = args.id_list
  if args.sort_by:
    params["sortBy"] = args.sort_by
  if args.sort_order:
    params["sortOrder"] = args.sort_order

  # Use quote_plus to ensure spaces become '+' as required by arXiv API
  query_string = urllib.parse.urlencode(
      params, quote_via=urllib.parse.quote_plus
  )
  url = _BASE_URL + query_string

  results = []
  xml_data = _CLIENT.fetch_bytes(url)
  root = ET.fromstring(xml_data)

  for entry in root.findall("{http://www.w3.org/2005/Atom}entry"):
    paper = {}
    authors = []

    for child in entry:
      tag = strip_namespace(child.tag)
      if tag == "id":
        # Extract just the ID part (e.g., http://arxiv.org/abs/2305.10601v1
        # -> 2305.10601v1)
        if child.text:
          paper["id"] = child.text.split("/abs/")[-1]
      elif tag == "title":
        paper["title"] = (
            child.text.replace("\n", " ").strip() if child.text else ""
        )
      elif tag == "summary":
        paper["summary"] = (
            child.text.replace("\n", " ").strip() if child.text else ""
        )
      elif tag == "published":
        paper["published"] = child.text
      elif tag == "author":
        for name_node in child.findall("{http://www.w3.org/2005/Atom}name"):
          authors.append(name_node.text)
      elif tag == "link":
        if child.get("title") == "pdf":
          paper["pdf_url"] = child.get("href")
      elif tag == "primary_category":
        paper["primary_category"] = child.get("term")
      elif tag in {"doi", "journal_ref", "comment"}:
        paper[tag] = child.text

    paper["authors"] = authors
    results.append(paper)

    print(
        json.dumps(
            {
                "status": "success",
                "results_count": len(results),
                "papers": results,
            },
            indent=2,
        )
    )


if __name__ == "__main__":
  main_args = parse_args()
  if not main_args.query and not main_args.id_list:
    print(
        json.dumps(
            {
                "status": "error",
                "message": "Must provide either --query or --id_list",
            },
            indent=2,
        )
    )
    sys.exit(1)
  search_arxiv(main_args)