Twitter Reader

Security scan passed
Scanned at: 2026-04-06T16:24:46.159857
Tool: gitleaks + pattern-based validation
Content hash: bfcc70ffa4bfda5e6308942605ad8dc3cf62e8aeade5b00c6d9b774bebb190fb


#!/usr/bin/env python3
"""
Fetch Twitter/X Article with images using twitter-cli.

Usage:
    python fetch_article.py <article_url> [output_dir]

Example:
    python fetch_article.py https://x.com/HiTw93/status/2040047268221608281 ./Clippings

Features:
    - Fetches structured data via twitter-cli
    - Downloads all images to attachments folder
    - Generates Markdown with embedded image references
"""

import sys
import os
import re
import subprocess
import argparse
from pathlib import Path
from datetime import datetime


def run_twitter_cli(url: str) -> dict:
    """Fetch article data using twitter-cli via uv run."""
    cmd = ["uv", "run", "--with", "twitter-cli", "twitter", "article", url]

    result = subprocess.run(cmd, capture_output=True, text=True)

    if result.returncode != 0:
        print(f"Error fetching article: {result.stderr}", file=sys.stderr)
        sys.exit(1)

    return parse_yaml_output(result.stdout)


def run_jina_api(url: str) -> str:
    """Fetch article text with images using Jina API."""
    api_key = os.getenv("JINA_API_KEY", "")
    jina_url = f"https://r.jina.ai/{url}"

    cmd = ["curl", "-s", jina_url]
    if api_key:
        cmd.extend(["-H", f"Authorization: Bearer {api_key}"])

    result = subprocess.run(cmd, capture_output=True, text=True)

    if result.returncode != 0:
        print(f"Warning: Jina API failed: {result.stderr}", file=sys.stderr)
        return ""

    return result.stdout


def parse_yaml_output(output: str) -> dict:
    """Parse twitter-cli YAML output into dict."""
    try:
        import yaml
        data = yaml.safe_load(output)
        if data.get("ok") and "data" in data:
            return data["data"]
        return data
    except ImportError:
        print("Error: PyYAML required. Install with: uv pip install pyyaml", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Error parsing YAML: {e}", file=sys.stderr)
        sys.exit(1)


def extract_image_urls(text: str) -> list:
    """Extract image URLs from markdown text."""
    # Extract all pbs.twimg.com URLs (note: twimg not twitter)
    pattern = r'https://pbs\.twimg\.com/media/[^\s\)"\']+'
    matches = re.findall(pattern, text)

    # Deduplicate and normalize to large size
    seen = set()
    urls = []
    for url in matches:
        base_url = url.split('?')[0]
        if base_url not in seen:
            seen.add(base_url)
            urls.append(f"{base_url}?format=jpg&name=large")

    return urls


def download_images(image_urls: list, attachments_dir: Path) -> list:
    """Download images and return list of local paths."""
    attachments_dir.mkdir(parents=True, exist_ok=True)
    local_paths = []

    for i, url in enumerate(image_urls, 1):
        filename = f"{i:02d}-image.jpg"
        filepath = attachments_dir / filename

        cmd = ["curl", "-sL", url, "-o", str(filepath)]
        result = subprocess.run(cmd, capture_output=True)

        if result.returncode == 0 and filepath.exists() and filepath.stat().st_size > 0:
            local_paths.append(f"attachments/{attachments_dir.name}/{filename}")
            print(f"  ✓ {filename}")
        else:
            print(f"  ✗ Failed: {filename}")

    return local_paths


def replace_image_urls(text: str, image_urls: list, local_paths: list) -> str:
    """Replace remote image URLs with local paths in markdown text."""
    for remote_url, local_path in zip(image_urls, local_paths):
        # Extract base URL pattern
        base_url = remote_url.split('?')[0].replace('?format=jpg&name=large', '')
        # Replace all variations of this URL
        pattern = re.escape(base_url) + r'(\?[^\)]*)?'
        text = re.sub(pattern, local_path, text)
    return text


def sanitize_filename(name: str) -> str:
    """Sanitize string for use i