Twitter Reader

Security scan passed
Scanned at: 2026-04-06T16:24:46.159857
Tool: gitleaks + pattern-based validation
Content hash: bfcc70ffa4bfda5e6308942605ad8dc3cf62e8aeade5b00c6d9b774bebb190fb

#!/usr/bin/env python3
"""
Fetch Twitter/X Article with images using twitter-cli.

Usage:
    python fetch_article.py <article_url> [output_dir]

Example:
    python fetch_article.py https://x.com/HiTw93/status/2040047268221608281 ./Clippings

Features:
    - Fetches structured data via twitter-cli
    - Downloads all images to attachments folder
    - Generates Markdown with embedded image references
"""

import sys
import os
import re
import subprocess
import argparse
from pathlib import Path
from datetime import datetime


def run_twitter_cli(url: str) -> dict:
    """Fetch article data using twitter-cli via uv run."""
    cmd = ["uv", "run", "--with", "twitter-cli", "twitter", "article", url]

    result = subprocess.run(cmd, capture_output=True, text=True)

    if result.returncode != 0:
        print(f"Error fetching article: {result.stderr}", file=sys.stderr)
        sys.exit(1)

    return parse_yaml_output(result.stdout)


def run_jina_api(url: str) -> str:
    """Fetch article text with images using Jina API."""
    api_key = os.getenv("JINA_API_KEY", "")
    jina_url = f"https://r.jina.ai/{url}"

    cmd = ["curl", "-s", jina_url]
    if api_key:
        cmd.extend(["-H", f"Authorization: Bearer {api_key}"])

    result = subprocess.run(cmd, capture_output=True, text=True)

    if result.returncode != 0:
        print(f"Warning: Jina API failed: {result.stderr}", file=sys.stderr)
        return ""

    return result.stdout


def parse_yaml_output(output: str) -> dict:
    """Parse twitter-cli YAML output into dict."""
    try:
        import yaml
        data = yaml.safe_load(output)
        if data.get("ok") and "data" in data:
            return data["data"]
        return data
    except ImportError:
        print("Error: PyYAML required. Install with: uv pip install pyyaml", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Error parsing YAML: {e}", file=sys.stderr)
        sys.exit(1)


def extract_image_urls(text: str) -> list:
    """Extract image URLs from markdown text."""
    # Extract all pbs.twimg.com URLs (note: twimg not twitter)
    pattern = r'https://pbs\.twimg\.com/media/[^\s\)"\']+'
    matches = re.findall(pattern, text)

    # Deduplicate and normalize to large size
    seen = set()
    urls = []
    for url in matches:
        base_url = url.split('?')[0]
        if base_url not in seen:
            seen.add(base_url)
            urls.append(f"{base_url}?format=jpg&name=large")

    return urls


def download_images(image_urls: list, attachments_dir: Path) -> list:
    """Download images and return list of local paths."""
    attachments_dir.mkdir(parents=True, exist_ok=True)
    local_paths = []

    for i, url in enumerate(image_urls, 1):
        filename = f"{i:02d}-image.jpg"
        filepath = attachments_dir / filename

        cmd = ["curl", "-sL", url, "-o", str(filepath)]
        result = subprocess.run(cmd, capture_output=True)

        if result.returncode == 0 and filepath.exists() and filepath.stat().st_size > 0:
            local_paths.append(f"attachments/{attachments_dir.name}/{filename}")
            print(f"  ✓ {filename}")
        else:
            print(f"  ✗ Failed: {filename}")

    return local_paths


def replace_image_urls(text: str, image_urls: list, local_paths: list) -> str:
    """Replace remote image URLs with local paths in markdown text."""
    for remote_url, local_path in zip(image_urls, local_paths):
        # Extract base URL pattern
        base_url = remote_url.split('?')[0].replace('?format=jpg&name=large', '')
        # Replace all variations of this URL
        pattern = re.escape(base_url) + r'(\?[^\)]*)?'
        text = re.sub(pattern, local_path, text)
    return text


def sanitize_filename(name: str) -> str:
    """Sanitize string for use in filename."""
    # Remove special chars, keep alphanumeric, CJK, and some safe chars
    name = re.sub(r'[^\w\s\-\u4e00-\u9fff]', '', name)
    name = re.sub(r'\s+', '-', name.strip())
    return name[:60]  # Limit length


def generate_markdown(data: dict, text: str, image_urls: list, local_paths: list, source_url: str) -> str:
    """Generate complete Markdown document."""
    # Parse date
    created = data.get("createdAtLocal", "")
    if created:
        date_str = created[:10]
    else:
        date_str = datetime.now().strftime("%Y-%m-%d")

    author = data.get("author", {})
    metrics = data.get("metrics", {})
    title = data.get("articleTitle", "Untitled")

    # Build frontmatter
    md = f"""---
source: {source_url}
author: {author.get("name", "")}
date: {date_str}
likes: {metrics.get("likes", 0)}
retweets: {metrics.get("retweets", 0)}
bookmarks: {metrics.get("bookmarks", 0)}
---

# {title}

"""

    # Replace image URLs with local paths
    if image_urls and local_paths:
        text = replace_image_urls(text, image_urls, local_paths)

    md += text
    return md


def main():
    parser = argparse.ArgumentParser(description="Fetch Twitter/X Article with images")
    parser.add_argument("url", help="Twitter/X article URL")
    parser.add_argument("output_dir", nargs="?", default=".", help="Output directory (default: current)")
    args = parser.parse_args()

    if not args.url.startswith(("https://x.com/", "https://twitter.com/")):
        print("Error: URL must be from x.com or twitter.com", file=sys.stderr)
        sys.exit(1)

    print(f"Fetching: {args.url}")
    print("-" * 50)

    # Fetch metadata from twitter-cli
    print("Getting metadata...")
    data = run_twitter_cli(args.url)

    title = data.get("articleTitle", "")
    if not title:
        print("Error: Could not fetch article data", file=sys.stderr)
        sys.exit(1)

    author = data.get("author", {})

    print(f"Title: {title}")
    print(f"Author: {author.get('name', 'Unknown')}")
    print(f"Likes: {data.get('metrics', {}).get('likes', 0)}")

    # Fetch content with images from Jina API
    print("\nGetting content and images...")
    jina_content = run_jina_api(args.url)

    # Use Jina content if available, otherwise fall back to twitter-cli text
    if jina_content:
        text = jina_content
        # Remove Jina header lines to get clean markdown
        # Find "Markdown Content:" and keep everything after it
        marker = "Markdown Content:"
        idx = text.find(marker)
        if idx != -1:
            text = text[idx + len(marker):].lstrip()
    else:
        text = data.get("articleText", "")

    # Extract image URLs
    image_urls = extract_image_urls(text)
    print(f"Images: {len(image_urls)}")

    # Setup output paths
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    # Create attachments folder
    date_str = data.get("createdAtLocal", "")[:10] if data.get("createdAtLocal") else datetime.now().strftime("%Y-%m-%d")
    safe_author = sanitize_filename(author.get("screenName", "unknown"))
    safe_title = sanitize_filename(title)
    attachments_name = f"{date_str}-{safe_author}-{safe_title[:30]}"
    attachments_dir = output_dir / "attachments" / attachments_name

    # Download images
    local_paths = []
    if image_urls:
        print(f"\nDownloading {len(image_urls)} images...")
        local_paths = download_images(image_urls, attachments_dir)

    # Generate Markdown
    md_content = generate_markdown(data, text, image_urls, local_paths, args.url)

    # Save Markdown
    md_filename = f"{date_str}-{safe_title}.md"
    md_path = output_dir / md_filename
    md_path.write_text(md_content, encoding="utf-8")

    print(f"\n✓ Saved: {md_path}")
    if local_paths:
        print(f"✓ Images: {attachments_dir} ({len(local_paths)} downloaded)")

    return md_path


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Fetch Twitter/X post content using jina.ai API.

Usage:
    python fetch_tweet.py <tweet_url> [output_file]

Example:
    python fetch_tweet.py https://x.com/dabit3/status/2009131298250428923 tweet.md

Requires:
    JINA_API_KEY environment variable set with your Jina.ai API key
"""

import sys
import os
import argparse
import subprocess
from pathlib import Path


def fetch_tweet(url: str, output_file: str = None) -> str:
    """Fetch tweet content using jina.ai API via curl."""
    api_key = os.getenv("JINA_API_KEY")
    if not api_key:
        print("Error: JINA_API_KEY environment variable is not set", file=sys.stderr)
        print("Get your API key from https://jina.ai/ and set:", file=sys.stderr)
        print("  export JINA_API_KEY='your_api_key_here'", file=sys.stderr)
        sys.exit(1)

    jina_api_url = f"https://r.jina.ai/{url}"

    cmd = [
        "curl", "-s", jina_api_url,
        "-H", f"Authorization: Bearer {api_key}"
    ]

    result = subprocess.run(cmd, capture_output=True, text=True)

    if result.returncode != 0:
        print(f"Error fetching tweet: {result.stderr}", file=sys.stderr)
        sys.exit(1)

    content = result.stdout

    if output_file:
        Path(output_file).write_text(content, encoding="utf-8")
        print(f"Saved to {output_file}")

    return content


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Fetch Twitter/X post content")
    parser.add_argument("url", help="Twitter/X post URL")
    parser.add_argument("output", nargs="?", help="Optional output file path")
    args = parser.parse_args()

    if not args.url.startswith(("https://x.com/", "https://twitter.com/")):
        print("Error: URL must be from x.com or twitter.com (HTTPS only)", file=sys.stderr)
        sys.exit(1)

    content = fetch_tweet(args.url, args.output)

    if not args.output:
        print(content)

#!/bin/bash
# Fetch Twitter/X post content using jina.ai API
# Usage: ./fetch_tweets.sh <url1> [url2] ...
# Requires: JINA_API_KEY environment variable

if [ -z "$JINA_API_KEY" ]; then
    echo "Error: JINA_API_KEY environment variable is not set" >&2
    echo "Get your API key from https://jina.ai/ and set:" >&2
    echo "  export JINA_API_KEY='your_api_key_here'" >&2
    exit 1
fi

if [ $# -eq 0 ]; then
    echo "Usage: $0 <tweet_url> [tweet_url2] ..."
    echo "Example: $0 https://x.com/dabit3/status/2009131298250428923"
    exit 1
fi

for url in "$@"; do
    if [[ ! "$url" =~ ^https?://(x\.com|twitter\.com)/ ]]; then
        echo "Skipping invalid URL: $url" >&2
        continue
    fi

    echo "Fetching: $url"
    curl -s "https://r.jina.ai/${url}" \
        -H "Authorization: Bearer ${JINA_API_KEY}"
    echo ""
    echo "---"
    echo ""
done