Fetch Url

#!/usr/bin/env -S uv run --script
#
# /// script
# requires-python = ">=3.14"
# dependencies = [
#     "playwright>=1.49.0",
#     "rich>=14.2.0",
#     "trafilatura>=2.0.0",
#     "typer>=0.20.1",
# ]
# ///

from __future__ import annotations

from pathlib import Path
from urllib.parse import urlparse

import typer
from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from playwright.sync_api import sync_playwright
from rich.console import Console
from rich.panel import Panel
import trafilatura

APP = typer.Typer(add_completion=False)
CONSOLE = Console()


def detect_browser_path() -> str | None:
    """Try common local browser paths to avoid Playwright download."""

    candidates = [
        "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
        "/Applications/Chromium.app/Contents/MacOS/Chromium",
        "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
        "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser",
        "/Applications/Arc.app/Contents/MacOS/Arc",
        "/usr/bin/google-chrome",
        "/usr/bin/google-chrome-stable",
        "/usr/bin/chromium",
        "/usr/bin/chromium-browser",
        "/usr/bin/microsoft-edge",
        "/usr/bin/microsoft-edge-stable",
        "/usr/bin/brave-browser",
        "/usr/bin/brave-browser-stable",
        "/snap/bin/chromium",
        "/snap/bin/brave",
    ]
    for path in candidates:
        if Path(path).exists():
            return path
    return None


def render_html(url: str, timeout_ms: int, browser_path: str | None) -> str:
    """使用 Playwright 渲染页面并返回完整 HTML。"""

    with sync_playwright() as playwright:
        launch_options: dict[str, Any] = {"headless": True}
        if browser_path:
            launch_options["executable_path"] = browser_path
        browser = playwright.chromium.launch(**launch_options)
        context = browser.new_context()
        page = context.new_page()
        page.goto(url, wait_until="networkidle", timeout=timeout_ms)
        html = page.content()
        context.close()
        browser.close()
    return html


def extract_content(html: str, url: str, output_format: str) -> str:
    """使用 trafilatura 从 HTML 提取内容。"""

    content = trafilatura.extract(
        html,
        url=url,
        output_format=output_format,
        include_formatting=True,
        include_links=True,
    )
    if not content:
        raise ValueError("Failed to extract main content from the rendered HTML.")
    return content


@APP.command()
def fetch(
    url: str = typer.Argument(..., help="Target URL to render into content."),
    output: Path | None = typer.Option(None, help="Write output to file instead of stdout."),
    timeout_ms: int = typer.Option(60000, help="Playwright navigation timeout in milliseconds."),
    browser_path: Path | None = typer.Option(
        None,
        help="Optional local Chromium-based browser path. Auto-detected if omitted.",
    ),
    output_format: str = typer.Option(
        "markdown",
        help="Output format: csv, html, json, markdown, raw-html, txt, xml, xmltei.",
    ),
) -> None:
    """通过 Playwright 渲染并用 trafilatura 提取内容。"""
    parsed = urlparse(url)
    if parsed.scheme not in {"http", "https"}:
        raise typer.BadParameter("Only http or https URLs are supported.")

    resolved_browser_path = str(browser_path) if browser_path else detect_browser_path()
    try:
        html = render_html(url, timeout_ms=timeout_ms, browser_path=resolved_browser_path)
        content = html if output_format == "raw-html" else extract_content(html, url, output_format)
    except PlaywrightTimeoutError as exc:
        CONSOLE.print(
            Panel.fit(
                f"[red]Playwright timeout[/red]\n{exc}",
                title="Request Failed",
            )
        )
        raise typer.Exit(code=1) from exc
    except ValueError as exc:
        CONSOLE.print(
            Panel.fit(
                f"[red]Extraction failed[/red]\n{exc}",
                title="Request Failed",
            )
        )
        raise typer.Exit(code=1) from exc
    except PlaywrightError as exc:
        hint = "Install Playwright browsers with: uv run playwright install chromium"
        CONSOLE.print(
            Panel.fit(
                f"[red]Playwright launch failed[/red]\\n{exc}\\n{hint}",
                title="Request Failed",
            )
        )
        raise typer.Exit(code=1) from exc

    if output:
        output.write_text(content, encoding="utf-8")
        CONSOLE.print(f"[green]Saved output to[/green] {output}")
    else:
        CONSOLE.print(content, markup=False)


if __name__ == "__main__":
    APP()

{
  "schema_version": "2.0",
  "meta": {
    "generated_at": "2026-01-17T01:52:37.197Z",
    "slug": "dcjanus-fetch-url",
    "source_url": "https://github.com/DCjanus/prompts/tree/master/skills/fetch-url",
    "source_ref": "master",
    "model": "claude",
    "analysis_version": "3.0.0",
    "source_type": "community",
    "content_hash": "80536e9611e9ec0234ebbd2b676723ae86e944c8c1b552ee83aab0d4daeff77d",
    "tree_hash": "e3fe57ca613e135db58884519a7aa05acb4e9ff7b2f21a5b6953f98ba31750ea"
  },
  "skill": {
    "name": "fetch-url",
    "description": "渲染网页 URL，去噪提取正文并输出为 Markdown（默认）或其他格式/原始 HTML，以减少 Token。",
    "summary": "渲染网页 URL，去噪提取正文并输出为 Markdown（默认）或其他格式/原始 HTML，以减少 Token。",
    "icon": "🔗",
    "version": "1.0.0",
    "author": "DCjanus",
    "license": "MIT",
    "category": "productivity",
    "tags": [
      "web",
      "content extraction",
      "markdown",
      "documentation"
    ],
    "supported_tools": [
      "claude",
      "codex",
      "claude-code"
    ],
    "risk_factors": [
      "scripts",
      "network",
      "filesystem",
      "external_commands"
    ]
  },
  "security_audit": {
    "risk_level": "low",
    "is_blocked": false,
    "safe_to_publish": true,
    "summary": "This is a legitimate web scraping skill that renders URLs using Playwright and extracts content via trafilatura. All detected patterns are consistent with the stated purpose. The network requests (page.goto) are core functionality for web scraping. The browser path detection (lines 33-52) is a performance optimization, not reconnaissance. All 10 HIGH severity 'weak cryptographic algorithm' findings are false positives - no cryptographic code exists in the codebase. The MEDIUM 'external_commands' findings are documentation examples, not actual command execution. Input validation (line 104) restricts URLs to http/https only.",
    "risk_factor_evidence": [
      {
        "factor": "scripts",
        "evidence": [
          {
            "file": "scripts/fetch_url.py",
            "line_start": 1,
            "line_end": 146
          }
        ]
      },
      {
        "factor": "network",
        "evidence": [
          {
            "file": "scripts/fetch_url.py",
            "line_start": 56,
            "line_end": 70
          }
        ]
      },
      {
        "factor": "filesystem",
        "evidence": [
          {
            "file": "scripts/fetch_url.py",
            "line_start": 137,
            "line_end": 138
          }
        ]
      },
      {
        "factor": "external_commands",
        "evidence": [
          {
            "file": "scripts/fetch_url.py",
            "line_start": 59,
            "line_end": 63
          }
        ]
      }
    ],
    "critical_findings": [],
    "high_findings": [],
    "medium_findings": [],
    "low_findings": [],
    "dangerous_patterns": [],
    "files_scanned": 3,
    "total_lines": 411,
    "audit_model": "claude",
    "audited_at": "2026-01-17T01:52:37.197Z"
  },
  "content": {
    "user_title": "Fetch and render web page content",
    "value_statement": "Web pages contain heavy HTML that wastes tokens. This skill renders URLs with a headless browser, extracts clean text, and outputs as Markdown or other formats. Perfect for turning documentation and articles into efficient AI input.",
    "seo_keywords": [
      "fetch url",
      "web scraping",
      "markdown converter",
      "content extraction",
      "Claude",
      "Codex",
      "Claude Code",
      "token reduction",
      "HTML to Markdown",
      "web page renderer"
    ],
    "actual_capabilities": [
      "Renders URLs using headless Chromium browser via Playwright",
      "Extracts clean text content using trafilatura library",
      "Outputs in 8 formats: markdown, HTML, JSON, CSV, TXT, XML, XML-TEI, raw HTML",
      "Auto-detects local browser installations to avoid downloads",
      "Configurable timeout and browser path options",
      "Writes output directly to file or stdout"
    ],
    "limitations": [
      "Only supports http and https URL schemes",
      "Requires Chromium-based browser (local or Playwright-installed)",
      "Cannot render pages requiring authentication or login",
      "JavaScript-heavy pages may have reduced content"
    ],
    "use_cases": [
      {
        "target_user": "Documentation writers",
        "title": "Convert online docs",
        "description": "Turn web documentation into local Markdown files for offline access and cleaner editing."
      },
      {
        "target_user": "AI practitioners",
        "title": "Optimize AI context",
        "description": "Convert articles and guides to Markdown to maximize useful content within token limits."
      },
      {
        "target_user": "Researchers",
        "title": "Archive web content",
        "description": "Extract and save content from multiple URLs in structured formats for analysis."
      }
    ],
    "prompt_templates": [
      {
        "title": "Basic fetch",
        "scenario": "Get page content",
        "prompt": "Use fetch-url to render https://example.com/docs/getting-started and output as markdown"
      },
      {
        "title": "Save to file",
        "scenario": "Archive page content",
        "prompt": "Use fetch-url to extract https://api.example.com/docs --output ./api-docs.md --output-format markdown"
      },
      {
        "title": "Custom format",
        "scenario": "Get raw HTML",
        "prompt": "Use fetch-url to fetch https://blog.example.com/post --output-format raw-html --timeout-ms 30000"
      },
      {
        "title": "Multiple pages",
        "scenario": "Batch extraction",
        "prompt": "Extract these three URLs as JSON: docs page, API reference, and changelog. Save each to output files."
      }
    ],
    "output_examples": [
      {
        "input": "Fetch the Python documentation homepage as markdown",
        "output": [
          "# Python Documentation",
          "",
          "## Getting Started",
          "",
          "Python is a programming language that lets you work quickly...",
          "[Saved output to stdout - 2.3KB of Markdown content]",
          "Extracted 15 headings, 84 paragraphs, and 23 links from https://docs.python.org/3/"
        ]
      },
      {
        "input": "Convert a blog post to structured JSON",
        "output": [
          "{",
          "  \"title\": \"My Blog Post\",",
          "  \"author\": \"Jane Smith\",",
          "  \"date\": \"2024-01-15\",",
          "  \"content\": \"The article body goes here...\"",
          "}",
          "[Saved to output file]",
          "Extracted structured data from https://blog.example.com/post"
        ]
      },
      {
        "input": "Extract API documentation to text file",
        "output": [
          "API Reference Documentation",
          "===========================",
          "",
          "## Authentication",
          "",
          "All API requests require...",
          "[Saved to ./api-reference.txt - 15KB]",
          "Extracted 42 endpoints, 156 parameters, and 89 examples"
        ]
      }
    ],
    "best_practices": [
      "Use --timeout-ms flag for slow-loading pages to avoid hanging",
      "Specify --browser-path if you have Chrome/Edge installed locally",
      "Use --output-format raw-html when you need unmodified HTML"
    ],
    "anti_patterns": [
      "Do not try to fetch local file URLs (file:// scheme not supported)",
      "Avoid fetching pages behind login walls without proper authentication",
      "Do not use extremely long timeouts for unresponsive servers"
    ],
    "faq": [
      {
        "question": "Which browsers are supported?",
        "answer": "Any Chromium-based browser: Chrome, Edge, Brave, Chromium, or Playwright's bundled browser."
      },
      {
        "question": "What happens to JavaScript content?",
        "answer": "Playwright renders JavaScript, but dynamically loaded content may be missing if it loads after networkidle."
      },
      {
        "question": "How is this different from curl or wget?",
        "answer": "This tool uses a real browser to render JavaScript and extracts clean text, not raw HTML source."
      },
      {
        "question": "Is my browsing data saved?",
        "answer": "No persistent data. Each run creates a fresh browser context that is closed after extraction."
      },
      {
        "question": "Why does it need a browser?",
        "answer": "Modern web pages use JavaScript to load content. A browser renderer captures the final rendered page."
      },
      {
        "question": "Can this access authenticated content?",
        "answer": "Not directly. The tool runs headless without login credentials. Authenticated content requires separate handling."
      }
    ]
  },
  "file_structure": [
    {
      "name": "scripts",
      "type": "dir",
      "path": "scripts",
      "children": [
        {
          "name": "fetch_url.py",
          "type": "file",
          "path": "scripts/fetch_url.py",
          "lines": 146
        }
      ]
    },
    {
      "name": "SKILL.md",
      "type": "file",
      "path": "SKILL.md",
      "lines": 28
    }
  ]
}