Web Scraper

#!/usr/bin/env python3
"""Crawl JavaScript-rendered (dynamic) web pages using Crawl4AI.

Uses a headless Chromium browser to render pages that require JavaScript,
then extracts clean Markdown content. Use this when fetch_page.py returns
empty or incomplete content (SPAs, React/Vue apps, etc.).

Dependencies: pip install crawl4ai && crawl4ai-setup
"""

import sys
import argparse
import asyncio


def setup_encoding():
    """Setup proper encoding for Windows console output."""
    if sys.platform == "win32":
        import io
        try:
            sys.stdout.reconfigure(encoding='utf-8', errors='replace')
            sys.stderr.reconfigure(encoding='utf-8', errors='replace')
        except (AttributeError, io.UnsupportedOperation):
            sys.stdout = io.TextIOWrapper(
                sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )
            sys.stderr = io.TextIOWrapper(
                sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )


def check_dependencies():
    """Check that Crawl4AI is installed."""
    try:
        import crawl4ai  # noqa: F401
    except ImportError:
        print("Error: crawl4ai not installed.", file=sys.stderr)
        print("Install with:", file=sys.stderr)
        print("  pip install crawl4ai", file=sys.stderr)
        print("  crawl4ai-setup", file=sys.stderr)
        sys.exit(1)


async def crawl_page(url, wait_seconds=3, css_selector=None, scroll=False):
    """Crawl a page with headless browser and return Markdown content."""
    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
    from crawl4ai.content_filter_strategy import PruningContentFilter
    from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator

    browser_conf = BrowserConfig(
        headless=True,
        verbose=False,
    )

    md_generator = DefaultMarkdownGenerator(
        content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
    )

    run_conf = CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        markdown_generator=md_generator,
        page_timeout=60000,  # 60s
        wait_until="networkidle",
    )

    # Add wait time if specified
    if wait_seconds > 0:
        run_conf.delay_before_return_html = wait_seconds

    # Wait for specific CSS selector
    if css_selector:
        run_conf.wait_for = f"css:{css_selector}"

    async with AsyncWebCrawler(config=browser_conf) as crawler:
        result = await crawler.arun(url=url, config=run_conf)

        if not result.success:
            return None, result.error_message or "Unknown error"

        # Get the best available markdown
        md = ""
        if result.markdown:
            if hasattr(result.markdown, 'fit_markdown') and result.markdown.fit_markdown:
                md = result.markdown.fit_markdown
            elif hasattr(result.markdown, 'raw_markdown') and result.markdown.raw_markdown:
                md = result.markdown.raw_markdown
            elif isinstance(result.markdown, str):
                md = result.markdown

        title = ""
        if hasattr(result, 'metadata') and result.metadata:
            title = result.metadata.get('title', '')

        return {
            "title": title,
            "url": result.url or url,
            "markdown": md,
            "status_code": getattr(result, 'status_code', None),
        }, None


async def crawl_with_scroll(url, wait_seconds=3, css_selector=None):
    """Crawl with infinite scroll support."""
    from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
    from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
    from crawl4ai.content_filter_strategy import PruningContentFilter

    browser_conf = BrowserConfig(
        headless=True,
        verbose=False,
    )

    # JavaScript to scroll to bottom
    scroll_js = """
    async () => {
        await new Promise((resolve) => {
            let totalHeight = 0;
            const distance = 500;
            const timer = setInterval(() => {
                const scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;
                if (totalHeight >= scrollHeight) {
                    clearInterval(timer);
                    resolve();
                }
            }, 300);
            // Safety timeout
            setTimeout(() => { clearInterval(timer); resolve(); }, 15000);
        });
    }
    """

    md_generator = DefaultMarkdownGenerator(
        content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
    )

    run_conf = CrawlerRunConfig(
        cache_mode=CacheMode.BYPASS,
        markdown_generator=md_generator,
        page_timeout=60000,
        js_code=scroll_js,
        wait_until="networkidle",
    )

    if wait_seconds > 0:
        run_conf.delay_before_return_html = wait_seconds

    if css_selector:
        run_conf.wait_for = f"css:{css_selector}"

    async with AsyncWebCrawler(config=browser_conf) as crawler:
        result = await crawler.arun(url=url, config=run_conf)

        if not result.success:
            return None, result.error_message or "Unknown error"

        md = ""
        if result.markdown:
            if hasattr(result.markdown, 'fit_markdown') and result.markdown.fit_markdown:
                md = result.markdown.fit_markdown
            elif hasattr(result.markdown, 'raw_markdown') and result.markdown.raw_markdown:
                md = result.markdown.raw_markdown
            elif isinstance(result.markdown, str):
                md = result.markdown

        title = ""
        if hasattr(result, 'metadata') and result.metadata:
            title = result.metadata.get('title', '')

        return {
            "title": title,
            "url": result.url or url,
            "markdown": md,
            "status_code": getattr(result, 'status_code', None),
        }, None


def main():
    setup_encoding()
    check_dependencies()

    parser = argparse.ArgumentParser(
        description="Crawl JavaScript-rendered pages with headless browser"
    )
    parser.add_argument("url", help="URL to crawl")
    parser.add_argument("--wait", type=int, default=3,
                        help="Seconds to wait after page load (default: 3)")
    parser.add_argument("--selector", type=str, default=None,
                        help="CSS selector to wait for before extracting")
    parser.add_argument("--scroll", action="store_true",
                        help="Scroll to bottom to trigger lazy loading")
    parser.add_argument("--save", type=str, default=None,
                        help="Also save output to this file path")
    parser.add_argument("--max-length", type=int, default=None,
                        help="Truncate output to N characters")

    args = parser.parse_args()

    url = args.url.strip()
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    print(f"Crawling (dynamic): {url}", file=sys.stderr)
    print(f"Options: wait={args.wait}s, selector={args.selector}, scroll={args.scroll}", file=sys.stderr)

    # Run async crawl
    if args.scroll:
        data, error = asyncio.run(crawl_with_scroll(url, args.wait, args.selector))
    else:
        data, error = asyncio.run(crawl_page(url, args.wait, args.selector))

    if error:
        print(f"Error: crawl failed: {error}", file=sys.stderr)
        sys.exit(1)

    if not data or not data["markdown"]:
        print("Warning: no content extracted from page", file=sys.stderr)
        print("[No content could be extracted from this page]")
        sys.exit(0)

    # Build output
    parts = []
    if data["title"]:
        parts.append(f"# {data['title']}\n")
    parts.append(f"**Source**: {data['url']}")
    if data.get("status_code"):
        parts.append(f"**Status**: {data['status_code']}")
    parts.append("\n---\n")
    parts.append(data["markdown"])

    output = "\n".join(parts)

    # Truncate if requested
    if args.max_length and len(output) > args.max_length:
        output = output[:args.max_length] + f"\n\n[... truncated at {args.max_length} characters, total {len(output)}]"

    print(output)

    content_len = len(data["markdown"])
    print(f"\nExtracted: {content_len} characters (dynamic crawl)", file=sys.stderr)

    # Save to file if requested
    if args.save:
        try:
            with open(args.save, "w", encoding="utf-8") as f:
                f.write(output)
            print(f"Saved to: {args.save}", file=sys.stderr)
        except Exception as e:
            print(f"Error saving file: {e}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Extract and categorize all links from a web page.

Fetches the page and extracts all <a> tags, categorizing them as
internal, external, or resource links. Useful for site navigation
and discovery before deeper scraping.

Dependencies: pip install requests beautifulsoup4
"""

import sys
import argparse
import json
import re
from urllib.parse import urlparse, urljoin


def setup_encoding():
    """Setup proper encoding for Windows console output."""
    if sys.platform == "win32":
        import io
        try:
            sys.stdout.reconfigure(encoding='utf-8', errors='replace')
            sys.stderr.reconfigure(encoding='utf-8', errors='replace')
        except (AttributeError, io.UnsupportedOperation):
            sys.stdout = io.TextIOWrapper(
                sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )
            sys.stderr = io.TextIOWrapper(
                sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )


def check_dependencies():
    """Check that required packages are installed."""
    missing = []
    try:
        import requests  # noqa: F401
    except ImportError:
        missing.append("requests")
    try:
        from bs4 import BeautifulSoup  # noqa: F401
    except ImportError:
        missing.append("beautifulsoup4")
    if missing:
        print(f"Error: missing dependencies: {', '.join(missing)}", file=sys.stderr)
        print(f"Install with: pip install {' '.join(missing)}", file=sys.stderr)
        sys.exit(1)


RESOURCE_EXTENSIONS = {
    '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx',
    '.zip', '.rar', '.tar', '.gz', '.7z',
    '.jpg', '.jpeg', '.png', '.gif', '.svg', '.webp', '.ico',
    '.mp3', '.mp4', '.avi', '.mov', '.webm',
    '.css', '.js', '.woff', '.woff2', '.ttf', '.eot',
}


def classify_link(href, base_domain):
    """Classify a link as internal, external, or resource."""
    parsed = urlparse(href)

    # Check for resource files
    path_lower = parsed.path.lower()
    for ext in RESOURCE_EXTENSIONS:
        if path_lower.endswith(ext):
            return "resource"

    # Check domain
    link_domain = parsed.netloc.lower()
    if not link_domain or link_domain == base_domain:
        return "internal"

    # Check for common CDN / same-org subdomains
    base_parts = base_domain.split(".")
    link_parts = link_domain.split(".")
    if len(base_parts) >= 2 and len(link_parts) >= 2:
        if base_parts[-2:] == link_parts[-2:]:
            return "internal"

    return "external"


def extract_links(html, base_url):
    """Extract all links from HTML."""
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "html.parser")
    base_domain = urlparse(base_url).netloc.lower()
    links = []
    seen = set()

    for a_tag in soup.find_all("a", href=True):
        href = a_tag["href"].strip()

        # Skip anchors, javascript:, mailto:, tel:
        if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):
            continue

        # Resolve relative URLs
        full_url = urljoin(base_url, href)

        # Deduplicate
        if full_url in seen:
            continue
        seen.add(full_url)

        # Extract link text
        text = a_tag.get_text(strip=True) or ""
        text = re.sub(r'\s+', ' ', text)  # normalize whitespace
        if len(text) > 100:
            text = text[:100] + "..."

        link_type = classify_link(full_url, base_domain)

        links.append({
            "url": full_url,
            "text": text,
            "type": link_type,
        })

    return links


def format_markdown(links, url, filter_pattern=None, external_only=False):
    """Format links as Markdown."""
    # Apply filters
    filtered = links
    if external_only:
        filtered = [link for link in filtered if link["type"] == "external"]
    if filter_pattern:
        try:
            pattern = re.compile(filter_pattern, re.IGNORECASE)
            filtered = [link for link in filtered if pattern.search(link["url"])]
        except re.error as e:
            print(f"Warning: invalid regex pattern '{filter_pattern}': {e}", file=sys.stderr)

    # Group by type
    internal = [link for link in filtered if link["type"] == "internal"]
    external = [link for link in filtered if link["type"] == "external"]
    resources = [link for link in filtered if link["type"] == "resource"]

    parts = [f"# Links from {url}\n"]
    parts.append(f"Total: **{len(filtered)}** links ({len(internal)} internal, {len(external)} external, {len(resources)} resource)\n")

    if internal:
        parts.append("## Internal Links\n")
        for lk in internal:
            text = f" — {lk['text']}" if lk['text'] else ""
            parts.append(f"- {lk['url']}{text}")
        parts.append("")

    if external:
        parts.append("## External Links\n")
        for lk in external:
            text = f" — {lk['text']}" if lk['text'] else ""
            parts.append(f"- {lk['url']}{text}")
        parts.append("")

    if resources:
        parts.append("## Resource Links\n")
        for lk in resources:
            text = f" — {lk['text']}" if lk['text'] else ""
            parts.append(f"- {lk['url']}{text}")
        parts.append("")

    return "\n".join(parts)


def main():
    setup_encoding()
    check_dependencies()

    parser = argparse.ArgumentParser(
        description="Extract and categorize links from a web page"
    )
    parser.add_argument("url", help="URL to extract links from")
    parser.add_argument("--filter", type=str, default=None,
                        help="Regex pattern to filter URLs")
    parser.add_argument("--external-only", action="store_true",
                        help="Only show external links")
    parser.add_argument("--json", action="store_true",
                        help="Output as JSON instead of Markdown")
    parser.add_argument("--timeout", type=int, default=30,
                        help="Request timeout in seconds (default: 30)")

    args = parser.parse_args()

    import requests

    url = args.url.strip()
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    print(f"Extracting links from: {url}", file=sys.stderr)

    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
            "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
        ),
    }

    try:
        resp = requests.get(url, headers=headers, timeout=args.timeout, allow_redirects=True)
        resp.raise_for_status()
        if resp.encoding and resp.encoding.lower() != 'utf-8':
            resp.encoding = resp.apparent_encoding or resp.encoding
        html = resp.text
        final_url = resp.url
    except requests.exceptions.RequestException as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)

    links = extract_links(html, final_url)
    print(f"Found {len(links)} unique links", file=sys.stderr)

    if args.json:
        # Apply filters for JSON output too
        filtered = links
        if args.external_only:
            filtered = [lk for lk in filtered if lk["type"] == "external"]
        if args.filter:
            try:
                pattern = re.compile(args.filter, re.IGNORECASE)
                filtered = [lk for lk in filtered if pattern.search(lk["url"])]
            except re.error:
                pass
        print(json.dumps(filtered, indent=2, ensure_ascii=False))
    else:
        print(format_markdown(links, final_url, args.filter, args.external_only))


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Fetch a web page and extract readable content as clean Markdown.

Uses requests + BeautifulSoup + readability-lxml + html2text for lightweight,
fast extraction without a headless browser. Works well for articles, docs,
blogs, wikis, and most static websites.

Dependencies: pip install requests beautifulsoup4 readability-lxml html2text
"""

import sys
import argparse
import random
import time
from urllib.parse import urlparse


USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
    "(KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6_1) AppleWebKit/605.1.15 "
    "(KHTML, like Gecko) Version/17.2 Safari/605.1.15",
    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",
]

BASE_HEADERS = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
    "Accept-Encoding": "gzip, deflate, br",
    "Cache-Control": "no-cache",
    "Pragma": "no-cache",
    "Upgrade-Insecure-Requests": "1",
    "Sec-Ch-Ua": '"Not/A)Brand";v="8", "Chromium";v="121", "Google Chrome";v="121"',
    "Sec-Ch-Ua-Mobile": "?0",
    "Sec-Ch-Ua-Platform": '"Windows"',
    "Sec-Fetch-Dest": "document",
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
}

ANTI_BOT_MARKERS = (
    "captcha",
    "verify you are human",
    "security verification",
    "access denied",
    "robot check",
    "访问过于频繁",
    "安全验证",
)


def build_headers(url: str):
    parsed = urlparse(url)
    scheme = parsed.scheme or "https"
    headers = dict(BASE_HEADERS)
    headers["User-Agent"] = random.choice(USER_AGENTS)

    if parsed.netloc:
        headers["Host"] = parsed.netloc
        origin = f"{scheme}://{parsed.netloc}"
        headers["Origin"] = origin
        headers["Referer"] = origin + "/"

    return headers


def is_anti_bot_response(html: str, status_code: int) -> bool:
    if status_code in (403, 429):
        return True

    snippet = html[:2000].lower()
    for marker in ANTI_BOT_MARKERS:
        if marker.lower() in snippet:
            return True

    return False


def try_cloudscraper(url: str, headers: dict, timeout: int):
    try:
        import cloudscraper
    except ImportError:
        return None

    scraper = cloudscraper.create_scraper(
        browser={
            "browser": "chrome",
            "platform": "windows",
            "mobile": False,
        }
    )

    try:
        resp = scraper.get(url, headers=headers, timeout=timeout, allow_redirects=True)
        resp.raise_for_status()
    except Exception:
        return None

    if resp.encoding and resp.encoding.lower() != 'utf-8':
        resp.encoding = resp.apparent_encoding or resp.encoding

    return resp.text, resp.url, resp.status_code


def setup_encoding():
    """Setup proper encoding for Windows console output."""
    if sys.platform == "win32":
        import io
        try:
            sys.stdout.reconfigure(encoding='utf-8', errors='replace')
            sys.stderr.reconfigure(encoding='utf-8', errors='replace')
        except (AttributeError, io.UnsupportedOperation):
            sys.stdout = io.TextIOWrapper(
                sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )
            sys.stderr = io.TextIOWrapper(
                sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )


def check_dependencies():
    """Check that required packages are installed."""
    missing = []
    try:
        import requests  # noqa: F401
    except ImportError:
        missing.append("requests")
    try:
        from bs4 import BeautifulSoup  # noqa: F401
    except ImportError:
        missing.append("beautifulsoup4")
    try:
        from readability import Document  # noqa: F401
    except ImportError:
        missing.append("readability-lxml")
    try:
        import html2text  # noqa: F401
    except ImportError:
        missing.append("html2text")

    if missing:
        print(f"Error: missing dependencies: {', '.join(missing)}", file=sys.stderr)
        print(f"Install with: pip install {' '.join(missing)}", file=sys.stderr)
        sys.exit(1)


def fetch_url(url, timeout=30, max_attempts=3):
    """Fetch URL content, retrying through common anti-bot challenges."""
    import requests
    from requests.adapters import HTTPAdapter
    from urllib3.util.retry import Retry

    session = requests.Session()
    retry_strategy = Retry(
        total=0,
        connect=0,
        read=0,
        redirect=3,
        status=0,
        backoff_factor=0,
    )
    adapter = HTTPAdapter(max_retries=retry_strategy)
    session.mount("http://", adapter)
    session.mount("https://", adapter)

    last_error = None

    for attempt in range(1, max_attempts + 1):
        try:
            resp = session.get(
                url,
                headers=build_headers(url),
                timeout=timeout,
                allow_redirects=True,
            )
        except requests.exceptions.Timeout as e:
            last_error = f"request timed out after {timeout}s"
            print(f"Warning: {last_error} (attempt {attempt}/{max_attempts})", file=sys.stderr)
            if attempt < max_attempts:
                time.sleep(min(2 ** attempt, 4))
            continue
        except requests.exceptions.ConnectionError as e:
            last_error = f"connection failed: {e}"
            print(f"Warning: {last_error} (attempt {attempt}/{max_attempts})", file=sys.stderr)
            if attempt < max_attempts:
                time.sleep(min(2 ** attempt, 4))
            continue

        if resp.encoding and resp.encoding.lower() != 'utf-8':
            resp.encoding = resp.apparent_encoding or resp.encoding
        html_text = resp.text

        anti_bot = is_anti_bot_response(html_text, resp.status_code)

        if not anti_bot and resp.status_code >= 400:
            try:
                resp.raise_for_status()
            except requests.exceptions.HTTPError as e:
                last_error = f"HTTP {resp.status_code}: {e}"
                break

        if not anti_bot:
            return html_text, resp.url, resp.status_code

        last_error = (
            f"Detected anti-bot response (status {resp.status_code})"
        )
        print(
            f"Warning: {last_error} - retrying with different headers (attempt {attempt}/{max_attempts})",
            file=sys.stderr,
        )
        if attempt < max_attempts:
            time.sleep(min(2 ** attempt, 4))

    # cloudscraper fallback for challenging sites
    scraper_result = try_cloudscraper(url, build_headers(url), timeout)
    if scraper_result:
        html, final_url, status = scraper_result
        if not is_anti_bot_response(html, status):
            return scraper_result

    error_message = last_error or "unable to fetch page"
    print(f"Error: {error_message}", file=sys.stderr)
    sys.exit(1)


def extract_with_readability(html, url):
    """Extract main article content using readability-lxml."""
    from readability import Document

    doc = Document(html, url=url)
    title = doc.short_title()
    content_html = doc.summary()
    return title, content_html


def extract_with_selector(html, selector):
    """Extract content matching a CSS selector."""
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "html.parser")
    elements = soup.select(selector)
    if not elements:
        return None

    # Combine all matching elements
    parts = []
    for el in elements:
        parts.append(str(el))
    return "\n".join(parts)


def html_to_markdown(html, base_url=None):
    """Convert HTML to clean Markdown."""
    import html2text

    converter = html2text.HTML2Text()
    converter.body_width = 0  # Don't wrap lines
    converter.ignore_images = False
    converter.ignore_links = False
    converter.ignore_emphasis = False
    converter.protect_links = True
    converter.unicode_snob = True
    converter.mark_code = True
    converter.wrap_links = False
    converter.single_line_break = False

    if base_url:
        converter.baseurl = base_url

    md = converter.handle(html)

    # Clean up excessive blank lines
    import re
    md = re.sub(r'\n{3,}', '\n\n', md)
    return md.strip()


def extract_metadata(html):
    """Extract page metadata (title, description, etc.)."""
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(html, "html.parser")
    meta = {}

    # Title
    title_tag = soup.find("title")
    if title_tag:
        meta["title"] = title_tag.get_text(strip=True)

    # Meta description
    desc_tag = soup.find("meta", attrs={"name": "description"})
    if desc_tag and desc_tag.get("content"):
        meta["description"] = desc_tag["content"].strip()

    # OG tags
    for prop in ["og:title", "og:description", "og:type", "og:site_name"]:
        tag = soup.find("meta", attrs={"property": prop})
        if tag and tag.get("content"):
            meta[prop.replace("og:", "og_")] = tag["content"].strip()

    # Author
    author_tag = soup.find("meta", attrs={"name": "author"})
    if author_tag and author_tag.get("content"):
        meta["author"] = author_tag["content"].strip()

    # Published date
    for attr in ["article:published_time", "datePublished", "date"]:
        date_tag = soup.find("meta", attrs={"property": attr}) or soup.find("meta", attrs={"name": attr})
        if date_tag and date_tag.get("content"):
            meta["published"] = date_tag["content"].strip()
            break

    return meta


def main():
    setup_encoding()
    check_dependencies()

    parser = argparse.ArgumentParser(
        description="Fetch a web page and extract content as Markdown"
    )
    parser.add_argument("url", help="URL to fetch")
    parser.add_argument("--raw", action="store_true",
                        help="Output full page Markdown (no readability extraction)")
    parser.add_argument("--selector", type=str, default=None,
                        help="CSS selector to extract specific elements")
    parser.add_argument("--save", type=str, default=None,
                        help="Also save output to this file path")
    parser.add_argument("--max-length", type=int, default=None,
                        help="Truncate output to N characters")
    parser.add_argument("--timeout", type=int, default=30,
                        help="Request timeout in seconds (default: 30)")
    parser.add_argument("--max-attempts", type=int, default=3,
                        help="Max HTTP attempts before giving up (default: 3)")
    parser.add_argument("--no-metadata", action="store_true",
                        help="Skip metadata header in output")

    args = parser.parse_args()

    # Normalize URL
    url = args.url.strip()
    if not url.startswith(("http://", "https://")):
        url = "https://" + url

    print(f"Fetching: {url}", file=sys.stderr)

    # Fetch
    html, final_url, status = fetch_url(url, timeout=args.timeout, max_attempts=args.max_attempts)
    print(f"Status: {status}, Size: {len(html)} bytes", file=sys.stderr)

    if final_url != url:
        print(f"Redirected to: {final_url}", file=sys.stderr)

    # Extract metadata
    meta = extract_metadata(html) if not args.no_metadata else {}

    # Extract content
    if args.selector:
        # CSS selector mode
        selected_html = extract_with_selector(html, args.selector)
        if not selected_html:
            print(f"Warning: no elements matched selector '{args.selector}'", file=sys.stderr)
            print(f"[No elements matched CSS selector: {args.selector}]")
            sys.exit(0)
        title = meta.get("title", "")
        content_md = html_to_markdown(selected_html, base_url=final_url)
    elif args.raw:
        # Raw full-page mode
        title = meta.get("title", "")
        content_md = html_to_markdown(html, base_url=final_url)
    else:
        # Readability extraction mode (default)
        title, article_html = extract_with_readability(html, final_url)
        content_md = html_to_markdown(article_html, base_url=final_url)

    # Build output
    parts = []

    if not args.no_metadata and meta:
        parts.append(f"# {title or meta.get('title', 'Untitled')}")
        parts.append(f"\n**Source**: {final_url}")
        if meta.get("author"):
            parts.append(f"**Author**: {meta['author']}")
        if meta.get("published"):
            parts.append(f"**Published**: {meta['published']}")
        if meta.get("description"):
            parts.append(f"**Description**: {meta['description']}")
        parts.append("\n---\n")
    elif title and not args.no_metadata:
        parts.append(f"# {title}\n")

    parts.append(content_md)

    output = "\n".join(parts)

    # Truncate if requested
    if args.max_length and len(output) > args.max_length:
        output = output[:args.max_length] + f"\n\n[... truncated at {args.max_length} characters, total {len(output)}]"

    # Print to stdout
    print(output)

    content_length = len(content_md)
    print(f"\nExtracted: {content_length} characters", file=sys.stderr)

    # Save to file if requested
    if args.save:
        try:
            with open(args.save, "w", encoding="utf-8") as f:
                f.write(output)
            print(f"Saved to: {args.save}", file=sys.stderr)
        except Exception as e:
            print(f"Error saving file: {e}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Search the web using DuckDuckGo and return structured results.

No API key required. Returns results with title, URL, and snippet.

Dependencies: pip install ddgs
"""

import sys
import argparse
import json


def setup_encoding():
    """Setup proper encoding for Windows console output."""
    if sys.platform == "win32":
        import io
        try:
            sys.stdout.reconfigure(encoding='utf-8', errors='replace')
            sys.stderr.reconfigure(encoding='utf-8', errors='replace')
        except (AttributeError, io.UnsupportedOperation):
            sys.stdout = io.TextIOWrapper(
                sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )
            sys.stderr = io.TextIOWrapper(
                sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True
            )


def check_dependencies():
    """Check that required packages are installed."""
    try:
        from ddgs import DDGS  # noqa: F401
    except ImportError:
        try:
            from duckduckgo_search import DDGS  # noqa: F401
        except ImportError:
            print("Error: ddgs not installed.", file=sys.stderr)
            print("Install with: pip install ddgs", file=sys.stderr)
            sys.exit(1)


def _get_ddgs_class():
    """Import DDGS from ddgs (new) or duckduckgo_search (legacy)."""
    try:
        from ddgs import DDGS
        return DDGS
    except ImportError:
        from duckduckgo_search import DDGS
        return DDGS


def search_text(query, max_results=10, region="wt-wt"):
    """Perform a text search."""
    DDGS = _get_ddgs_class()
    ddgs = DDGS()
    try:
        # New ddgs package: positional 'query' arg
        results = list(ddgs.text(query, region=region, max_results=max_results))
    except TypeError:
        # Legacy duckduckgo_search: 'keywords' kwarg + context manager
        with DDGS() as d:
            results = list(d.text(keywords=query, region=region, max_results=max_results))
    return results


def search_news(query, max_results=10, region="wt-wt"):
    """Perform a news search."""
    DDGS = _get_ddgs_class()
    ddgs = DDGS()
    try:
        results = list(ddgs.news(query, region=region, max_results=max_results))
    except TypeError:
        with DDGS() as d:
            results = list(d.news(keywords=query, region=region, max_results=max_results))
    return results


def format_results_markdown(results, query, is_news=False):
    """Format search results as Markdown."""
    search_type = "News" if is_news else "Web"
    parts = [f"# {search_type} Search Results: {query}\n"]
    parts.append(f"Found **{len(results)}** results.\n")

    for i, r in enumerate(results, 1):
        title = r.get("title", "Untitled")
        url = r.get("href") or r.get("url") or r.get("link", "")
        body = r.get("body") or r.get("snippet", "")
        date = r.get("date", "")

        parts.append(f"## {i}. {title}")
        parts.append(f"**URL**: {url}")
        if date:
            parts.append(f"**Date**: {date}")
        if body:
            parts.append(f"\n{body}")
        parts.append("")  # blank line

    return "\n".join(parts)


def format_results_json(results):
    """Format search results as JSON."""
    return json.dumps(results, indent=2, ensure_ascii=False)


def main():
    setup_encoding()
    check_dependencies()

    parser = argparse.ArgumentParser(
        description="Search the web via DuckDuckGo"
    )
    parser.add_argument("query", help="Search query")
    parser.add_argument("--max-results", type=int, default=10,
                        help="Number of results (default: 10)")
    parser.add_argument("--region", type=str, default="wt-wt",
                        help="Region code, e.g. cn-zh, us-en, jp-jp (default: wt-wt)")
    parser.add_argument("--news", action="store_true",
                        help="Search news instead of general web")
    parser.add_argument("--json", action="store_true",
                        help="Output as JSON instead of Markdown")

    args = parser.parse_args()

    query = args.query.strip()
    if not query:
        print("Error: empty query", file=sys.stderr)
        sys.exit(1)

    print(f"Searching: {query} (region={args.region}, max={args.max_results})", file=sys.stderr)

    try:
        if args.news:
            results = search_news(query, args.max_results, args.region)
        else:
            results = search_text(query, args.max_results, args.region)
    except Exception as e:
        print(f"Error: search failed: {e}", file=sys.stderr)
        sys.exit(1)

    if not results:
        print(f"No results found for: {query}")
        sys.exit(0)

    print(f"Got {len(results)} results", file=sys.stderr)

    if args.json:
        print(format_results_json(results))
    else:
        print(format_results_markdown(results, query, is_news=args.news))


if __name__ == "__main__":
    main()