Browser Automation

#!/usr/bin/env python3
"""
Anti-Detection Checker - Audit browser automation scripts for bot detection signatures.

Analyzes automation code for patterns that trigger bot detection systems including
fingerprinting tells, timing issues, and missing evasion techniques.

Author: Claude Skills Engineering Team
License: MIT
"""

import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import List, Dict, Optional


@dataclass
class DetectionSignature:
    """A detected bot detection signature."""
    category: str
    severity: str  # critical, high, medium, low
    title: str
    description: str
    line_number: int
    code_snippet: str
    fix_suggestion: str


@dataclass
class CheckResult:
    """Overall check result."""
    file_path: str
    total_signatures: int = 0
    risk_score: int = 0  # 0-100
    signatures: List[DetectionSignature] = field(default_factory=list)
    recommendations: List[str] = field(default_factory=list)
    missing_protections: List[str] = field(default_factory=list)


SEVERITY_WEIGHTS = {"critical": 25, "high": 15, "medium": 8, "low": 3}

# (pattern, category, severity, title, description, fix)
DETECTION_PATTERNS = [
    # Webdriver detection
    (r'navigator\.webdriver\s*=\s*false',
     "fingerprint", "high",
     "Naive webdriver override",
     "Setting navigator.webdriver=false is easily detected by advanced bot systems that check the property descriptor.",
     "Use CDP commands to remove webdriver property before page load, or use undetected-chromedriver."),

    (r'(?:headless|headless_mode)\s*[:=]\s*(?:True|true|1)',
     "fingerprint", "critical",
     "Headless mode enabled without evasion",
     "Running in headless mode without anti-fingerprinting is the most common detection trigger.",
     "Use --headless=new (Chrome 112+) or apply comprehensive anti-fingerprinting patches."),

    # Fixed timing
    (r'(?:sleep|wait|delay)\s*\(\s*(\d+(?:\.\d+)?)\s*\)',
     "timing", "medium",
     "Fixed delay detected",
     "Constant delays create predictable timing patterns that bot detectors recognize.",
     "Use randomized delays: random.uniform(min_delay, max_delay)."),

    (r'time\.sleep\s*\(\s*(?:0\.\d|0\.0)',
     "timing", "high",
     "Very short fixed delay",
     "Sub-second fixed delays are a strong bot indicator.",
     "Use randomized delays of at least 1-3 seconds between actions."),

    # User-Agent issues
    (r'["\'](?:User-Agent|user-agent)["\']\s*:\s*["\'][^"\']+["\']',
     "headers", "medium",
     "Hardcoded User-Agent string",
     "A single hardcoded User-Agent is easily fingerprinted and blocked.",
     "Rotate User-Agent strings from a realistic pool matching the browser being automated."),

    (r'(?:HeadlessChrome|PhantomJS|Selenium|puppeteer)',
     "fingerprint", "critical",
     "Bot identifier in User-Agent or code",
     "Bot-identifying strings in the User-Agent or runtime environment are trivially detected.",
     "Remove all bot-identifying strings. Use realistic User-Agent rotation."),

    # Missing protections
    (r'\.get\s*\(\s*["\']https?://',
     "request", "low",
     "HTTP request without explicit headers",
     "Requests without custom headers may use default library headers that identify automation.",
     "Set realistic Accept, Accept-Language, Accept-Encoding, and Connection headers."),

    # Cookie handling
    (r'(?:cookies|cookie_jar)\s*[:=]\s*(?:\{\}|\[\]|None)',
     "session", "medium",
     "Empty cookie initialization",
     "Starting with empty cookies on a site you've 'visited' before is suspicious.",
     "Persist and reuse cookies across sessions. Pre-load common cookies."),

    # Viewport/resolution
    (r'(?:window_size|viewport|set_window_size)\s*\(\s*(\d+)\s*,\s*(\d+)\s*\)',
     "fingerprint", "low",
     "Fixed viewport size",
     "Using a fixed viewport without variation can be a fingerprinting signal.",
     "Randomize viewport dimensions slightly around common resolutions."),

    # Proxy patterns
    (r'(?:proxy|PROXY)\s*[:=]\s*["\'](?:http|socks)',
     "network", "low",
     "Single proxy configuration",
     "Using a single proxy IP is easily blocked. Detected as automation if IP is in known ranges.",
     "Use rotating residential proxies or a proxy pool with automatic rotation."),

    # Selenium-specific
    (r'(?:execute_cdp_cmd|execute_script)\s*\(["\'].*(?:Object\.defineProperty|delete\s+)',
     "fingerprint", "medium",
     "JavaScript property manipulation for evasion",
     "Manual JS property overrides can be detected by checking property descriptors and prototype chains.",
     "Use comprehensive stealth plugins (e.g., selenium-stealth, puppeteer-extra-plugin-stealth)."),

    # No error handling
    (r'\.(?:click|send_keys|submit)\s*\([^)]*\)\s*$',
     "reliability", "low",
     "Action without explicit wait or error handling",
     "Actions without waits may fail on slow pages, causing detectable error patterns.",
     "Use explicit waits (WebDriverWait) before interactions. Add try/except for retries."),

    # Referrer issues
    (r'(?:Referer|referrer)\s*[:=]\s*(?:None|""|\'\')',
     "headers", "medium",
     "Missing or empty Referer header",
     "Navigation without a Referer header is suspicious for internal page transitions.",
     "Set appropriate Referer headers matching natural navigation flow."),
]

# Protections that SHOULD be present
EXPECTED_PROTECTIONS = [
    (r'(?:random|randint|uniform|randrange)', "Randomized timing/delays"),
    (r'(?:User-Agent|user.agent).*(?:random|choice|rotate|pool|list)', "User-Agent rotation"),
    (r'(?:cookie|session).*(?:save|persist|load|store)', "Cookie persistence"),
    (r'(?:retry|backoff|exponential)', "Retry/backoff logic"),
    (r'(?:robots\.txt|robotparser)', "robots.txt compliance"),
    (r'(?:rate.limit|throttle|semaphore)', "Rate limiting"),
]


def check_file(file_path: Path) -> CheckResult:
    """Analyze a file for detection signatures."""
    result = CheckResult(file_path=str(file_path))

    try:
        content = file_path.read_text(encoding="utf-8", errors="ignore")
    except (OSError, PermissionError) as e:
        result.recommendations.append(f"Could not read file: {e}")
        return result

    lines = content.split("\n")

    # Check for detection signatures
    for pattern_str, category, severity, title, desc, fix in DETECTION_PATTERNS:
        try:
            pattern = re.compile(pattern_str, re.IGNORECASE)
        except re.error:
            continue

        for i, line in enumerate(lines, 1):
            if pattern.search(line):
                result.signatures.append(DetectionSignature(
                    category=category,
                    severity=severity,
                    title=title,
                    description=desc,
                    line_number=i,
                    code_snippet=line.strip()[:120],
                    fix_suggestion=fix,
                ))

    result.total_signatures = len(result.signatures)

    # Calculate risk score
    risk = 0
    for sig in result.signatures:
        risk += SEVERITY_WEIGHTS.get(sig.severity, 0)
    result.risk_score = min(100, risk)

    # Check for missing protections
    for pattern_str, protection_name in EXPECTED_PROTECTIONS:
        if not re.search(pattern_str, content, re.IGNORECASE):
            result.missing_protections.append(protection_name)

    # Generate recommendations
    if result.risk_score >= 75:
        result.recommendations.append("HIGH RISK: This script is very likely to be detected. Major refactoring needed.")
    elif result.risk_score >= 40:
        result.recommendations.append("MODERATE RISK: Several detection signatures found. Address high/critical items first.")
    elif result.risk_score > 0:
        result.recommendations.append("LOW RISK: Minor signatures found. Script is reasonably stealthy.")
    else:
        result.recommendations.append("MINIMAL RISK: No obvious detection signatures found.")

    if result.missing_protections:
        result.recommendations.append(f"Missing protections: {', '.join(result.missing_protections)}")

    return result


def format_human(result: CheckResult) -> str:
    """Format results for human reading."""
    lines = []
    lines.append("=" * 65)
    lines.append("ANTI-DETECTION CHECK REPORT")
    lines.append("=" * 65)
    lines.append(f"File: {result.file_path}")
    lines.append(f"Detection Signatures Found: {result.total_signatures}")
    lines.append(f"Risk Score: {result.risk_score}/100")
    lines.append("")

    if result.recommendations:
        lines.append("Assessment:")
        for rec in result.recommendations:
            lines.append(f"  > {rec}")
        lines.append("")

    if result.missing_protections:
        lines.append("Missing Protections:")
        for mp in result.missing_protections:
            lines.append(f"  [ ] {mp}")
        lines.append("")

    for i, sig in enumerate(result.signatures, 1):
        lines.append("-" * 50)
        lines.append(f"[{i}] [{sig.severity.upper()}] {sig.title}")
        lines.append(f"    Category: {sig.category}")
        lines.append(f"    Line: {sig.line_number}")
        lines.append(f"    Code: {sig.code_snippet}")
        lines.append(f"    Issue: {sig.description}")
        lines.append(f"    Fix: {sig.fix_suggestion}")
        lines.append("")

    lines.append("=" * 65)
    return "\n".join(lines)


def format_json(result: CheckResult) -> str:
    """Format results as JSON."""
    data = {
        "file_path": result.file_path,
        "total_signatures": result.total_signatures,
        "risk_score": result.risk_score,
        "signatures": [asdict(s) for s in result.signatures],
        "missing_protections": result.missing_protections,
        "recommendations": result.recommendations,
    }
    return json.dumps(data, indent=2)


def main():
    parser = argparse.ArgumentParser(
        description="Anti-Detection Checker - Audit browser automation scripts for bot detection signatures"
    )
    parser.add_argument("--file", required=True, help="Path to automation script to check")
    parser.add_argument("--format", choices=["human", "json"], default="human",
                        help="Output format (default: human)")

    args = parser.parse_args()
    path = Path(args.file)

    if not path.exists():
        print(f"Error: File not found: {args.file}", file=sys.stderr)
        sys.exit(1)

    result = check_file(path)

    if args.format == "json":
        print(format_json(result))
    else:
        print(format_human(result))

    sys.exit(1 if result.risk_score >= 50 else 0)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Form Automation Builder - Generate form automation scripts from HTML analysis.

Parses HTML forms, identifies field types and validation requirements, and generates
ready-to-use automation scripts with proper field handling and submission logic.

Author: Claude Skills Engineering Team
License: MIT
"""

import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import List, Dict, Optional, Tuple
from html.parser import HTMLParser
import textwrap


@dataclass
class FormField:
    """Represents a form field."""
    name: str
    field_type: str  # text, email, password, select, checkbox, radio, textarea, file, hidden, date, number
    label: Optional[str] = None
    required: bool = False
    placeholder: Optional[str] = None
    pattern: Optional[str] = None
    min_length: Optional[int] = None
    max_length: Optional[int] = None
    options: List[str] = field(default_factory=list)  # for select/radio
    default_value: Optional[str] = None


@dataclass
class FormInfo:
    """Represents an HTML form."""
    action: str = ""
    method: str = "GET"
    enctype: str = "application/x-www-form-urlencoded"
    form_id: Optional[str] = None
    fields: List[FormField] = field(default_factory=list)
    has_csrf: bool = False
    csrf_field_name: Optional[str] = None
    has_captcha: bool = False


class FormHTMLParser(HTMLParser):
    """Parse HTML to extract form structure."""

    def __init__(self):
        super().__init__()
        self.forms: List[FormInfo] = []
        self.current_form: Optional[FormInfo] = None
        self.current_select_name: Optional[str] = None
        self.current_select_options: List[str] = []
        self.current_label_for: Optional[str] = None
        self.current_label_text: str = ""
        self.in_label = False
        self.in_select = False
        self.in_option = False
        self.current_option_value: Optional[str] = None
        self.labels: Dict[str, str] = {}
        self.in_textarea = False
        self.current_textarea_name: Optional[str] = None

    def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]):
        attr_dict = {k: v for k, v in attrs if v is not None}

        if tag == "form":
            self.current_form = FormInfo(
                action=attr_dict.get("action", ""),
                method=attr_dict.get("method", "GET").upper(),
                enctype=attr_dict.get("enctype", "application/x-www-form-urlencoded"),
                form_id=attr_dict.get("id"),
            )

        elif tag == "input" and self.current_form is not None:
            input_type = attr_dict.get("type", "text").lower()
            name = attr_dict.get("name", "")
            if not name:
                return

            # Check for CSRF tokens
            if any(tok in name.lower() for tok in ["csrf", "_token", "authenticity_token", "__requestverificationtoken"]):
                self.current_form.has_csrf = True
                self.current_form.csrf_field_name = name
                self.current_form.fields.append(FormField(
                    name=name, field_type="hidden", default_value=attr_dict.get("value", "")
                ))
                return

            # Check for CAPTCHA
            if any(tok in name.lower() for tok in ["captcha", "recaptcha", "hcaptcha"]):
                self.current_form.has_captcha = True
                return

            ff = FormField(
                name=name,
                field_type=input_type,
                required="required" in {k for k, _ in attrs},
                placeholder=attr_dict.get("placeholder"),
                pattern=attr_dict.get("pattern"),
                default_value=attr_dict.get("value"),
            )
            if attr_dict.get("minlength"):
                try:
                    ff.min_length = int(attr_dict["minlength"])
                except ValueError:
                    pass
            if attr_dict.get("maxlength"):
                try:
                    ff.max_length = int(attr_dict["maxlength"])
                except ValueError:
                    pass
            self.current_form.fields.append(ff)

        elif tag == "select" and self.current_form is not None:
            self.in_select = True
            self.current_select_name = attr_dict.get("name", "")
            self.current_select_options = []

        elif tag == "option" and self.in_select:
            self.in_option = True
            self.current_option_value = attr_dict.get("value", "")

        elif tag == "textarea" and self.current_form is not None:
            self.in_textarea = True
            self.current_textarea_name = attr_dict.get("name", "")

        elif tag == "label":
            self.in_label = True
            self.current_label_for = attr_dict.get("for")
            self.current_label_text = ""

    def handle_data(self, data: str):
        if self.in_label:
            self.current_label_text += data.strip()
        if self.in_option:
            text = data.strip()
            if text and self.current_option_value is not None:
                self.current_select_options.append(self.current_option_value or text)

    def handle_endtag(self, tag: str):
        if tag == "form" and self.current_form is not None:
            # Apply collected labels
            for f in self.current_form.fields:
                if f.name in self.labels:
                    f.label = self.labels[f.name]
            self.forms.append(self.current_form)
            self.current_form = None
            self.labels = {}

        elif tag == "select" and self.in_select and self.current_form is not None:
            self.current_form.fields.append(FormField(
                name=self.current_select_name or "",
                field_type="select",
                options=self.current_select_options,
            ))
            self.in_select = False

        elif tag == "option":
            self.in_option = False
            self.current_option_value = None

        elif tag == "textarea" and self.in_textarea and self.current_form is not None:
            self.current_form.fields.append(FormField(
                name=self.current_textarea_name or "",
                field_type="textarea",
            ))
            self.in_textarea = False

        elif tag == "label" and self.in_label:
            self.in_label = False
            if self.current_label_for:
                self.labels[self.current_label_for] = self.current_label_text


def parse_html(html_content: str) -> List[FormInfo]:
    """Parse HTML content and extract form information."""
    parser = FormHTMLParser()
    parser.feed(html_content)
    return parser.forms


def generate_requests_script(form: FormInfo, base_url: str) -> str:
    """Generate a Python requests-based automation script."""
    lines = []
    lines.append('#!/usr/bin/env python3')
    lines.append('"""Auto-generated form automation script using requests."""')
    lines.append('')
    lines.append('import requests')
    lines.append('import time')
    lines.append('import random')
    lines.append('')
    lines.append('')
    lines.append('def submit_form(session=None):')
    lines.append('    """Submit the form with the configured data."""')
    lines.append('    if session is None:')
    lines.append('        session = requests.Session()')
    lines.append('')
    lines.append('    # Configure headers')
    lines.append('    session.headers.update({')
    lines.append('        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",')
    lines.append('        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",')
    lines.append('        "Accept-Language": "en-US,en;q=0.9",')
    lines.append('    })')
    lines.append('')

    if form.has_csrf:
        lines.append(f'    # Fetch CSRF token from form page')
        action_url = form.action if form.action.startswith("http") else f"{base_url}{form.action}"
        lines.append(f'    page = session.get("{action_url}")')
        lines.append(f'    # TODO: Extract CSRF token "{form.csrf_field_name}" from page.text')
        lines.append(f'    csrf_token = ""  # Extract from HTML')
        lines.append('')

    lines.append('    # Form data')
    lines.append('    data = {')

    for f in form.fields:
        if f.field_type == "hidden":
            if form.has_csrf and f.name == form.csrf_field_name:
                lines.append(f'        "{f.name}": csrf_token,')
            else:
                lines.append(f'        "{f.name}": "{f.default_value or ""}",')
        elif f.field_type == "select":
            opts = f.options[:3] if f.options else ["option1"]
            lines.append(f'        "{f.name}": "{opts[0]}",  # Options: {opts}')
        elif f.field_type == "checkbox":
            lines.append(f'        "{f.name}": "on",  # checkbox')
        elif f.field_type == "file":
            continue  # handled separately
        else:
            label = f.label or f.name
            req = " (REQUIRED)" if f.required else ""
            lines.append(f'        "{f.name}": "",  # {f.field_type}: {label}{req}')

    lines.append('    }')
    lines.append('')

    # Handle file uploads
    file_fields = [f for f in form.fields if f.field_type == "file"]
    if file_fields:
        lines.append('    # File uploads')
        lines.append('    files = {')
        for f in file_fields:
            lines.append(f'        "{f.name}": ("filename.ext", open("path/to/file", "rb"), "application/octet-stream"),')
        lines.append('    }')
        lines.append('')

    action = form.action if form.action.startswith("http") else f"{base_url}{form.action}"
    method = form.method.lower()

    lines.append(f'    # Submit form ({form.method} {form.action})')
    lines.append(f'    time.sleep(random.uniform(1.0, 3.0))  # Human-like delay')

    if file_fields:
        lines.append(f'    response = session.{method}("{action}", data=data, files=files)')
    else:
        lines.append(f'    response = session.{method}("{action}", data=data)')

    lines.append('')
    lines.append('    print(f"Status: {response.status_code}")')
    lines.append('    return response')
    lines.append('')
    lines.append('')
    lines.append('if __name__ == "__main__":')
    lines.append('    submit_form()')

    return "\n".join(lines)


def generate_analysis(forms: List[FormInfo]) -> Dict:
    """Generate analysis summary of discovered forms."""
    analysis = {
        "total_forms": len(forms),
        "forms": [],
    }
    for i, form in enumerate(forms):
        form_info = {
            "index": i,
            "action": form.action,
            "method": form.method,
            "enctype": form.enctype,
            "total_fields": len(form.fields),
            "required_fields": sum(1 for f in form.fields if f.required),
            "has_csrf": form.has_csrf,
            "has_captcha": form.has_captcha,
            "has_file_upload": any(f.field_type == "file" for f in form.fields),
            "field_types": {},
            "fields": [asdict(f) for f in form.fields],
        }
        for f in form.fields:
            form_info["field_types"][f.field_type] = form_info["field_types"].get(f.field_type, 0) + 1
        analysis["forms"].append(form_info)
    return analysis


def format_human(analysis: Dict, script: Optional[str]) -> str:
    """Format for human output."""
    lines = []
    lines.append("=" * 60)
    lines.append("FORM AUTOMATION ANALYSIS")
    lines.append("=" * 60)
    lines.append(f"Total forms found: {analysis['total_forms']}")
    lines.append("")

    for form in analysis["forms"]:
        lines.append(f"Form #{form['index']}: {form['method']} {form['action']}")
        lines.append(f"  Fields: {form['total_fields']} ({form['required_fields']} required)")
        lines.append(f"  CSRF: {'Yes' if form['has_csrf'] else 'No'}")
        lines.append(f"  CAPTCHA: {'Yes (manual intervention needed)' if form['has_captcha'] else 'No'}")
        lines.append(f"  File Upload: {'Yes' if form['has_file_upload'] else 'No'}")
        lines.append(f"  Field Types: {form['field_types']}")
        lines.append("")
        for f in form["fields"]:
            req = "*" if f.get("required") else " "
            lines.append(f"  {req} [{f['field_type']:10s}] {f['name']}")
        lines.append("")

    if script:
        lines.append("-" * 60)
        lines.append("GENERATED SCRIPT:")
        lines.append("-" * 60)
        lines.append(script)

    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(
        description="Form Automation Builder - Generate form automation scripts from HTML"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--html-file", help="Path to HTML file containing forms")
    group.add_argument("--url", help="Base URL (used with --html-file for resolving relative actions)")
    parser.add_argument("--output", help="Write generated script to file")
    parser.add_argument("--form-index", type=int, default=0,
                        help="Index of form to generate script for (default: 0)")
    parser.add_argument("--format", choices=["human", "json"], default="human",
                        help="Output format (default: human)")
    parser.add_argument("--base-url", default="https://example.com",
                        help="Base URL for resolving relative form actions")

    args = parser.parse_args()

    if args.html_file:
        path = Path(args.html_file)
        if not path.exists():
            print(f"Error: File not found: {args.html_file}", file=sys.stderr)
            sys.exit(1)
        html_content = path.read_text(encoding="utf-8", errors="ignore")
        base_url = args.base_url
    else:
        print("Error: --url requires fetching which is not supported. Use --html-file instead.", file=sys.stderr)
        print("Save the HTML page locally first, then use --html-file.", file=sys.stderr)
        sys.exit(1)

    forms = parse_html(html_content)
    if not forms:
        print("No forms found in the HTML content.", file=sys.stderr)
        sys.exit(1)

    analysis = generate_analysis(forms)

    script = None
    if args.form_index < len(forms):
        script = generate_requests_script(forms[args.form_index], base_url)
        if args.output:
            Path(args.output).write_text(script)
            print(f"Script written to: {args.output}")

    if args.format == "json":
        output = json.dumps(analysis, indent=2)
    else:
        output = format_human(analysis, script)

    print(output)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Scraping Toolkit - Generate web scraping code with rate limiting and best practices.

Analyzes target URLs and generates production-ready scraping scripts with proper
rate limiting, error handling, data extraction, and anti-detection measures.

Author: Claude Skills Engineering Team
License: MIT
"""

import argparse
import json
import os
import re
import sys
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import List, Dict, Optional
from urllib.parse import urlparse, urljoin
import textwrap


RATE_STRATEGIES = {
    "aggressive": {"min_delay": 0.5, "max_delay": 1.5, "concurrent": 5, "description": "Fast, for own APIs or test environments"},
    "normal": {"min_delay": 1.0, "max_delay": 3.0, "concurrent": 3, "description": "Balanced speed and politeness"},
    "polite": {"min_delay": 3.0, "max_delay": 7.0, "concurrent": 1, "description": "Respectful, minimizes server impact"},
    "stealth": {"min_delay": 5.0, "max_delay": 15.0, "concurrent": 1, "description": "Maximum stealth, mimics human browsing"},
}

USER_AGENTS = [
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0",
]


@dataclass
class ScrapingConfig:
    """Configuration for generated scraper."""
    target_url: str
    strategy: str
    output_format: str  # csv, json, jsonl
    selectors: Dict[str, str] = field(default_factory=dict)
    pagination: bool = False
    respect_robots: bool = True
    max_pages: int = 100
    proxy_support: bool = False


def generate_scraper_script(config: ScrapingConfig) -> str:
    """Generate a complete scraping script."""
    strategy = RATE_STRATEGIES[config.strategy]
    parsed = urlparse(config.target_url)
    domain = parsed.netloc

    script = textwrap.dedent(f'''\
        #!/usr/bin/env python3
        """
        Web Scraper for {domain}
        Generated by Scraping Toolkit

        Strategy: {config.strategy} ({strategy["description"]})
        Rate: {strategy["min_delay"]}-{strategy["max_delay"]}s between requests
        """

        import csv
        import json
        import logging
        import os
        import random
        import sys
        import time
        from dataclasses import dataclass, asdict
        from datetime import datetime
        from pathlib import Path
        from typing import List, Dict, Optional, Generator
        from urllib.parse import urljoin, urlparse
        from urllib.robotparser import RobotFileParser

        # Configure logging
        logging.basicConfig(
            level=logging.INFO,
            format="%(asctime)s [%(levelname)s] %(message)s",
        )
        logger = logging.getLogger(__name__)

        # --- Configuration ---
        TARGET_URL = "{config.target_url}"
        DOMAIN = "{domain}"
        MIN_DELAY = {strategy["min_delay"]}
        MAX_DELAY = {strategy["max_delay"]}
        MAX_PAGES = {config.max_pages}
        MAX_RETRIES = 3
        BACKOFF_FACTOR = 2
        RESPECT_ROBOTS = {config.respect_robots}
        OUTPUT_FORMAT = "{config.output_format}"

        USER_AGENTS = {json.dumps(USER_AGENTS[:3], indent=4)}

        # --- Rate Limiter ---
        class RateLimiter:
            """Enforces delays between requests with jitter."""

            def __init__(self, min_delay: float, max_delay: float):
                self.min_delay = min_delay
                self.max_delay = max_delay
                self.last_request = 0.0

            def wait(self):
                elapsed = time.time() - self.last_request
                delay = random.uniform(self.min_delay, self.max_delay)
                remaining = delay - elapsed
                if remaining > 0:
                    time.sleep(remaining)
                self.last_request = time.time()


        # --- Robots.txt Checker ---
        class RobotsChecker:
            """Check robots.txt compliance."""

            def __init__(self, base_url: str):
                self.parser = RobotFileParser()
                self.base_url = base_url
                self.loaded = False

            def load(self):
                try:
                    robots_url = urljoin(self.base_url, "/robots.txt")
                    self.parser.set_url(robots_url)
                    self.parser.read()
                    self.loaded = True
                    logger.info("Loaded robots.txt from %s", robots_url)
                except Exception as e:
                    logger.warning("Could not load robots.txt: %s", e)
                    self.loaded = True  # Allow if can't load

            def can_fetch(self, url: str) -> bool:
                if not self.loaded:
                    self.load()
                return self.parser.can_fetch("*", url)


        # --- Session Manager ---
        class ScraperSession:
            """
            Manages HTTP sessions with retry logic.
            NOTE: This is a template. Install requests library: pip install requests
            """

            def __init__(self):
                try:
                    import requests
                    self.session = requests.Session()
                    self.session.headers.update({{
                        "User-Agent": random.choice(USER_AGENTS),
                        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                        "Accept-Language": "en-US,en;q=0.9",
                        "Accept-Encoding": "gzip, deflate, br",
                        "Connection": "keep-alive",
                    }})
                    self.has_requests = True
                except ImportError:
                    logger.error("requests library not installed. Run: pip install requests")
                    self.has_requests = False

            def get(self, url: str, retries: int = MAX_RETRIES) -> Optional[str]:
                if not self.has_requests:
                    return None
                import requests

                for attempt in range(retries):
                    try:
                        # Rotate User-Agent periodically
                        if random.random() < 0.2:
                            self.session.headers["User-Agent"] = random.choice(USER_AGENTS)

                        response = self.session.get(url, timeout=30)

                        if response.status_code == 200:
                            return response.text
                        elif response.status_code == 429:
                            wait = BACKOFF_FACTOR ** (attempt + 2)
                            logger.warning("Rate limited (429). Waiting %ds...", wait)
                            time.sleep(wait)
                        elif response.status_code >= 500:
                            wait = BACKOFF_FACTOR ** attempt
                            logger.warning("Server error %d. Retry in %ds...", response.status_code, wait)
                            time.sleep(wait)
                        else:
                            logger.error("HTTP %d for %s", response.status_code, url)
                            return None
                    except requests.RequestException as e:
                        logger.error("Request failed: %s", e)
                        if attempt < retries - 1:
                            time.sleep(BACKOFF_FACTOR ** attempt)

                return None


        # --- Data Writer ---
        class DataWriter:
            """Write scraped data to file."""

            def __init__(self, output_path: str, fmt: str):
                self.output_path = output_path
                self.fmt = fmt
                self.count = 0

                if fmt == "csv":
                    self._file = open(output_path, "w", newline="", encoding="utf-8")
                    self._writer = None  # initialized on first write
                elif fmt == "jsonl":
                    self._file = open(output_path, "w", encoding="utf-8")
                elif fmt == "json":
                    self._items = []

            def write(self, item: dict):
                self.count += 1
                if self.fmt == "csv":
                    if self._writer is None:
                        self._writer = csv.DictWriter(self._file, fieldnames=item.keys())
                        self._writer.writeheader()
                    self._writer.writerow(item)
                elif self.fmt == "jsonl":
                    self._file.write(json.dumps(item, ensure_ascii=False) + "\\n")
                elif self.fmt == "json":
                    self._items.append(item)

            def close(self):
                if self.fmt == "json":
                    with open(self.output_path, "w", encoding="utf-8") as f:
                        json.dumps(self._items, f, indent=2, ensure_ascii=False)
                elif hasattr(self, "_file"):
                    self._file.close()
                logger.info("Wrote %d items to %s", self.count, self.output_path)


        # --- Main Scraper ---
        def scrape():
            """Main scraping function - customize the extraction logic below."""
            rate_limiter = RateLimiter(MIN_DELAY, MAX_DELAY)
            robots = RobotsChecker(TARGET_URL) if RESPECT_ROBOTS else None
            session = ScraperSession()

            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
            output_file = f"scraped_{{DOMAIN}}_{{timestamp}}.{{OUTPUT_FORMAT}}"
            writer = DataWriter(output_file, OUTPUT_FORMAT)

            logger.info("Starting scrape of %s", TARGET_URL)
            logger.info("Strategy: {config.strategy}, Delay: %.1f-%.1fs", MIN_DELAY, MAX_DELAY)

            url = TARGET_URL
            pages_scraped = 0

            while url and pages_scraped < MAX_PAGES:
                # Check robots.txt
                if robots and not robots.can_fetch(url):
                    logger.warning("Blocked by robots.txt: %s", url)
                    break

                # Rate limit
                rate_limiter.wait()

                # Fetch page
                html = session.get(url)
                if html is None:
                    logger.error("Failed to fetch: %s", url)
                    break

                pages_scraped += 1
                logger.info("Page %d: %s (%d chars)", pages_scraped, url, len(html))

                # =====================================================
                # TODO: CUSTOMIZE EXTRACTION LOGIC BELOW
                # Use BeautifulSoup, lxml, or regex to extract data
                # Example with BeautifulSoup:
                #   from bs4 import BeautifulSoup
                #   soup = BeautifulSoup(html, "html.parser")
                #   for item in soup.select(".item-class"):
                #       writer.write({{"title": item.text, "url": url}})
                # =====================================================

                item = {{
                    "url": url,
                    "page": pages_scraped,
                    "content_length": len(html),
                    "scraped_at": datetime.now().isoformat(),
                }}
                writer.write(item)

                # =====================================================
                # TODO: CUSTOMIZE PAGINATION LOGIC
                # Example: find next page link
                #   next_link = soup.select_one("a.next-page")
                #   url = urljoin(url, next_link["href"]) if next_link else None
                # =====================================================
                url = None  # Stop after first page by default

            writer.close()
            logger.info("Scraping complete. %d pages processed.", pages_scraped)


        if __name__ == "__main__":
            scrape()
    ''')
    return script


def generate_config_summary(config: ScrapingConfig) -> Dict:
    """Generate a summary of the scraping configuration."""
    strategy = RATE_STRATEGIES[config.strategy]
    parsed = urlparse(config.target_url)
    return {
        "target": {
            "url": config.target_url,
            "domain": parsed.netloc,
            "scheme": parsed.scheme,
        },
        "rate_limiting": {
            "strategy": config.strategy,
            "min_delay_seconds": strategy["min_delay"],
            "max_delay_seconds": strategy["max_delay"],
            "max_concurrent": strategy["concurrent"],
            "description": strategy["description"],
        },
        "output": {
            "format": config.output_format,
        },
        "safety": {
            "respect_robots_txt": config.respect_robots,
            "max_pages": config.max_pages,
            "retry_with_backoff": True,
            "user_agent_rotation": True,
        },
    }


def format_human_output(config: ScrapingConfig, script: str) -> str:
    """Format for human-readable output."""
    strategy = RATE_STRATEGIES[config.strategy]
    lines = []
    lines.append("=" * 60)
    lines.append("SCRAPING TOOLKIT - CODE GENERATOR")
    lines.append("=" * 60)
    lines.append(f"Target: {config.target_url}")
    lines.append(f"Strategy: {config.strategy} ({strategy['description']})")
    lines.append(f"Rate: {strategy['min_delay']}-{strategy['max_delay']}s between requests")
    lines.append(f"Output Format: {config.output_format}")
    lines.append(f"Robots.txt: {'Respected' if config.respect_robots else 'Ignored'}")
    lines.append(f"Max Pages: {config.max_pages}")
    lines.append("")
    lines.append("Features included:")
    lines.append("  [x] Rate limiting with randomized delays")
    lines.append("  [x] User-Agent rotation")
    lines.append("  [x] Exponential backoff on errors")
    lines.append("  [x] robots.txt compliance")
    lines.append("  [x] Session management with cookies")
    lines.append("  [x] Structured data output")
    lines.append("")
    lines.append("-" * 60)
    lines.append("GENERATED SCRIPT:")
    lines.append("-" * 60)
    lines.append(script)
    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(
        description="Scraping Toolkit - Generate web scraping code with rate limiting"
    )
    parser.add_argument("--url", required=True, help="Target URL to scrape")
    parser.add_argument("--strategy", choices=list(RATE_STRATEGIES.keys()), default="polite",
                        help="Rate limiting strategy (default: polite)")
    parser.add_argument("--output", help="Write generated script to file")
    parser.add_argument("--output-format", choices=["csv", "json", "jsonl"], default="json",
                        help="Data output format in generated script (default: json)")
    parser.add_argument("--no-robots", action="store_true",
                        help="Don't include robots.txt checking")
    parser.add_argument("--max-pages", type=int, default=100,
                        help="Maximum pages to scrape (default: 100)")
    parser.add_argument("--format", choices=["human", "json"], default="human",
                        help="Output format for this tool (default: human)")

    args = parser.parse_args()

    config = ScrapingConfig(
        target_url=args.url,
        strategy=args.strategy,
        output_format=args.output_format,
        respect_robots=not args.no_robots,
        max_pages=args.max_pages,
    )

    script = generate_scraper_script(config)

    if args.output:
        Path(args.output).write_text(script)
        print(f"Script written to: {args.output}")

    if args.format == "json":
        summary = generate_config_summary(config)
        summary["generated_script_lines"] = len(script.split("\n"))
        print(json.dumps(summary, indent=2))
    else:
        print(format_human_output(config, script))


if __name__ == "__main__":
    main()