Seo Audit

#!/usr/bin/env python3
"""
SEO Content Quality Scorer

Scores content across SEO dimensions including keyword usage, readability,
structure, E-E-A-T signals, AI content detection markers, and on-page
optimization. Produces a weighted score 0-100 with actionable recommendations.

Usage:
    python content_scorer.py article.md
    python content_scorer.py article.md --keyword "cloud cost optimization" --json
    python content_scorer.py article.md --keyword "SEO audit" --verbose
"""

import argparse
import json
import math
import re
import sys
from pathlib import Path


def count_words(text):
    """Count words in text."""
    return len(text.split())


def count_sentences(text):
    """Count sentences in text."""
    sentences = re.split(r'[.!?]+', text)
    return len([s for s in sentences if s.strip()])


def count_syllables(word):
    """Estimate syllable count for a word."""
    word = word.lower().strip()
    if len(word) <= 3:
        return 1
    count = 0
    vowels = 'aeiouy'
    prev_vowel = False
    for char in word:
        is_vowel = char in vowels
        if is_vowel and not prev_vowel:
            count += 1
        prev_vowel = is_vowel
    if word.endswith('e'):
        count -= 1
    if word.endswith('le') and len(word) > 2 and word[-3] not in vowels:
        count += 1
    return max(count, 1)


def flesch_reading_ease(text):
    """Calculate Flesch Reading Ease score."""
    words = text.split()
    word_count = len(words)
    sentence_count = count_sentences(text)
    if word_count == 0 or sentence_count == 0:
        return 0

    syllable_count = sum(count_syllables(w) for w in words)
    score = 206.835 - 1.015 * (word_count / sentence_count) - 84.6 * (syllable_count / word_count)
    return round(max(0, min(100, score)), 1)


def analyze_keyword_usage(text, keyword):
    """Analyze keyword placement and density."""
    if not keyword:
        return {"score": 0, "checks": {}, "detail": "No keyword provided"}

    text_lower = text.lower()
    keyword_lower = keyword.lower()
    words = text_lower.split()
    word_count = len(words)

    checks = {}

    # Keyword density
    keyword_count = text_lower.count(keyword_lower)
    density = (keyword_count * len(keyword_lower.split()) / max(word_count, 1)) * 100
    checks["density"] = {
        "value": round(density, 2),
        "pass": 0.5 <= density <= 2.5,
        "detail": f"Keyword density: {round(density, 2)}% (target: 0.5-2.5%)",
    }

    # Keyword in first 100 words
    first_100 = " ".join(words[:100])
    in_first_100 = keyword_lower in first_100
    checks["in_first_100_words"] = {
        "pass": in_first_100,
        "detail": "Keyword found in first 100 words" if in_first_100
        else "Keyword NOT in first 100 words",
    }

    # Keyword in headings
    headings = re.findall(r'^#{1,3}\s+(.+)', text, re.MULTILINE)
    headings_with_kw = sum(1 for h in headings if keyword_lower in h.lower())
    checks["in_headings"] = {
        "count": headings_with_kw,
        "total_headings": len(headings),
        "pass": headings_with_kw >= 1,
        "detail": f"Keyword in {headings_with_kw}/{len(headings)} headings",
    }

    # Keyword in H1
    h1s = re.findall(r'^#\s+(.+)', text, re.MULTILINE)
    in_h1 = any(keyword_lower in h.lower() for h in h1s)
    checks["in_h1"] = {
        "pass": in_h1,
        "detail": "Keyword found in H1" if in_h1 else "Keyword NOT in H1",
    }

    passed = sum(1 for c in checks.values() if c["pass"])
    score = (passed / len(checks)) * 100
    return {"score": round(score, 1), "checks": checks}


def analyze_structure(text):
    """Analyze content structure for SEO."""
    checks = {}

    lines = text.splitlines()
    word_count = count_words(text)

    # H1 presence
    h1s = re.findall(r'^#\s+', text, re.MULTILINE)
    checks["h1_present"] = {
        "pass": len(h1s) == 1,
        "count": len(h1s),
        "detail": f"{len(h1s)} H1 tag(s) — should be exactly 1",
    }

    # H2 sections
    h2s = re.findall(r'^##\s+', text, re.MULTILINE)
    checks["h2_sections"] = {
        "pass": len(h2s) >= 3,
        "count": len(h2s),
        "detail": f"{len(h2s)} H2 sections found",
    }

    # Heading hierarchy (no skipped levels)
    heading_levels = [len(m.group(1)) for m in re.finditer(r'^(#{1,6})\s+', text, re.MULTILINE)]
    skipped = False
    for i in range(1, len(heading_levels)):
        if heading_levels[i] > heading_levels[i-1] + 1:
            skipped = True
            break
    checks["heading_hierarchy"] = {
        "pass": not skipped,
        "detail": "Heading hierarchy is clean" if not skipped
        else "Heading levels are skipped (e.g., H2 to H4)",
    }

    # Word count
    checks["word_count"] = {
        "value": word_count,
        "pass": word_count >= 800,
        "detail": f"{word_count} words (minimum 800 for comprehensive content)",
    }

    # Paragraph length
    paragraphs = [p for p in re.split(r'\n\s*\n', text) if p.strip() and not p.strip().startswith('#')]
    long_paragraphs = sum(1 for p in paragraphs if count_words(p) > 150)
    checks["paragraph_length"] = {
        "pass": long_paragraphs == 0,
        "long_count": long_paragraphs,
        "detail": f"{long_paragraphs} paragraphs exceed 150 words",
    }

    # Lists present
    list_items = len(re.findall(r'^\s*[\-\*\d+\.]\s+', text, re.MULTILINE))
    checks["lists_present"] = {
        "pass": list_items >= 3,
        "count": list_items,
        "detail": f"{list_items} list items found",
    }

    # Tables present
    table_rows = len(re.findall(r'^\s*\|.*\|', text, re.MULTILINE))
    checks["tables_present"] = {
        "pass": table_rows >= 3,
        "count": table_rows,
        "detail": f"{table_rows} table rows found",
    }

    passed = sum(1 for c in checks.values() if c["pass"])
    score = (passed / len(checks)) * 100
    return {"score": round(score, 1), "checks": checks}


def analyze_eeat(text):
    """Analyze E-E-A-T signals in content."""
    checks = {}

    # Author attribution
    has_author = bool(re.search(
        r'(author|written by|by\s+[A-Z][a-z]+\s+[A-Z])',
        text, re.IGNORECASE
    ))
    checks["author_attribution"] = {
        "pass": has_author,
        "detail": "Author attribution found" if has_author else "No author attribution",
    }

    # Date signals
    has_dates = bool(re.search(r'(published|updated|modified|date).*\d{4}', text, re.IGNORECASE))
    checks["date_signals"] = {
        "pass": has_dates,
        "detail": "Publication date signals found" if has_dates else "No date signals",
    }

    # Source citations
    citations = len(re.findall(
        r'(according to|source:|cited from|published in|\(\d{4}\)|reported by)',
        text, re.IGNORECASE
    ))
    checks["source_citations"] = {
        "pass": citations >= 2,
        "count": citations,
        "detail": f"{citations} source citations found",
    }

    # External links
    ext_links = len(re.findall(r'\[.*?\]\(https?://[^)]+\)', text))
    checks["external_references"] = {
        "pass": ext_links >= 1,
        "count": ext_links,
        "detail": f"{ext_links} external links found",
    }

    # Experience markers
    experience_markers = len(re.findall(
        r'\b(we found|we tested|we noticed|in our experience|I\'ve seen|'
        r'we observed|our data shows|in practice|real-world)\b',
        text, re.IGNORECASE
    ))
    checks["experience_signals"] = {
        "pass": experience_markers >= 1,
        "count": experience_markers,
        "detail": f"{experience_markers} first-hand experience markers found",
    }

    passed = sum(1 for c in checks.values() if c["pass"])
    score = (passed / len(checks)) * 100
    return {"score": round(score, 1), "checks": checks}


def analyze_ai_patterns(text):
    """Detect AI content patterns that may reduce SEO performance."""
    checks = {}
    word_count = count_words(text)

    # AI filler words
    filler_words = [
        'delve', 'landscape', 'crucial', 'vital', 'pivotal', 'leverage',
        'robust', 'comprehensive', 'holistic', 'foster', 'facilitate',
        'utilize', 'furthermore', 'moreover', 'streamline', 'cutting-edge',
        'game-changer', 'paradigm', 'synergy', 'ecosystem', 'transformative',
        'seamless', 'navigate', 'embark',
    ]
    filler_count = sum(
        len(re.findall(r'\b' + w + r'\b', text, re.IGNORECASE))
        for w in filler_words
    )
    per_1000 = (filler_count / max(word_count, 1)) * 1000
    checks["ai_filler_density"] = {
        "pass": per_1000 < 3,
        "count": filler_count,
        "per_1000": round(per_1000, 1),
        "detail": f"{filler_count} AI filler words ({round(per_1000, 1)} per 1000 words)",
    }

    # Hedging phrases
    hedging = len(re.findall(
        r"(it'?s important to note|it'?s worth mentioning|one might argue|"
        r"it goes without saying|needless to say)",
        text, re.IGNORECASE
    ))
    checks["hedging_phrases"] = {
        "pass": hedging == 0,
        "count": hedging,
        "detail": f"{hedging} hedging phrases found",
    }

    # Sentence length uniformity
    sentences = [s.strip() for s in re.split(r'[.!?]+', text) if s.strip()]
    if len(sentences) >= 5:
        lengths = [len(s.split()) for s in sentences]
        avg = sum(lengths) / len(lengths)
        variance = sum((l - avg) ** 2 for l in lengths) / len(lengths)
        std_dev = math.sqrt(variance)
        checks["sentence_variety"] = {
            "pass": std_dev > 4,
            "std_dev": round(std_dev, 1),
            "detail": f"Sentence length std dev: {round(std_dev, 1)} (>4 is natural variety)",
        }
    else:
        checks["sentence_variety"] = {
            "pass": True,
            "detail": "Not enough sentences to analyze variety",
        }

    passed = sum(1 for c in checks.values() if c["pass"])
    score = (passed / len(checks)) * 100
    return {"score": round(score, 1), "checks": checks}


def calculate_overall(keyword_score, structure_score, eeat_score, ai_score, readability):
    """Calculate weighted overall SEO content score."""
    # Readability factor (target 60-70)
    readability_score = 100 if 50 <= readability <= 80 else max(0, 100 - abs(readability - 65) * 2)

    weighted = (
        keyword_score * 0.25 +
        structure_score * 0.25 +
        eeat_score * 0.20 +
        ai_score * 0.15 +
        readability_score * 0.15
    )
    return round(weighted, 1)


def main():
    parser = argparse.ArgumentParser(
        description="Score content quality for SEO"
    )
    parser.add_argument("file", help="Path to content file (Markdown or HTML)")
    parser.add_argument("--keyword", help="Target keyword to analyze placement")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument("--verbose", action="store_true", help="Show all check details")
    args = parser.parse_args()

    filepath = Path(args.file)
    if not filepath.exists():
        print(f"Error: File not found: {filepath}", file=sys.stderr)
        sys.exit(1)

    text = filepath.read_text(encoding="utf-8", errors="replace")
    word_count = count_words(text)
    readability = flesch_reading_ease(text)

    keyword_analysis = analyze_keyword_usage(text, args.keyword)
    structure_analysis = analyze_structure(text)
    eeat_analysis = analyze_eeat(text)
    ai_analysis = analyze_ai_patterns(text)

    overall = calculate_overall(
        keyword_analysis["score"],
        structure_analysis["score"],
        eeat_analysis["score"],
        ai_analysis["score"],
        readability,
    )

    grade = "A" if overall >= 85 else "B" if overall >= 70 else "C" if overall >= 55 else "D" if overall >= 40 else "F"

    result = {
        "file": str(filepath),
        "word_count": word_count,
        "readability_score": readability,
        "overall_score": overall,
        "grade": grade,
        "keyword_analysis": keyword_analysis,
        "structure_analysis": structure_analysis,
        "eeat_analysis": eeat_analysis,
        "ai_pattern_analysis": ai_analysis,
    }

    if args.json:
        print(json.dumps(result, indent=2))
    else:
        print(f"\n{'='*60}")
        print(f"  SEO CONTENT SCORE: {overall}/100 (Grade: {grade})")
        print(f"{'='*60}")
        print(f"  File: {filepath}")
        print(f"  Words: {word_count}")
        print(f"  Readability: {readability} (Flesch Reading Ease)")
        if args.keyword:
            print(f"  Keyword: {args.keyword}")
        print()
        print(f"  Keyword Usage:  {keyword_analysis['score']}/100")
        print(f"  Structure:      {structure_analysis['score']}/100")
        print(f"  E-E-A-T:        {eeat_analysis['score']}/100")
        print(f"  AI Patterns:    {ai_analysis['score']}/100")

        if args.verbose:
            for label, analysis in [
                ("Keyword", keyword_analysis),
                ("Structure", structure_analysis),
                ("E-E-A-T", eeat_analysis),
                ("AI Patterns", ai_analysis),
            ]:
                print(f"\n  --- {label} Details ---")
                for key, check in analysis.get("checks", {}).items():
                    status = "PASS" if check.get("pass") else "FAIL"
                    print(f"  [{status}] {key}: {check['detail']}")

        print()


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Redirect Chain Checker

Checks URLs for redirect chains, status codes, and common redirect issues.
Identifies chains longer than 2 hops, mixed HTTP/HTTPS redirects, redirect
loops, and temporary (302) redirects that should be permanent (301).

Usage:
    python redirect_checker.py --url https://example.com/old-page
    python redirect_checker.py --file urls.txt --json
    python redirect_checker.py --url https://example.com/page --max-hops 10
"""

import argparse
import json
import re
import sys
from pathlib import Path
from urllib.parse import urlparse, urljoin
try:
    from urllib.request import urlopen, Request
    from urllib.error import URLError, HTTPError
    HAS_URLLIB = True
except ImportError:
    HAS_URLLIB = False


def normalize_url(url):
    """Ensure URL has a scheme."""
    if not url.startswith(('http://', 'https://')):
        url = 'https://' + url
    return url.rstrip('/')


def simulate_redirect_check(url, max_hops=10):
    """
    Check a URL for redirect chains.

    Uses urllib to follow redirects manually, recording each hop.
    Falls back to simulation if network is unavailable.
    """
    url = normalize_url(url)
    chain = []
    visited = set()
    current_url = url
    issues = []

    if HAS_URLLIB:
        for hop in range(max_hops + 1):
            if current_url in visited:
                issues.append({
                    "type": "redirect_loop",
                    "severity": "Critical",
                    "detail": f"Redirect loop detected at {current_url}",
                })
                break
            visited.add(current_url)

            try:
                req = Request(current_url, method='HEAD')
                req.add_header('User-Agent', 'SEO-Audit-Bot/1.0')

                import http.client
                # Use lower-level to avoid auto-redirect
                parsed = urlparse(current_url)
                if parsed.scheme == 'https':
                    import ssl
                    ctx = ssl.create_default_context()
                    conn = http.client.HTTPSConnection(parsed.netloc, timeout=10, context=ctx)
                else:
                    conn = http.client.HTTPConnection(parsed.netloc, timeout=10)

                path = parsed.path or '/'
                if parsed.query:
                    path += '?' + parsed.query

                conn.request('HEAD', path, headers={'User-Agent': 'SEO-Audit-Bot/1.0'})
                resp = conn.getresponse()
                status = resp.status

                chain.append({
                    "hop": hop + 1,
                    "url": current_url,
                    "status": status,
                })

                if status in (301, 302, 303, 307, 308):
                    location = resp.getheader('Location', '')
                    if location:
                        # Handle relative redirects
                        next_url = urljoin(current_url, location)

                        # Check for HTTP to HTTPS or vice versa
                        if urlparse(current_url).scheme != urlparse(next_url).scheme:
                            issues.append({
                                "type": "scheme_change",
                                "severity": "Medium",
                                "detail": f"Scheme change: {urlparse(current_url).scheme} -> {urlparse(next_url).scheme} at hop {hop + 1}",
                            })

                        # Flag 302 that should be 301
                        if status == 302:
                            issues.append({
                                "type": "temporary_redirect",
                                "severity": "High",
                                "detail": f"302 temporary redirect at hop {hop + 1} — should this be 301 permanent?",
                            })

                        current_url = next_url
                    else:
                        issues.append({
                            "type": "missing_location",
                            "severity": "Critical",
                            "detail": f"Redirect status {status} but no Location header at hop {hop + 1}",
                        })
                        break
                else:
                    # Final destination reached
                    break

                conn.close()

            except Exception as e:
                chain.append({
                    "hop": hop + 1,
                    "url": current_url,
                    "status": "error",
                    "error": str(e),
                })
                issues.append({
                    "type": "connection_error",
                    "severity": "Critical",
                    "detail": f"Connection error at hop {hop + 1}: {str(e)}",
                })
                break
    else:
        # Simulation mode when no network available
        chain.append({
            "hop": 1,
            "url": current_url,
            "status": "simulated",
            "note": "Network check unavailable — install urllib for live checks",
        })

    # Analyze chain
    hop_count = len(chain)
    if hop_count > 3:
        issues.append({
            "type": "long_chain",
            "severity": "High",
            "detail": f"Redirect chain has {hop_count} hops — should be 2 or fewer",
        })

    final_url = chain[-1]["url"] if chain else url
    final_status = chain[-1].get("status", "unknown") if chain else "unknown"

    return {
        "original_url": url,
        "final_url": final_url,
        "final_status": final_status,
        "total_hops": max(hop_count - 1, 0),
        "chain": chain,
        "issues": issues,
        "passed": len(issues) == 0,
    }


def check_url_patterns(url):
    """Check URL for common SEO issues without making requests."""
    issues = []
    parsed = urlparse(url)

    # Check for HTTP (should be HTTPS)
    if parsed.scheme == 'http':
        issues.append({
            "type": "no_https",
            "severity": "High",
            "detail": "URL uses HTTP instead of HTTPS",
        })

    # Check for trailing slash inconsistency
    if parsed.path and parsed.path != '/' and not parsed.path.endswith('/'):
        issues.append({
            "type": "trailing_slash",
            "severity": "Low",
            "detail": "No trailing slash — verify consistency with canonical URL",
        })

    # Check for URL parameters
    if parsed.query:
        issues.append({
            "type": "url_parameters",
            "severity": "Medium",
            "detail": f"URL contains parameters: {parsed.query} — verify canonical handling",
        })

    # Check for uppercase in path
    if parsed.path != parsed.path.lower():
        issues.append({
            "type": "uppercase_url",
            "severity": "Medium",
            "detail": "URL path contains uppercase characters — may cause duplicate content",
        })

    # Check path depth
    depth = len([s for s in parsed.path.split('/') if s])
    if depth > 4:
        issues.append({
            "type": "deep_url",
            "severity": "Medium",
            "detail": f"URL depth is {depth} levels — consider flattening for crawl efficiency",
        })

    return issues


def main():
    parser = argparse.ArgumentParser(
        description="Check URLs for redirect chains and SEO issues"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--url", help="Single URL to check")
    group.add_argument("--file", help="File with one URL per line")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument(
        "--max-hops", type=int, default=10,
        help="Maximum redirect hops to follow (default: 10)"
    )
    parser.add_argument(
        "--patterns-only", action="store_true",
        help="Only check URL patterns without making network requests"
    )
    args = parser.parse_args()

    urls = []
    if args.url:
        urls = [args.url]
    else:
        filepath = Path(args.file)
        if not filepath.exists():
            print(f"Error: File not found: {filepath}", file=sys.stderr)
            sys.exit(1)
        urls = [line.strip() for line in filepath.read_text().splitlines() if line.strip()]

    results = []
    for url in urls:
        if args.patterns_only:
            pattern_issues = check_url_patterns(normalize_url(url))
            results.append({
                "url": url,
                "pattern_issues": pattern_issues,
                "passed": len(pattern_issues) == 0,
            })
        else:
            result = simulate_redirect_check(url, args.max_hops)
            result["pattern_issues"] = check_url_patterns(normalize_url(url))
            results.append(result)

    if args.json:
        print(json.dumps({"urls_checked": len(results), "results": results}, indent=2))
    else:
        print(f"\n{'='*60}")
        print(f"  REDIRECT CHECK — {len(results)} URL(s)")
        print(f"{'='*60}")

        for r in results:
            url = r.get("original_url", r.get("url", ""))
            print(f"\n  URL: {url}")

            if "chain" in r:
                print(f"  Final URL: {r['final_url']}")
                print(f"  Total hops: {r['total_hops']}")
                print(f"  Final status: {r['final_status']}")

                if r["chain"]:
                    print(f"  Chain:")
                    for hop in r["chain"]:
                        print(f"    Hop {hop['hop']}: [{hop['status']}] {hop['url']}")

            all_issues = r.get("issues", []) + r.get("pattern_issues", [])
            if all_issues:
                print(f"  Issues ({len(all_issues)}):")
                for issue in all_issues:
                    print(f"    [{issue['severity']}] {issue['type']}: {issue['detail']}")
            else:
                print(f"  Status: PASSED — no issues found")

        passed = sum(1 for r in results if r.get("passed", False))
        print(f"\n  Summary: {passed}/{len(results)} URLs passed")
        print()


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
XML Sitemap Analyzer

Analyzes XML sitemaps for SEO issues including missing URLs, invalid entries,
stale lastmod dates, priority misconfigurations, and size violations.
Supports both sitemap index files and individual sitemaps.

Usage:
    python sitemap_analyzer.py --sitemap https://example.com/sitemap.xml
    python sitemap_analyzer.py --file sitemap.xml --json
    python sitemap_analyzer.py --file sitemap.xml --check-urls
"""

import argparse
import json
import re
import sys
from datetime import datetime, timedelta
from pathlib import Path
from urllib.parse import urlparse
try:
    import xml.etree.ElementTree as ET
    HAS_XML = True
except ImportError:
    HAS_XML = False


SITEMAP_NS = {
    'sm': 'http://www.sitemaps.org/schemas/sitemap/0.9',
    'xhtml': 'http://www.w3.org/1999/xhtml',
    'image': 'http://www.google.com/schemas/sitemap-image/1.1',
}

MAX_URLS_PER_SITEMAP = 50000
MAX_SIZE_BYTES = 50 * 1024 * 1024  # 50MB uncompressed


def parse_sitemap_xml(content):
    """Parse sitemap XML content and extract URLs."""
    if not HAS_XML:
        return [], [], "XML parser not available"

    try:
        root = ET.fromstring(content)
    except ET.ParseError as e:
        return [], [], f"XML parse error: {str(e)}"

    tag = root.tag.split('}')[-1] if '}' in root.tag else root.tag

    urls = []
    sub_sitemaps = []

    if tag == 'sitemapindex':
        # Sitemap index file
        for sitemap in root.findall('.//sm:sitemap', SITEMAP_NS):
            loc = sitemap.find('sm:loc', SITEMAP_NS)
            lastmod = sitemap.find('sm:lastmod', SITEMAP_NS)
            if loc is not None and loc.text:
                sub_sitemaps.append({
                    "url": loc.text.strip(),
                    "lastmod": lastmod.text.strip() if lastmod is not None and lastmod.text else None,
                })
        # Also try without namespace
        if not sub_sitemaps:
            for sitemap in root.findall('.//sitemap'):
                loc = sitemap.find('loc')
                lastmod = sitemap.find('lastmod')
                if loc is not None and loc.text:
                    sub_sitemaps.append({
                        "url": loc.text.strip(),
                        "lastmod": lastmod.text.strip() if lastmod is not None and lastmod.text else None,
                    })

    elif tag == 'urlset':
        # Regular sitemap
        for url_elem in root.findall('.//sm:url', SITEMAP_NS):
            entry = {}
            loc = url_elem.find('sm:loc', SITEMAP_NS)
            lastmod = url_elem.find('sm:lastmod', SITEMAP_NS)
            changefreq = url_elem.find('sm:changefreq', SITEMAP_NS)
            priority = url_elem.find('sm:priority', SITEMAP_NS)

            if loc is not None and loc.text:
                entry["url"] = loc.text.strip()
                entry["lastmod"] = lastmod.text.strip() if lastmod is not None and lastmod.text else None
                entry["changefreq"] = changefreq.text.strip() if changefreq is not None and changefreq.text else None
                entry["priority"] = priority.text.strip() if priority is not None and priority.text else None
                urls.append(entry)

        # Also try without namespace
        if not urls:
            for url_elem in root.findall('.//url'):
                entry = {}
                loc = url_elem.find('loc')
                lastmod = url_elem.find('lastmod')
                changefreq = url_elem.find('changefreq')
                priority = url_elem.find('priority')

                if loc is not None and loc.text:
                    entry["url"] = loc.text.strip()
                    entry["lastmod"] = lastmod.text.strip() if lastmod is not None and lastmod.text else None
                    entry["changefreq"] = changefreq.text.strip() if changefreq is not None and changefreq.text else None
                    entry["priority"] = priority.text.strip() if priority is not None and priority.text else None
                    urls.append(entry)

    return urls, sub_sitemaps, None


def parse_date(date_str):
    """Parse various date formats."""
    if not date_str:
        return None
    formats = [
        "%Y-%m-%dT%H:%M:%S%z",
        "%Y-%m-%dT%H:%M:%S",
        "%Y-%m-%d",
        "%Y-%m-%dT%H:%M:%SZ",
    ]
    # Handle timezone offset without colon
    date_str = re.sub(r'(\+\d{2}):(\d{2})$', r'\1\2', date_str)
    for fmt in formats:
        try:
            return datetime.strptime(date_str, fmt)
        except ValueError:
            continue
    return None


def analyze_urls(urls, content_size=None):
    """Analyze sitemap URLs for issues."""
    issues = []
    stats = {
        "total_urls": len(urls),
        "with_lastmod": 0,
        "with_changefreq": 0,
        "with_priority": 0,
        "stale_urls": 0,
        "duplicate_urls": 0,
        "domains": set(),
    }

    seen_urls = set()
    now = datetime.now()
    stale_threshold = now - timedelta(days=365)

    valid_changefreqs = {'always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'}
    stale_urls = []

    for entry in urls:
        url = entry.get("url", "")
        parsed = urlparse(url)
        stats["domains"].add(parsed.netloc)

        # Check for duplicates
        normalized = url.rstrip('/')
        if normalized in seen_urls:
            stats["duplicate_urls"] += 1
            issues.append({
                "type": "duplicate_url",
                "severity": "Medium",
                "url": url,
                "detail": "Duplicate URL in sitemap",
            })
        seen_urls.add(normalized)

        # Check lastmod
        if entry.get("lastmod"):
            stats["with_lastmod"] += 1
            parsed_date = parse_date(entry["lastmod"])
            if parsed_date:
                if parsed_date.replace(tzinfo=None) < stale_threshold:
                    stats["stale_urls"] += 1
                    stale_urls.append(url)
            else:
                issues.append({
                    "type": "invalid_lastmod",
                    "severity": "Low",
                    "url": url,
                    "detail": f"Invalid date format: {entry['lastmod']}",
                })

        # Check changefreq
        if entry.get("changefreq"):
            stats["with_changefreq"] += 1
            if entry["changefreq"].lower() not in valid_changefreqs:
                issues.append({
                    "type": "invalid_changefreq",
                    "severity": "Low",
                    "url": url,
                    "detail": f"Invalid changefreq: {entry['changefreq']}",
                })

        # Check priority
        if entry.get("priority"):
            stats["with_priority"] += 1
            try:
                pval = float(entry["priority"])
                if pval < 0 or pval > 1:
                    issues.append({
                        "type": "invalid_priority",
                        "severity": "Low",
                        "url": url,
                        "detail": f"Priority out of range: {entry['priority']}",
                    })
            except ValueError:
                issues.append({
                    "type": "invalid_priority",
                    "severity": "Low",
                    "url": url,
                    "detail": f"Non-numeric priority: {entry['priority']}",
                })

        # Check URL format
        if not url.startswith(('http://', 'https://')):
            issues.append({
                "type": "invalid_url",
                "severity": "High",
                "url": url,
                "detail": "URL missing scheme (http/https)",
            })

        # Check for URL parameters (potential duplicates)
        if parsed.query:
            issues.append({
                "type": "parameterized_url",
                "severity": "Medium",
                "url": url,
                "detail": "URL contains query parameters — verify canonical handling",
            })

    # Size checks
    if len(urls) > MAX_URLS_PER_SITEMAP:
        issues.append({
            "type": "too_many_urls",
            "severity": "Critical",
            "detail": f"Sitemap contains {len(urls)} URLs — max is {MAX_URLS_PER_SITEMAP}",
        })

    if content_size and content_size > MAX_SIZE_BYTES:
        issues.append({
            "type": "file_too_large",
            "severity": "Critical",
            "detail": f"Sitemap is {content_size / 1024 / 1024:.1f}MB — max is 50MB",
        })

    # Coverage checks
    if stats["with_lastmod"] == 0 and len(urls) > 0:
        issues.append({
            "type": "no_lastmod",
            "severity": "Medium",
            "detail": "No URLs have lastmod dates — Google uses this for crawl prioritization",
        })
    elif stats["with_lastmod"] < len(urls):
        issues.append({
            "type": "partial_lastmod",
            "severity": "Low",
            "detail": f"Only {stats['with_lastmod']}/{len(urls)} URLs have lastmod dates",
        })

    if stats["stale_urls"] > 0:
        issues.append({
            "type": "stale_content",
            "severity": "Medium",
            "detail": f"{stats['stale_urls']} URLs have lastmod dates older than 1 year",
        })

    # Multiple domains check
    if len(stats["domains"]) > 1:
        issues.append({
            "type": "multiple_domains",
            "severity": "High",
            "detail": f"Sitemap contains URLs from {len(stats['domains'])} different domains",
        })

    stats["domains"] = list(stats["domains"])
    return stats, issues, stale_urls


def main():
    parser = argparse.ArgumentParser(
        description="Analyze XML sitemap for SEO issues"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--sitemap", help="URL of sitemap to fetch and analyze")
    group.add_argument("--file", help="Local XML sitemap file to analyze")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument(
        "--check-urls", action="store_true",
        help="List all URLs found in the sitemap"
    )
    args = parser.parse_args()

    content = None
    content_size = None

    if args.file:
        filepath = Path(args.file)
        if not filepath.exists():
            print(f"Error: File not found: {filepath}", file=sys.stderr)
            sys.exit(1)
        content = filepath.read_text(encoding="utf-8", errors="replace")
        content_size = filepath.stat().st_size
    elif args.sitemap:
        try:
            from urllib.request import urlopen, Request
            req = Request(args.sitemap, headers={'User-Agent': 'SEO-Audit-Bot/1.0'})
            resp = urlopen(req, timeout=30)
            content = resp.read().decode("utf-8", errors="replace")
            content_size = len(content.encode("utf-8"))
        except Exception as e:
            print(f"Error fetching sitemap: {e}", file=sys.stderr)
            sys.exit(1)

    urls, sub_sitemaps, parse_error = parse_sitemap_xml(content)

    if parse_error:
        print(f"Error: {parse_error}", file=sys.stderr)
        sys.exit(1)

    stats, issues, stale_urls = analyze_urls(urls, content_size)

    # Determine sitemap type
    sitemap_type = "sitemap_index" if sub_sitemaps else "urlset"

    result = {
        "source": args.file or args.sitemap,
        "type": sitemap_type,
        "file_size_bytes": content_size,
        "stats": stats,
        "sub_sitemaps": sub_sitemaps if sub_sitemaps else None,
        "issues": issues,
        "issue_count": len(issues),
        "passed": all(i["severity"] not in ("Critical", "High") for i in issues),
    }

    if args.check_urls:
        result["all_urls"] = [u["url"] for u in urls]

    if args.json:
        print(json.dumps(result, indent=2))
    else:
        print(f"\n{'='*60}")
        print(f"  SITEMAP ANALYSIS")
        print(f"{'='*60}")
        print(f"  Source: {args.file or args.sitemap}")
        print(f"  Type: {sitemap_type}")
        if content_size:
            print(f"  File size: {content_size / 1024:.1f} KB")

        if sub_sitemaps:
            print(f"\n  Sitemap Index: {len(sub_sitemaps)} sub-sitemaps")
            for sm in sub_sitemaps[:10]:
                mod = f" (lastmod: {sm['lastmod']})" if sm['lastmod'] else ""
                print(f"    - {sm['url']}{mod}")
            if len(sub_sitemaps) > 10:
                print(f"    ... and {len(sub_sitemaps) - 10} more")

        if urls:
            print(f"\n  URL Statistics:")
            print(f"    Total URLs: {stats['total_urls']}")
            print(f"    With lastmod: {stats['with_lastmod']}")
            print(f"    With changefreq: {stats['with_changefreq']}")
            print(f"    With priority: {stats['with_priority']}")
            print(f"    Duplicates: {stats['duplicate_urls']}")
            print(f"    Stale (>1 year): {stats['stale_urls']}")
            print(f"    Domains: {', '.join(stats['domains'])}")

        if issues:
            print(f"\n  Issues ({len(issues)}):")
            for issue in issues:
                url_info = f" — {issue['url']}" if 'url' in issue else ""
                print(f"    [{issue['severity']}] {issue['type']}: {issue['detail']}{url_info}")
        else:
            print(f"\n  No issues found.")

        status = "PASSED" if result["passed"] else "NEEDS ATTENTION"
        print(f"\n  Overall: {status}")

        if args.check_urls and urls:
            print(f"\n  All URLs:")
            for u in urls:
                mod = f" [{u['lastmod']}]" if u.get('lastmod') else ""
                print(f"    {u['url']}{mod}")

        print()


if __name__ == "__main__":
    main()