Site Architecture

#!/usr/bin/env python3
"""
Internal Link Structure Mapper

Maps internal linking structure from a sitemap, analyzes hub-spoke
relationships, identifies potential orphan pages, and scores link
equity distribution across the site architecture.

Usage:
    python link_mapper.py --sitemap sitemap.xml
    python link_mapper.py --sitemap sitemap.xml --json
    python link_mapper.py --urls urls.txt --hub-analysis
"""

import argparse
import json
import re
import sys
from collections import Counter, defaultdict
from pathlib import Path
from urllib.parse import urlparse

try:
    import xml.etree.ElementTree as ET
except ImportError:
    ET = None


SITEMAP_NS = {'sm': 'http://www.sitemaps.org/schemas/sitemap/0.9'}


def parse_sitemap(filepath):
    """Extract URLs from XML sitemap."""
    if not ET:
        return [], "XML parser not available"
    content = filepath.read_text(encoding="utf-8", errors="replace")
    try:
        root = ET.fromstring(content)
    except ET.ParseError as e:
        return [], f"Parse error: {e}"

    urls = []
    for loc in root.findall('.//sm:url/sm:loc', SITEMAP_NS):
        if loc.text:
            urls.append(loc.text.strip())
    if not urls:
        for loc in root.findall('.//url/loc'):
            if loc.text:
                urls.append(loc.text.strip())
    return urls, None


def load_urls(filepath):
    """Load URLs from a text file."""
    return [l.strip() for l in filepath.read_text().splitlines()
            if l.strip() and not l.startswith('#')]


def build_hierarchy(urls):
    """Build a hierarchical structure from URLs."""
    hierarchy = defaultdict(list)
    url_info = {}

    for url in urls:
        parsed = urlparse(url)
        segments = [s for s in parsed.path.split('/') if s]
        depth = len(segments)

        # Determine parent
        if depth == 0:
            parent = None
        elif depth == 1:
            parent = "/"
        else:
            parent_path = "/" + "/".join(segments[:-1]) + "/"
            parent = parent_path

        # Determine section (first segment)
        section = segments[0] if segments else "root"

        url_info[url] = {
            "url": url,
            "path": parsed.path,
            "depth": depth,
            "section": section,
            "parent_path": parent,
            "segments": segments,
        }

        hierarchy[section].append(url)

    return hierarchy, url_info


def identify_hubs(hierarchy, url_info):
    """Identify potential hub pages in each section."""
    hubs = []

    for section, urls in hierarchy.items():
        if section == "root":
            continue

        # The shallowest page in a section is likely the hub
        section_urls = sorted(urls, key=lambda u: url_info[u]["depth"])
        if section_urls:
            hub_url = section_urls[0]
            spoke_count = len(section_urls) - 1

            hubs.append({
                "section": section,
                "hub_url": hub_url,
                "hub_depth": url_info[hub_url]["depth"],
                "spoke_count": spoke_count,
                "spoke_urls": section_urls[1:],
            })

    return hubs


def analyze_link_equity(url_info, hubs):
    """Estimate link equity distribution."""
    total = len(url_info)
    depth_distribution = Counter()
    section_distribution = Counter()

    for info in url_info.values():
        depth_distribution[info["depth"]] += 1
        section_distribution[info["section"]] += 1

    # Estimate equity concentration
    # Pages at depth 1-2 get more equity than deep pages
    equity_estimates = {}
    for url, info in url_info.items():
        depth = info["depth"]
        if depth == 0:
            equity = 100  # Homepage
        elif depth == 1:
            equity = 50
        elif depth == 2:
            equity = 25
        elif depth == 3:
            equity = 12
        else:
            equity = 5
        equity_estimates[url] = equity

    # Section balance
    sections = list(section_distribution.keys())
    if sections:
        avg_pages = total / len(sections)
        imbalanced = {s: c for s, c in section_distribution.items()
                      if c > avg_pages * 3 or (c < avg_pages * 0.2 and c < 3)}
    else:
        imbalanced = {}

    return {
        "depth_distribution": dict(depth_distribution),
        "section_distribution": dict(section_distribution.most_common(20)),
        "equity_estimates": equity_estimates,
        "imbalanced_sections": imbalanced,
    }


def detect_potential_orphans(url_info, hubs):
    """Identify pages that might be orphans based on structure."""
    potential_orphans = []

    hub_sections = {h["section"] for h in hubs}

    for url, info in url_info.items():
        # Deep pages not in any hub section
        if info["depth"] >= 3 and info["section"] not in hub_sections:
            potential_orphans.append({
                "url": url,
                "reason": "Deep page in section with no hub",
                "depth": info["depth"],
            })

        # Pages at unusual depth for their section
        section_urls = [u for u, i in url_info.items() if i["section"] == info["section"]]
        section_depths = [url_info[u]["depth"] for u in section_urls]
        if section_depths:
            avg_depth = sum(section_depths) / len(section_depths)
            if info["depth"] > avg_depth + 2:
                potential_orphans.append({
                    "url": url,
                    "reason": "Unusually deep for its section",
                    "depth": info["depth"],
                })

    return potential_orphans


def main():
    parser = argparse.ArgumentParser(
        description="Map internal link structure from sitemap"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--sitemap", help="XML sitemap file")
    group.add_argument("--urls", help="Text file with URLs")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument("--hub-analysis", action="store_true", help="Show hub-spoke analysis")
    args = parser.parse_args()

    urls = []
    if args.sitemap:
        fp = Path(args.sitemap)
        if not fp.exists():
            print(f"Error: {fp} not found", file=sys.stderr)
            sys.exit(1)
        urls, err = parse_sitemap(fp)
        if err:
            print(f"Error: {err}", file=sys.stderr)
            sys.exit(1)
    else:
        fp = Path(args.urls)
        if not fp.exists():
            print(f"Error: {fp} not found", file=sys.stderr)
            sys.exit(1)
        urls = load_urls(fp)

    if not urls:
        print("No URLs found.", file=sys.stderr)
        sys.exit(1)

    hierarchy, url_info = build_hierarchy(urls)
    hubs = identify_hubs(hierarchy, url_info)
    equity = analyze_link_equity(url_info, hubs)
    orphans = detect_potential_orphans(url_info, hubs)

    result = {
        "total_urls": len(urls),
        "sections": len(hierarchy),
        "hubs": hubs,
        "equity_analysis": {
            "depth_distribution": equity["depth_distribution"],
            "section_distribution": equity["section_distribution"],
            "imbalanced_sections": equity["imbalanced_sections"],
        },
        "potential_orphans": orphans,
    }

    if args.json:
        print(json.dumps(result, indent=2, default=str))
    else:
        print(f"\n{'='*60}")
        print(f"  LINK STRUCTURE MAP — {len(urls)} URLs, {len(hierarchy)} sections")
        print(f"{'='*60}")

        print(f"\n  Depth Distribution:")
        for depth in sorted(equity["depth_distribution"].keys()):
            count = equity["depth_distribution"][depth]
            pct = round(count / len(urls) * 100, 1)
            print(f"    Depth {depth}: {count} pages ({pct}%)")

        print(f"\n  Top Sections:")
        for section, count in list(equity["section_distribution"].items())[:10]:
            print(f"    /{section}/: {count} pages")

        if args.hub_analysis and hubs:
            print(f"\n  Hub-Spoke Analysis:")
            for hub in sorted(hubs, key=lambda h: h["spoke_count"], reverse=True):
                print(f"\n    Hub: /{hub['section']}/ ({hub['spoke_count']} spokes)")
                print(f"    Hub URL: {hub['hub_url']}")
                for spoke in hub["spoke_urls"][:5]:
                    print(f"      -> {spoke}")
                if hub["spoke_count"] > 5:
                    print(f"      ... and {hub['spoke_count'] - 5} more")

        if equity["imbalanced_sections"]:
            print(f"\n  Imbalanced Sections:")
            for section, count in equity["imbalanced_sections"].items():
                print(f"    /{section}/: {count} pages (check balance)")

        if orphans:
            print(f"\n  Potential Orphan Pages ({len(orphans)}):")
            for o in orphans[:10]:
                print(f"    {o['url']} — {o['reason']} (depth: {o['depth']})")
            if len(orphans) > 10:
                print(f"    ... and {len(orphans) - 10} more")

        print()


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
URL Redirect Chain Checker for Site Architecture

Checks URLs for redirect chains, HTTP/HTTPS consistency, trailing slash
consistency, and common URL pattern issues. Designed for architecture
migration audits and restructuring projects.

Usage:
    python redirect_checker.py --file urls.txt
    python redirect_checker.py --file urls.txt --json
    python redirect_checker.py --url https://example.com/old-page
"""

import argparse
import json
import re
import sys
from pathlib import Path
from urllib.parse import urlparse


def normalize_url(url):
    """Ensure URL has a scheme."""
    if not url.startswith(('http://', 'https://')):
        url = 'https://' + url
    return url


def check_url_patterns(url):
    """Analyze URL for architectural issues."""
    issues = []
    parsed = urlparse(normalize_url(url))

    # HTTP check
    if parsed.scheme == 'http':
        issues.append({
            "type": "no_https",
            "severity": "High",
            "detail": "URL uses HTTP — should be HTTPS",
        })

    # Path depth
    segments = [s for s in parsed.path.split('/') if s]
    depth = len(segments)
    if depth > 3:
        issues.append({
            "type": "deep_url",
            "severity": "Medium",
            "detail": f"URL depth is {depth} — ideally 3 or fewer levels",
        })

    # Uppercase
    if parsed.path != parsed.path.lower():
        issues.append({
            "type": "uppercase",
            "severity": "Medium",
            "detail": "URL contains uppercase characters — may cause duplicates",
        })

    # Underscores
    if '_' in parsed.path:
        issues.append({
            "type": "underscores",
            "severity": "Low",
            "detail": "URL uses underscores — hyphens preferred by Google",
        })

    # Parameters
    if parsed.query:
        issues.append({
            "type": "parameters",
            "severity": "Medium",
            "detail": f"URL contains parameters: {parsed.query}",
        })

    # File extensions (non-standard)
    if re.search(r'\.(php|asp|aspx|jsp|cgi)$', parsed.path, re.IGNORECASE):
        issues.append({
            "type": "legacy_extension",
            "severity": "Low",
            "detail": "URL has legacy file extension — clean URLs preferred",
        })

    # Double slashes in path
    if '//' in parsed.path:
        issues.append({
            "type": "double_slash",
            "severity": "Medium",
            "detail": "URL contains double slash in path",
        })

    # IDs or numbers in URL
    if re.search(r'/\d{4,}', parsed.path):
        issues.append({
            "type": "numeric_id",
            "severity": "Low",
            "detail": "URL contains numeric IDs — descriptive slugs preferred",
        })

    return {
        "url": url,
        "normalized": normalize_url(url),
        "depth": depth,
        "segments": segments,
        "has_trailing_slash": parsed.path.endswith('/') and parsed.path != '/',
        "issues": issues,
        "passed": len([i for i in issues if i["severity"] in ("High", "Critical")]) == 0,
    }


def check_redirect(url, max_hops=10):
    """Check URL for redirect chains via HTTP."""
    url = normalize_url(url)
    chain = []
    current = url
    visited = set()
    issues = []

    try:
        import http.client
        import ssl

        for hop in range(max_hops):
            if current in visited:
                issues.append({
                    "type": "redirect_loop",
                    "severity": "Critical",
                    "detail": f"Loop detected at {current}",
                })
                break
            visited.add(current)

            parsed = urlparse(current)
            try:
                if parsed.scheme == 'https':
                    ctx = ssl.create_default_context()
                    conn = http.client.HTTPSConnection(parsed.netloc, timeout=10, context=ctx)
                else:
                    conn = http.client.HTTPConnection(parsed.netloc, timeout=10)

                path = parsed.path or '/'
                if parsed.query:
                    path += '?' + parsed.query

                conn.request('HEAD', path, headers={'User-Agent': 'Architecture-Audit/1.0'})
                resp = conn.getresponse()
                status = resp.status

                chain.append({"hop": hop + 1, "url": current, "status": status})

                if status in (301, 302, 303, 307, 308):
                    location = resp.getheader('Location', '')
                    if location:
                        from urllib.parse import urljoin
                        next_url = urljoin(current, location)

                        if status == 302:
                            issues.append({
                                "type": "temporary_redirect",
                                "severity": "High",
                                "detail": f"302 at hop {hop + 1} — should be 301 for permanent moves",
                            })

                        current = next_url
                    else:
                        break
                else:
                    break

                conn.close()
            except Exception as e:
                chain.append({"hop": hop + 1, "url": current, "status": "error", "error": str(e)})
                break

    except ImportError:
        chain.append({"hop": 1, "url": current, "status": "skipped", "note": "HTTP check unavailable"})

    hops = max(len(chain) - 1, 0)
    if hops > 2:
        issues.append({
            "type": "long_chain",
            "severity": "High",
            "detail": f"Redirect chain has {hops} hops — max 2 recommended",
        })

    return {
        "original_url": url,
        "final_url": chain[-1]["url"] if chain else url,
        "total_hops": hops,
        "chain": chain,
        "issues": issues,
    }


def main():
    parser = argparse.ArgumentParser(
        description="Check URLs for redirect chains and architecture issues"
    )
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--url", help="Single URL to check")
    group.add_argument("--file", help="File with URLs (one per line)")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument("--patterns-only", action="store_true", help="Only check URL patterns, no HTTP requests")
    args = parser.parse_args()

    urls = []
    if args.url:
        urls = [args.url]
    else:
        fp = Path(args.file)
        if not fp.exists():
            print(f"Error: {fp} not found", file=sys.stderr)
            sys.exit(1)
        urls = [l.strip() for l in fp.read_text().splitlines() if l.strip() and not l.startswith('#')]

    results = []
    trailing_slash_counts = {"with": 0, "without": 0}

    for url in urls:
        pattern_result = check_url_patterns(url)
        if pattern_result["has_trailing_slash"]:
            trailing_slash_counts["with"] += 1
        elif pattern_result["depth"] > 0:
            trailing_slash_counts["without"] += 1

        if not args.patterns_only:
            redirect_result = check_redirect(url)
            pattern_result["redirect"] = redirect_result
            pattern_result["issues"].extend(redirect_result["issues"])

        results.append(pattern_result)

    # Check trailing slash consistency across all URLs
    consistency_issue = None
    if trailing_slash_counts["with"] > 0 and trailing_slash_counts["without"] > 0:
        consistency_issue = {
            "type": "inconsistent_trailing_slash",
            "severity": "Medium",
            "detail": f"{trailing_slash_counts['with']} URLs with trailing slash, {trailing_slash_counts['without']} without",
        }

    passed = sum(1 for r in results if r["passed"])

    output = {
        "total_urls": len(results),
        "passed": passed,
        "trailing_slash_consistency": consistency_issue,
        "results": results,
    }

    if args.json:
        print(json.dumps(output, indent=2))
    else:
        print(f"\n{'='*60}")
        print(f"  URL ARCHITECTURE CHECK — {len(results)} URLs")
        print(f"{'='*60}")
        print(f"  Passed: {passed}/{len(results)}")

        if consistency_issue:
            print(f"  [{consistency_issue['severity']}] {consistency_issue['detail']}")

        for r in results:
            all_issues = r["issues"]
            status = "OK" if r["passed"] else "ISSUES"
            print(f"\n  [{status}] {r['url']} (depth: {r['depth']})")
            if all_issues:
                for issue in all_issues:
                    print(f"    [{issue['severity']}] {issue['detail']}")
            if "redirect" in r and r["redirect"]["total_hops"] > 0:
                print(f"    Redirects: {r['redirect']['total_hops']} hops -> {r['redirect']['final_url']}")

        print()


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Site Architecture Sitemap Analyzer

Analyzes XML sitemaps for architectural issues including URL depth
distribution, directory structure patterns, orphan detection signals,
and crawl equity concentration.

Usage:
    python sitemap_analyzer.py --file sitemap.xml
    python sitemap_analyzer.py --file sitemap.xml --json
    python sitemap_analyzer.py --file sitemap.xml --depth-report
"""

import argparse
import json
import re
import sys
from collections import Counter, defaultdict
from pathlib import Path
from urllib.parse import urlparse

try:
    import xml.etree.ElementTree as ET
except ImportError:
    print("Error: xml.etree.ElementTree not available", file=sys.stderr)
    sys.exit(1)


SITEMAP_NS = {'sm': 'http://www.sitemaps.org/schemas/sitemap/0.9'}


def parse_sitemap(content):
    """Parse sitemap XML and extract URLs."""
    try:
        root = ET.fromstring(content)
    except ET.ParseError as e:
        return [], f"XML parse error: {e}"

    urls = []
    # Try with namespace
    for url_elem in root.findall('.//sm:url/sm:loc', SITEMAP_NS):
        if url_elem.text:
            urls.append(url_elem.text.strip())
    # Try without namespace
    if not urls:
        for url_elem in root.findall('.//url/loc'):
            if url_elem.text:
                urls.append(url_elem.text.strip())
    # Check for sitemap index
    if not urls:
        for loc in root.findall('.//sm:sitemap/sm:loc', SITEMAP_NS):
            if loc.text:
                urls.append(loc.text.strip())
        if not urls:
            for loc in root.findall('.//sitemap/loc'):
                if loc.text:
                    urls.append(loc.text.strip())

    return urls, None


def analyze_depth(urls):
    """Analyze URL depth distribution."""
    depth_counts = Counter()
    depth_urls = defaultdict(list)

    for url in urls:
        parsed = urlparse(url)
        segments = [s for s in parsed.path.split('/') if s]
        depth = len(segments)
        depth_counts[depth] += 1
        if len(depth_urls[depth]) < 5:  # Keep max 5 examples per depth
            depth_urls[depth].append(url)

    return dict(depth_counts), {k: v for k, v in depth_urls.items()}


def analyze_directories(urls):
    """Analyze directory/section structure."""
    directories = Counter()
    for url in urls:
        parsed = urlparse(url)
        segments = [s for s in parsed.path.split('/') if s]
        if segments:
            directories[segments[0]] += 1

    return dict(directories.most_common(20))


def analyze_patterns(urls):
    """Detect URL patterns and anomalies."""
    issues = []
    stats = {
        "total_urls": len(urls),
        "unique_domains": len(set(urlparse(u).netloc for u in urls)),
        "with_params": 0,
        "with_uppercase": 0,
        "with_underscores": 0,
        "inconsistent_trailing_slash": False,
    }

    trailing_slash = {"with": 0, "without": 0}

    for url in urls:
        parsed = urlparse(url)

        if parsed.query:
            stats["with_params"] += 1

        if parsed.path != parsed.path.lower():
            stats["with_uppercase"] += 1

        if '_' in parsed.path:
            stats["with_underscores"] += 1

        if parsed.path.endswith('/') and parsed.path != '/':
            trailing_slash["with"] += 1
        elif parsed.path and parsed.path != '/':
            trailing_slash["without"] += 1

    # Check trailing slash consistency
    if trailing_slash["with"] > 0 and trailing_slash["without"] > 0:
        stats["inconsistent_trailing_slash"] = True
        issues.append({
            "type": "inconsistent_trailing_slash",
            "severity": "Medium",
            "detail": f"{trailing_slash['with']} URLs with trailing slash, {trailing_slash['without']} without — pick one convention",
        })

    if stats["with_params"] > 0:
        issues.append({
            "type": "parameterized_urls",
            "severity": "Medium",
            "detail": f"{stats['with_params']} URLs contain query parameters — verify canonical handling",
        })

    if stats["with_uppercase"] > 0:
        issues.append({
            "type": "uppercase_urls",
            "severity": "Medium",
            "detail": f"{stats['with_uppercase']} URLs contain uppercase characters",
        })

    if stats["with_underscores"] > 0:
        issues.append({
            "type": "underscore_urls",
            "severity": "Low",
            "detail": f"{stats['with_underscores']} URLs use underscores instead of hyphens",
        })

    if stats["unique_domains"] > 1:
        issues.append({
            "type": "multiple_domains",
            "severity": "High",
            "detail": f"Sitemap contains URLs from {stats['unique_domains']} domains",
        })

    return stats, issues


def analyze_architecture_health(depth_counts, total_urls):
    """Score architectural health based on depth distribution."""
    issues = []

    # Calculate percentage at each depth
    depth_pcts = {}
    for depth, count in depth_counts.items():
        depth_pcts[depth] = round(count / max(total_urls, 1) * 100, 1)

    # Deep pages check
    deep_pages = sum(count for depth, count in depth_counts.items() if depth >= 4)
    deep_pct = round(deep_pages / max(total_urls, 1) * 100, 1)

    if deep_pct > 5:
        issues.append({
            "type": "excessive_depth",
            "severity": "High",
            "detail": f"{deep_pct}% of pages are 4+ levels deep — target <5%",
        })

    # Shallow concentration check
    shallow = sum(count for depth, count in depth_counts.items() if depth <= 2)
    shallow_pct = round(shallow / max(total_urls, 1) * 100, 1)

    if shallow_pct < 50:
        issues.append({
            "type": "deep_heavy",
            "severity": "Medium",
            "detail": f"Only {shallow_pct}% of pages at depth 1-2 — architecture may be too deep",
        })

    # Score
    score = 100
    for issue in issues:
        if issue["severity"] == "High":
            score -= 20
        elif issue["severity"] == "Medium":
            score -= 10
    score = max(0, score)

    return {
        "depth_distribution": depth_pcts,
        "deep_page_percentage": deep_pct,
        "shallow_page_percentage": shallow_pct,
        "score": score,
        "issues": issues,
    }


def main():
    parser = argparse.ArgumentParser(
        description="Analyze sitemap for site architecture issues"
    )
    parser.add_argument("--file", required=True, help="XML sitemap file path")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    parser.add_argument("--depth-report", action="store_true", help="Show detailed depth analysis")
    args = parser.parse_args()

    fp = Path(args.file)
    if not fp.exists():
        print(f"Error: {fp} not found", file=sys.stderr)
        sys.exit(1)

    content = fp.read_text(encoding="utf-8", errors="replace")
    urls, error = parse_sitemap(content)

    if error:
        print(f"Error: {error}", file=sys.stderr)
        sys.exit(1)

    if not urls:
        print("No URLs found in sitemap.", file=sys.stderr)
        sys.exit(1)

    depth_counts, depth_examples = analyze_depth(urls)
    directories = analyze_directories(urls)
    url_stats, url_issues = analyze_patterns(urls)
    health = analyze_architecture_health(depth_counts, len(urls))

    all_issues = url_issues + health["issues"]

    result = {
        "source": str(fp),
        "total_urls": len(urls),
        "depth_distribution": depth_counts,
        "directories": directories,
        "url_stats": url_stats,
        "architecture_health": health,
        "issues": all_issues,
    }

    if args.json:
        print(json.dumps(result, indent=2))
    else:
        print(f"\n{'='*60}")
        print(f"  SITE ARCHITECTURE ANALYSIS — {len(urls)} URLs")
        print(f"{'='*60}")
        print(f"  Architecture Score: {health['score']}/100")

        print(f"\n  Depth Distribution:")
        for depth in sorted(depth_counts.keys()):
            count = depth_counts[depth]
            pct = health["depth_distribution"].get(depth, 0)
            bar = "#" * int(pct / 2)
            print(f"    Depth {depth}: {count:>6} pages ({pct:>5.1f}%) {bar}")

        if args.depth_report:
            print(f"\n  Examples by Depth:")
            for depth in sorted(depth_examples.keys()):
                print(f"    Depth {depth}:")
                for url in depth_examples[depth][:3]:
                    print(f"      {url}")

        print(f"\n  Top Directories:")
        for dir_name, count in list(directories.items())[:10]:
            print(f"    /{dir_name}/: {count} pages")

        if all_issues:
            print(f"\n  Issues ({len(all_issues)}):")
            for issue in all_issues:
                print(f"    [{issue['severity']}] {issue['detail']}")
        else:
            print(f"\n  No issues found.")

        print()


if __name__ == "__main__":
    main()