Github Research

#!/usr/bin/env python3
"""Map repo file tree, identify key files, and compute LOC stats.

Usage:
    python analyze_repo_structure.py --repo-dir /tmp/repos/myrepo --output analysis.json [--repo-id owner/name]

Writes analysis JSON to the specified output file.
"""

import argparse
import json
import os
import re
import sys

# Directories to skip during traversal
SKIP_DIRS = {
    ".git", "node_modules", "__pycache__", ".venv", "venv",
    ".tox", ".eggs", "dist", "build",
}

# File extension to language mapping
EXT_TO_LANG = {
    ".py": "Python",
    ".js": "JavaScript",
    ".ts": "TypeScript",
    ".rs": "Rust",
    ".go": "Go",
    ".java": "Java",
    ".cpp": "C++",
    ".cc": "C++",
    ".cxx": "C++",
    ".c": "C",
    ".rb": "Ruby",
    ".sh": "Shell",
    ".yaml": "YAML",
    ".yml": "YAML",
    ".json": "JSON",
    ".md": "Markdown",
}

# Known config file names
CONFIG_FILES = {
    "pyproject.toml", "setup.py", "setup.cfg", "package.json",
    "Cargo.toml", "go.mod", "Makefile", "Dockerfile",
    "docker-compose.yml", "docker-compose.yaml",
    ".env.example", "tox.ini",
}

# Known entry-point file names
ENTRY_POINT_NAMES = {"main.py", "app.py", "run.py", "train.py", "setup.py", "Makefile", "Dockerfile"}


def is_egg_info_dir(name: str) -> bool:
    return name.endswith(".egg-info")


def count_loc(filepath: str) -> int:
    """Count non-empty lines in a text file."""
    try:
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            return sum(1 for line in f if line.strip())
    except (OSError, UnicodeDecodeError):
        return 0


def has_shebang(filepath: str) -> bool:
    """Check if file starts with a shebang line."""
    try:
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            first_line = f.readline()
            return first_line.startswith("#!")
    except (OSError, UnicodeDecodeError):
        return False


def has_main_guard(filepath: str) -> bool:
    """Check if a Python file has `if __name__` guard."""
    try:
        with open(filepath, "r", encoding="utf-8", errors="replace") as f:
            for line in f:
                if re.search(r'if\s+__name__\s*==\s*["\']__main__["\']', line):
                    return True
    except (OSError, UnicodeDecodeError):
        pass
    return False


def should_skip_dir(name: str) -> bool:
    return name in SKIP_DIRS or is_egg_info_dir(name)


def build_tree(repo_dir: str, max_lines: int = 200) -> str:
    """Generate an indented file tree string, truncated to max_lines."""
    lines = []
    base = os.path.basename(os.path.abspath(repo_dir))

    for dirpath, dirnames, filenames in os.walk(repo_dir):
        # Filter out skipped dirs in-place
        dirnames[:] = sorted([d for d in dirnames if not should_skip_dir(d)])
        filenames = sorted(filenames)

        rel = os.path.relpath(dirpath, repo_dir)
        depth = 0 if rel == "." else rel.count(os.sep) + 1
        indent = "  " * depth

        if rel == ".":
            lines.append(f"{base}/")
        else:
            lines.append(f"{indent}{os.path.basename(dirpath)}/")

        if len(lines) >= max_lines:
            break

        file_indent = "  " * (depth + 1)
        for fname in filenames:
            lines.append(f"{file_indent}{fname}")
            if len(lines) >= max_lines:
                break
        if len(lines) >= max_lines:
            break

    if len(lines) >= max_lines:
        lines.append("  ... (truncated)")
    return "\n".join(lines)


def analyze(repo_dir: str, repo_id: str) -> dict:
    """Perform full repo structure analysis."""
    total_files = 0
    total_loc = 0
    languages: dict[str, int] = {}
    key_files: list[str] = []
    entry_points: list[str] = []
    config_files: list[str] = []
    test_files: list[str] = []
    doc_files: list[str] = []
    file_loc: list[tuple[str, int]] = []

    print(f"[analyze] Scanning {repo_dir} ...", file=sys.stderr)

    for dirpath, dirnames, filenames in os.walk(repo_dir):
        dirnames[:] = [d for d in dirnames if not should_skip_dir(d)]

        rel_dir = os.path.relpath(dirpath, repo_dir)

        for fname in filenames:
            filepath = os.path.join(dirpath, fname)
            rel_path = os.path.join(rel_dir, fname) if rel_dir != "." else fname
            # Normalize path separators
            rel_path = rel_path.replace(os.sep, "/")

            total_files += 1

            _, ext = os.path.splitext(fname)
            ext_lower = ext.lower()

            # LOC counting for known languages
            lang = EXT_TO_LANG.get(ext_lower)
            loc = 0
            if lang:
                loc = count_loc(filepath)
                languages[lang] = languages.get(lang, 0) + loc
                total_loc += loc
                file_loc.append((rel_path, loc))

            # Entry points detection
            if fname in ENTRY_POINT_NAMES:
                entry_points.append(rel_path)
            elif ext_lower == ".py" and has_main_guard(filepath):
                entry_points.append(rel_path)
            elif has_shebang(filepath):
                entry_points.append(rel_path)

            # Config files
            if fname in CONFIG_FILES:
                config_files.append(rel_path)

            # Test files
            if (fname.startswith("test_") or fname.endswith(("_test.py", "_test.js", "_test.ts", "_test.go"))
                    or "/tests/" in f"/{rel_path}/" or "/test/" in f"/{rel_path}/"
                    or "/spec/" in f"/{rel_path}/"):
                test_files.append(rel_path)

            # Doc files
            if ext_lower == ".md" or "/docs/" in f"/{rel_path}/" or "/doc/" in f"/{rel_path}/":
                doc_files.append(rel_path)

    # Sort files by LOC descending, take top 10
    file_loc.sort(key=lambda x: x[1], reverse=True)
    file_size_top10 = file_loc[:10]

    # Key files = top LOC files + entry points
    key_set = set()
    for path, _ in file_size_top10:
        key_set.add(path)
    for path in entry_points:
        key_set.add(path)
    key_files = sorted(key_set)

    # Sort languages by LOC descending
    languages = dict(sorted(languages.items(), key=lambda x: x[1], reverse=True))

    tree = build_tree(repo_dir)

    print(f"[analyze] Files: {total_files}, LOC: {total_loc}, Languages: {len(languages)}", file=sys.stderr)

    return {
        "repo_id": repo_id,
        "total_files": total_files,
        "total_loc": total_loc,
        "languages": languages,
        "key_files": key_files,
        "entry_points": sorted(set(entry_points)),
        "config_files": sorted(set(config_files)),
        "test_files": sorted(set(test_files)),
        "doc_files": sorted(set(doc_files)),
        "tree": tree,
        "file_size_top10": file_size_top10,
    }


def main():
    parser = argparse.ArgumentParser(description="Analyze repository structure, files, and LOC stats.")
    parser.add_argument("--repo-dir", required=True, help="Path to the cloned repository")
    parser.add_argument("--output", required=True, help="Output JSON file path")
    parser.add_argument("--repo-id", default="unknown/unknown", help="Repository identifier (owner/name)")
    args = parser.parse_args()

    if not os.path.isdir(args.repo_dir):
        print(f"Error: {args.repo_dir} is not a directory", file=sys.stderr)
        sys.exit(1)

    result = analyze(args.repo_dir, args.repo_id)

    with open(args.output, "w", encoding="utf-8") as f:
        json.dump(result, f, indent=2, ensure_ascii=False)

    print(f"[analyze] Output written to {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Shallow-clone GitHub repos for local code analysis.

Usage:
    python clone_repo.py --repo owner/name --output-dir /tmp/repos [--depth 1] [--branch main]

Outputs JSON to stdout with clone result; stats to stderr.
"""

import argparse
import json
import os
import subprocess
import sys


def get_dir_size(path: str) -> int:
    """Walk directory and sum all file sizes in bytes."""
    total = 0
    for dirpath, _dirnames, filenames in os.walk(path):
        for f in filenames:
            fp = os.path.join(dirpath, f)
            try:
                total += os.path.getsize(fp)
            except OSError:
                pass
    return total


def clone_repo(repo: str, output_dir: str, depth: int, branch: str | None) -> dict:
    """Clone a GitHub repo and return result metadata."""
    parts = repo.split("/")
    if len(parts) != 2:
        return {
            "repo_id": repo,
            "local_path": None,
            "clone_success": False,
            "error": f"Invalid repo format '{repo}', expected 'owner/name'",
        }

    owner, name = parts
    url = f"https://github.com/{owner}/{name}.git"
    target = os.path.join(output_dir, name)

    os.makedirs(output_dir, exist_ok=True)

    # Build clone command
    cmd = ["git", "clone", f"--depth={depth}", "--single-branch"]
    if branch:
        cmd += ["--branch", branch]
    cmd += [url, target]

    print(f"[clone] Running: {' '.join(cmd)}", file=sys.stderr)

    result = subprocess.run(cmd, capture_output=True, text=True)

    # If branch-specific clone fails, retry without --branch
    if result.returncode != 0 and branch:
        print(f"[clone] Branch '{branch}' failed, retrying with default branch...", file=sys.stderr)
        cmd_retry = ["git", "clone", f"--depth={depth}", "--single-branch", url, target]
        result = subprocess.run(cmd_retry, capture_output=True, text=True)

    # Validate clone
    if result.returncode != 0:
        return {
            "repo_id": repo,
            "local_path": target,
            "clone_success": False,
            "error": result.stderr.strip(),
        }

    if not os.path.isdir(target) or not os.listdir(target):
        return {
            "repo_id": repo,
            "local_path": target,
            "clone_success": False,
            "error": "Clone directory is empty or missing",
        }

    # Compute clone size
    size_bytes = get_dir_size(target)
    size_mb = size_bytes / (1024 * 1024)
    file_count = sum(len(files) for _, _, files in os.walk(target))

    print(f"[clone] Success: {target}", file=sys.stderr)
    print(f"[clone] Size: {size_mb:.1f} MB, Files: {file_count}", file=sys.stderr)

    return {
        "repo_id": repo,
        "local_path": os.path.abspath(target),
        "clone_success": True,
        "error": None,
    }


def main():
    parser = argparse.ArgumentParser(description="Shallow-clone a GitHub repo for local analysis.")
    parser.add_argument("--repo", required=True, help="Repository in owner/name format")
    parser.add_argument("--output-dir", required=True, help="Parent directory for the clone")
    parser.add_argument("--depth", type=int, default=1, help="Clone depth (default: 1)")
    parser.add_argument("--branch", default=None, help="Branch to clone (default: repo default branch)")
    args = parser.parse_args()

    result = clone_repo(args.repo, args.output_dir, args.depth, args.branch)
    print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Generate comparison matrix across analyzed repositories.

Reads analysis JSON files and dependency files to produce a structured comparison.

Usage:
    python compare_repos.py --input phase4_deep_dive/analyses/ --output comparison.json
"""

import argparse
import json
import os
import sys
from itertools import combinations
from pathlib import Path


# ---------------------------------------------------------------------------
# I/O helpers
# ---------------------------------------------------------------------------

def load_json(path: str) -> dict | None:
    """Load a JSON file, returning None on failure."""
    try:
        with open(path, encoding="utf-8") as f:
            return json.load(f)
    except (OSError, json.JSONDecodeError) as exc:
        print(f"  Warning: cannot load {path}: {exc}", file=sys.stderr)
        return None


def find_analysis_files(analyses_dir: str) -> dict[str, dict[str, dict | None]]:
    """Discover *_structure.json and *_deps.json files, grouped by repo name.

    Returns {repo_name: {"structure": <dict|None>, "deps": <dict|None>}}.
    """
    repos: dict[str, dict[str, dict | None]] = {}

    for fname in sorted(os.listdir(analyses_dir)):
        fpath = os.path.join(analyses_dir, fname)
        if not os.path.isfile(fpath):
            continue

        if fname.endswith("_structure.json"):
            name = fname.removesuffix("_structure.json")
            repos.setdefault(name, {"structure": None, "deps": None})
            repos[name]["structure"] = load_json(fpath)
        elif fname.endswith("_deps.json"):
            name = fname.removesuffix("_deps.json")
            repos.setdefault(name, {"structure": None, "deps": None})
            repos[name]["deps"] = load_json(fpath)

    return repos


# ---------------------------------------------------------------------------
# Dimension extractors
# ---------------------------------------------------------------------------

def _get(data: dict | None, *keys: str, default: object = None) -> object:
    """Try multiple keys on a dict, returning the first truthy value."""
    if data is None:
        return default
    for key in keys:
        val = data.get(key)
        if val is not None and val != "" and val != [] and val != {}:
            return val
    return default


def extract_repo_id(
    structure: dict | None, deps: dict | None, file_key: str,
) -> str:
    """Best-effort extraction of owner/name repo_id."""
    rid = _get(structure, "repo_id", "repo")
    if rid:
        return str(rid)
    rid = _get(deps, "repo_id", "repo")
    if rid:
        return str(rid)
    return file_key


def extract_primary_language(structure: dict | None) -> str:
    """Primary language from structure analysis."""
    lang = _get(structure, "primary_language", "language")
    if lang and isinstance(lang, str):
        return lang
    # Fall back: pick language with highest LOC
    langs = _get(structure, "languages", "language_stats", default={})
    if isinstance(langs, dict) and langs:
        return max(langs, key=lambda k: langs[k] if isinstance(langs[k], (int, float)) else 0)
    return "Unknown"


def extract_total_loc(structure: dict | None) -> int:
    """Total lines of code."""
    loc = _get(structure, "total_loc", "loc", default=0)
    if isinstance(loc, (int, float)):
        return int(loc)
    # Sum across languages
    langs = _get(structure, "languages", "language_stats", default={})
    if isinstance(langs, dict):
        return sum(v for v in langs.values() if isinstance(v, (int, float)))
    return 0


def extract_total_files(structure: dict | None) -> int:
    """Total file count."""
    count = _get(structure, "total_files", "file_count", default=0)
    if isinstance(count, (int, float)):
        return int(count)
    files = _get(structure, "files", "file_tree", default=[])
    if isinstance(files, list):
        return len(files)
    return 0


def extract_stars(structure: dict | None) -> int:
    """Star count."""
    return int(_get(structure, "stars", default=0) or 0)


def extract_ml_frameworks(deps: dict | None) -> list[str]:
    """Detect ML frameworks from dependency data."""
    if deps is None:
        return _get(deps, "ml_frameworks", default=[]) or []  # type: ignore[return-value]

    # If the deps file already has ml_frameworks, use it
    existing = deps.get("ml_frameworks")
    if existing and isinstance(existing, list):
        return existing

    return []


def extract_has_tests(structure: dict | None) -> bool:
    """Whether the repo has test files."""
    if structure is None:
        return False
    # Direct field
    test_files = _get(structure, "test_files", default=[])
    if isinstance(test_files, list) and test_files:
        return True
    # Check has_tests flag
    if structure.get("has_tests"):
        return True
    # Scan file list for test indicators
    files = _get(structure, "files", "file_tree", default=[])
    if isinstance(files, list):
        for f in files:
            name = f if isinstance(f, str) else (
                f.get("path", "") if isinstance(f, dict) else ""
            )
            name_lower = name.lower()
            if ("/test/" in name_lower or "/tests/" in name_lower
                    or name_lower.startswith("test") or "test_" in name_lower):
                return True
    # Check key directories
    key_dirs = _get(structure, "key_dirs", "directories", default=[])
    if isinstance(key_dirs, list):
        for d in key_dirs:
            d_name = d if isinstance(d, str) else (
                d.get("name", "") if isinstance(d, dict) else ""
            )
            if d_name.lower() in ("test", "tests", "testing", "test_suite"):
                return True
    return False


def extract_has_docker(structure: dict | None) -> bool:
    """Whether the repo has Docker configuration."""
    if structure is None:
        return False
    config_files = _get(structure, "config_files", default=[])
    if isinstance(config_files, list):
        for f in config_files:
            name = f if isinstance(f, str) else (
                f.get("path", f.get("name", "")) if isinstance(f, dict) else ""
            )
            name_lower = name.lower()
            if name_lower in ("dockerfile", "docker-compose.yml",
                              "docker-compose.yaml", ".dockerignore"):
                return True
    # Also check files list
    files = _get(structure, "files", "file_tree", default=[])
    if isinstance(files, list):
        for f in files:
            name = f if isinstance(f, str) else (
                f.get("path", "") if isinstance(f, dict) else ""
            )
            basename = os.path.basename(name).lower()
            if basename in ("dockerfile", "docker-compose.yml",
                            "docker-compose.yaml"):
                return True
    return False


def extract_license(structure: dict | None) -> str:
    """License identifier."""
    return str(_get(structure, "license", default="") or "")


def extract_entry_points(structure: dict | None) -> list[str]:
    """Main entry points."""
    entries = _get(structure, "entry_points", "main_files", default=[])
    if isinstance(entries, list):
        return [str(e) for e in entries[:10]]
    return []


def extract_config_format(structure: dict | None) -> str:
    """Dominant config file format."""
    config_files = _get(structure, "config_files", default=[])
    if not isinstance(config_files, list):
        return ""

    format_counts: dict[str, int] = {}
    ext_map: dict[str, str] = {
        ".yaml": "yaml", ".yml": "yaml",
        ".toml": "toml",
        ".json": "json",
        ".ini": "ini", ".cfg": "ini",
    }

    for f in config_files:
        name = f if isinstance(f, str) else (
            f.get("path", f.get("name", "")) if isinstance(f, dict) else ""
        )
        _, ext = os.path.splitext(name.lower())
        fmt = ext_map.get(ext, "")
        if fmt:
            format_counts[fmt] = format_counts.get(fmt, 0) + 1

    if not format_counts:
        return ""
    return max(format_counts, key=lambda k: format_counts[k])


# ---------------------------------------------------------------------------
# Dependency overlap
# ---------------------------------------------------------------------------

def collect_all_packages(deps: dict | None) -> set[str]:
    """Collect a flat set of normalized package names from a deps record."""
    if deps is None:
        return set()

    # Prefer the pre-computed all_packages field
    all_pkgs = deps.get("all_packages")
    if isinstance(all_pkgs, list) and all_pkgs:
        return {p.lower() for p in all_pkgs}

    # Otherwise, gather from per-ecosystem sections
    names: set[str] = set()

    # Python
    py = deps.get("python")
    if isinstance(py, dict):
        for spec in py.get("requirements", []):
            if isinstance(spec, str):
                bare = spec.split(">=")[0].split("<=")[0].split("==")[0]
                bare = bare.split("~=")[0].split("!=")[0].split("[")[0].strip()
                if bare:
                    names.add(bare.lower())

    # Node
    node = deps.get("node")
    if isinstance(node, dict):
        for key in ("dependencies", "devDependencies"):
            val = node.get(key, {})
            if isinstance(val, dict):
                names.update(k.lower() for k in val)

    # Rust
    rust = deps.get("rust")
    if isinstance(rust, dict):
        for key in ("dependencies",):
            val = rust.get(key, {})
            if isinstance(val, dict):
                names.update(k.lower() for k in val)

    # Go
    go = deps.get("go")
    if isinstance(go, dict):
        for mod in go.get("modules", []):
            if isinstance(mod, str):
                parts = mod.rstrip("/").split("/")
                names.add(parts[-1].lower())

    return names


def compute_dependency_overlap(
    repo_packages: dict[str, set[str]],
    min_shared: int = 3,
) -> list[dict]:
    """Compute shared packages for each pair of repos.

    Only includes pairs with at least min_shared common packages.
    """
    overlaps: list[dict] = []
    repo_ids = sorted(repo_packages.keys())
    for a, b in combinations(repo_ids, 2):
        shared = sorted(repo_packages[a] & repo_packages[b])
        if len(shared) >= min_shared:
            overlaps.append({
                "repos": [a, b],
                "shared": shared,
            })
    return overlaps


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Generate comparison matrix across analyzed repositories",
    )
    parser.add_argument(
        "--input", required=True,
        help="Directory containing *_structure.json and *_deps.json files",
    )
    parser.add_argument(
        "--output", required=True,
        help="Output comparison JSON file path",
    )
    parser.add_argument(
        "--repo-db", default=None,
        help="Optional repo_db.jsonl to enrich with stars/metadata",
    )

    args = parser.parse_args()

    if not os.path.isdir(args.input):
        print(f"Error: analyses directory not found: {args.input}",
              file=sys.stderr)
        sys.exit(1)

    # Load optional repo_db for enrichment (stars, etc.)
    repo_db_map: dict[str, dict] = {}
    if args.repo_db and os.path.isfile(args.repo_db):
        with open(args.repo_db, encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if line:
                    rec = json.loads(line)
                    rid = rec.get("repo_id", "")
                    # Index by both full id and short name
                    repo_db_map[rid] = rec
                    repo_db_map[rid.split("/")[-1]] = rec

    # Discover and load analysis files
    repos = find_analysis_files(args.input)
    if not repos:
        print("Error: no *_structure.json or *_deps.json files found",
              file=sys.stderr)
        sys.exit(1)

    print(f"Found analysis files for {len(repos)} repo(s)", file=sys.stderr)

    # --- Build dimension maps ---
    repo_ids: list[str] = []
    dim_primary_language: dict[str, str] = {}
    dim_total_loc: dict[str, int] = {}
    dim_total_files: dict[str, int] = {}
    dim_stars: dict[str, int] = {}
    dim_ml_framework: dict[str, list[str]] = {}
    dim_has_tests: dict[str, bool] = {}
    dim_has_docker: dict[str, bool] = {}
    dim_license: dict[str, str] = {}
    dim_entry_points: dict[str, list[str]] = {}
    dim_config_format: dict[str, str] = {}

    repo_packages: dict[str, set[str]] = {}

    for file_key, data in sorted(repos.items()):
        structure = data.get("structure")
        deps = data.get("deps")
        rid = extract_repo_id(structure, deps, file_key)
        repo_ids.append(rid)

        dim_primary_language[rid] = extract_primary_language(structure)
        dim_total_loc[rid] = extract_total_loc(structure)
        dim_total_files[rid] = extract_total_files(structure)
        # Stars from structure.json, fallback to repo_db
        stars = extract_stars(structure)
        if stars == 0 and rid in repo_db_map:
            stars = int(repo_db_map[rid].get("stars", 0))
        if stars == 0:
            # Try matching by short name
            short = rid.split("/")[-1]
            if short in repo_db_map:
                stars = int(repo_db_map[short].get("stars", 0))
        dim_stars[rid] = stars
        dim_ml_framework[rid] = extract_ml_frameworks(deps)
        dim_has_tests[rid] = extract_has_tests(structure)
        dim_has_docker[rid] = extract_has_docker(structure)
        dim_license[rid] = extract_license(structure)
        dim_entry_points[rid] = extract_entry_points(structure)
        dim_config_format[rid] = extract_config_format(structure)

        repo_packages[rid] = collect_all_packages(deps)

    # --- Compute pairwise dependency overlap ---
    dependency_overlap = compute_dependency_overlap(repo_packages)

    # --- Assemble output ---
    comparison: dict = {
        "repos": repo_ids,
        "dimensions": {
            "primary_language": dim_primary_language,
            "total_loc": dim_total_loc,
            "total_files": dim_total_files,
            "stars": dim_stars,
            "ml_framework": dim_ml_framework,
            "has_tests": dim_has_tests,
            "has_docker": dim_has_docker,
            "license": dim_license,
            "entry_points": dim_entry_points,
            "config_format": dim_config_format,
        },
        "dependency_overlap": dependency_overlap,
    }

    # --- Write output ---
    output_path = Path(args.output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(comparison, f, indent=2, ensure_ascii=False)
        f.write("\n")

    # --- Summary to stderr ---
    n_dims = len(comparison["dimensions"])
    lang_counts: dict[str, int] = {}
    for lang in dim_primary_language.values():
        lang_counts[lang] = lang_counts.get(lang, 0) + 1
    lang_summary = ", ".join(
        f"{l}({c})" for l, c in sorted(lang_counts.items(), key=lambda x: -x[1])
    )

    fw_counts: dict[str, int] = {}
    for fws in dim_ml_framework.values():
        for fw in fws:
            fw_counts[fw] = fw_counts.get(fw, 0) + 1
    fw_summary = ", ".join(
        f"{f}({c})" for f, c in sorted(fw_counts.items(), key=lambda x: -x[1])
    ) if fw_counts else "none"

    test_count = sum(1 for v in dim_has_tests.values() if v)
    docker_count = sum(1 for v in dim_has_docker.values() if v)

    print(f"Compared {len(repo_ids)} repos across {n_dims} dimensions",
          file=sys.stderr)
    print(f"  Languages: {lang_summary}", file=sys.stderr)
    print(f"  ML frameworks: {fw_summary}", file=sys.stderr)
    print(f"  With tests: {test_count}/{len(repo_ids)}", file=sys.stderr)
    print(f"  With Docker: {docker_count}/{len(repo_ids)}", file=sys.stderr)
    print(f"  Dependency overlaps: {len(dependency_overlap)} pair(s)",
          file=sys.stderr)
    print(f"  Output: {args.output}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Assemble final GitHub research report from all phase outputs.

Reads repo_db.jsonl and phase outputs to generate statistics and a comprehensive report.

Usage:
    python compile_github_report.py --topic-dir ./github-research-output/my-topic
"""

import argparse
import json
import math
import os
import sys
from datetime import datetime, timezone
from pathlib import Path


# -- I/O helpers --------------------------------------------------------------

def load_jsonl(path: str) -> list[dict]:
    """Load records from a JSONL file. Returns empty list if file missing."""
    records: list[dict] = []
    if not os.path.isfile(path):
        return records
    with open(path, encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                try:
                    records.append(json.loads(line))
                except json.JSONDecodeError:
                    continue
    return records


def load_text(path: str) -> str:
    """Load a text file. Returns empty string if file missing."""
    if not os.path.isfile(path):
        return ""
    try:
        with open(path, encoding="utf-8") as f:
            return f.read()
    except OSError:
        return ""


def format_table(headers: list[str], rows: list[list[str]]) -> str:
    """Generate a markdown table from headers and row data."""
    if not headers:
        return ""
    lines: list[str] = []
    lines.append("| " + " | ".join(headers) + " |")
    # Alignment: right-align columns that look numeric in first data row
    seps: list[str] = []
    for i, h in enumerate(headers):
        if rows and i < len(rows[0]):
            try:
                float(str(rows[0][i]).replace(",", ""))
                seps.append("------:")
            except (ValueError, IndexError):
                seps.append("------")
        else:
            seps.append("------")
    lines.append("| " + " | ".join(seps) + " |")
    for row in rows:
        cells = [str(c) for c in row]
        while len(cells) < len(headers):
            cells.append("")
        lines.append("| " + " | ".join(cells) + " |")
    return "\n".join(lines)


def truncate(text: str, max_chars: int = 200) -> str:
    """Truncate text with ellipsis if it exceeds max_chars."""
    if len(text) <= max_chars:
        return text
    return text[: max_chars - 3].rstrip() + "..."


# -- Statistics ---------------------------------------------------------------

def compute_stats(repos: list[dict]) -> dict:
    """Compute all statistics from repo records."""
    if not repos:
        return {
            "total_discovered": 0, "total_filtered": 0, "total_analyzed": 0,
            "sources": {}, "languages": {}, "stars": {},
            "activity": {}, "score_buckets": {}, "ml_frameworks": {}, "licenses": {},
        }

    now = datetime.now(timezone.utc)

    # -- By source --
    sources: dict[str, int] = {}
    for rec in repos:
        src = rec.get("source", "unknown")
        sources[src] = sources.get(src, 0) + 1

    # -- By language --
    languages: dict[str, int] = {}
    for rec in repos:
        lang = rec.get("language") or "Unknown"
        languages[lang] = languages.get(lang, 0) + 1
    languages = dict(sorted(languages.items(), key=lambda x: -x[1]))

    # -- Stars distribution --
    star_values = [rec.get("stars", 0) or 0 for rec in repos]
    star_values_sorted = sorted(star_values)
    n = len(star_values_sorted)
    stars_info: dict[str, float | int] = {
        "min": star_values_sorted[0],
        "max": star_values_sorted[-1],
        "median": star_values_sorted[n // 2] if n % 2 else
                  (star_values_sorted[n // 2 - 1] + star_values_sorted[n // 2]) / 2,
        "mean": round(sum(star_values) / n, 1),
        "p25": star_values_sorted[max(n // 4 - 1, 0)],
        "p75": star_values_sorted[min(3 * n // 4, n - 1)],
    }

    # -- Activity --
    activity: dict[str, int] = {"last_90d": 0, "last_year": 0, "older": 0}
    for rec in repos:
        pushed_str = rec.get("pushed_at") or rec.get("updated_at") or ""
        if not pushed_str:
            activity["older"] += 1
            continue
        try:
            pushed = datetime.fromisoformat(pushed_str.replace("Z", "+00:00"))
            days = (now - pushed).days
            if days <= 90:
                activity["last_90d"] += 1
            elif days <= 365:
                activity["last_year"] += 1
            else:
                activity["older"] += 1
        except (ValueError, TypeError):
            activity["older"] += 1

    # -- Composite score histogram --
    score_buckets: dict[str, int] = {
        "0.0-0.2": 0, "0.2-0.4": 0, "0.4-0.6": 0, "0.6-0.8": 0, "0.8-1.0": 0,
    }
    for rec in repos:
        cs = rec.get("composite_score", 0.0) or 0.0
        if cs >= 0.8:
            score_buckets["0.8-1.0"] += 1
        elif cs >= 0.6:
            score_buckets["0.6-0.8"] += 1
        elif cs >= 0.4:
            score_buckets["0.4-0.6"] += 1
        elif cs >= 0.2:
            score_buckets["0.2-0.4"] += 1
        else:
            score_buckets["0.0-0.2"] += 1

    # -- ML frameworks --
    ml_keywords = {
        "torch": "PyTorch", "pytorch": "PyTorch",
        "tensorflow": "TensorFlow", "tf-": "TensorFlow",
        "jax": "JAX", "flax": "JAX",
        "keras": "Keras", "sklearn": "scikit-learn",
        "transformers": "HuggingFace Transformers",
        "lightning": "PyTorch Lightning", "paddle": "PaddlePaddle",
    }
    ml_frameworks: dict[str, int] = {}
    for rec in repos:
        topics = rec.get("topics", []) or []
        desc = (rec.get("description") or "").lower()
        readme = (rec.get("readme_excerpt") or "").lower()
        combined = desc + " " + readme + " " + " ".join(str(t) for t in topics)
        found_for_repo: set[str] = set()
        for kw, framework in ml_keywords.items():
            if kw in combined:
                found_for_repo.add(framework)
        for fw in found_for_repo:
            ml_frameworks[fw] = ml_frameworks.get(fw, 0) + 1
    ml_frameworks = dict(sorted(ml_frameworks.items(), key=lambda x: -x[1]))

    # -- Licenses --
    licenses: dict[str, int] = {}
    for rec in repos:
        lic = rec.get("license") or "None/Unknown"
        if isinstance(lic, dict):
            lic = lic.get("spdx_id") or lic.get("name") or "None/Unknown"
        licenses[lic] = licenses.get(lic, 0) + 1
    licenses = dict(sorted(licenses.items(), key=lambda x: -x[1]))

    # -- Counts for filtered / analyzed --
    scored_repos = [r for r in repos if r.get("composite_score")]
    analyzed_repos = [r for r in repos if r.get("tags") and "deep-dived" in r.get("tags", [])]
    # Fallback: count repos with rank field as proxy for filtered
    ranked_repos = [r for r in repos if r.get("rank")]

    return {
        "total_discovered": len(repos),
        "total_filtered": len(ranked_repos) if ranked_repos else len(scored_repos),
        "total_analyzed": len(analyzed_repos),
        "num_sources": len(sources),
        "sources": sources,
        "languages": languages,
        "stars": stars_info,
        "activity": activity,
        "score_buckets": score_buckets,
        "ml_frameworks": ml_frameworks,
        "licenses": licenses,
    }


# -- Report sections ----------------------------------------------------------

def build_header(topic_dir: str) -> str:
    """Build the report header with topic name and date."""
    slug = os.path.basename(topic_dir.rstrip("/"))
    topic = slug.replace("-", " ").title()
    date = datetime.now().strftime("%Y-%m-%d %H:%M")
    return f"# GitHub Research Report: {topic}\n\nGenerated: {date}\n"


def build_executive_summary(stats: dict, blueprint_summary: str) -> str:
    """Build the executive summary section."""
    lines = ["## Executive Summary\n"]
    if blueprint_summary:
        lines.append(blueprint_summary.strip())
    else:
        total = stats["total_discovered"]
        ns = stats["num_sources"]
        filtered = stats["total_filtered"]
        analyzed = stats["total_analyzed"]
        top_lang = next(iter(stats["languages"]), "N/A")
        active = stats["activity"].get("last_90d", 0)
        lines.append(
            f"This report covers a systematic GitHub research effort that discovered "
            f"**{total} repositories** from **{ns} source(s)**. "
            f"After scoring and filtering, **{filtered}** repos were shortlisted "
            f"and **{analyzed or 'several'}** received deep code analysis. "
            f"The dominant language is **{top_lang}** and "
            f"**{active}** repos have been active in the last 90 days."
        )
    lines.append("")
    return "\n".join(lines)


def build_discovery_stats(stats: dict) -> str:
    """Build the discovery statistics section."""
    lines = ["## 1. Discovery Statistics\n"]
    lines.append(
        f"- Repos discovered: **{stats['total_discovered']}** "
        f"from **{stats['num_sources']}** sources"
    )
    lines.append(
        f"- After filtering: **{stats['total_filtered']}** repos "
        f"(top by composite score)"
    )
    lines.append(f"- Deep-dived: **{stats['total_analyzed']}** repos\n")

    # Source table
    if stats["sources"]:
        lines.append("### Repos by Source\n")
        rows = [[src, str(cnt)] for src, cnt in
                sorted(stats["sources"].items(), key=lambda x: -x[1])]
        lines.append(format_table(["Source", "Count"], rows))
        lines.append("")

    # Language table
    if stats["languages"]:
        lines.append("### Repos by Language\n")
        rows = [[lang, str(cnt)] for lang, cnt in list(stats["languages"].items())[:15]]
        lines.append(format_table(["Language", "Count"], rows))
        lines.append("")

    # Stars distribution
    stars = stats.get("stars", {})
    if stars:
        lines.append("### Stars Distribution\n")
        lines.append(f"- Min: **{stars.get('min', 0)}** | "
                     f"Max: **{stars.get('max', 0)}** | "
                     f"Median: **{stars.get('median', 0)}** | "
                     f"Mean: **{stars.get('mean', 0)}**")
        lines.append(f"- P25: **{stars.get('p25', 0)}** | P75: **{stars.get('p75', 0)}**\n")

    # Score buckets
    if stats["score_buckets"]:
        lines.append("### Composite Score Distribution\n")
        rows = [[bucket, str(cnt)] for bucket, cnt in stats["score_buckets"].items()]
        lines.append(format_table(["Score Range", "Count"], rows))
        lines.append("")

    # Activity
    if stats["activity"]:
        lines.append("### Activity\n")
        act = stats["activity"]
        lines.append(f"- Pushed in last 90 days: **{act.get('last_90d', 0)}**")
        lines.append(f"- Pushed in last year: **{act.get('last_year', 0)}**")
        lines.append(f"- Older / unknown: **{act.get('older', 0)}**\n")

    # ML frameworks
    if stats["ml_frameworks"]:
        lines.append("### ML Frameworks\n")
        rows = [[fw, str(cnt)] for fw, cnt in stats["ml_frameworks"].items()]
        lines.append(format_table(["Framework", "Repos"], rows))
        lines.append("")

    # Licenses
    if stats["licenses"]:
        lines.append("### License Distribution\n")
        rows = [[lic, str(cnt)] for lic, cnt in list(stats["licenses"].items())[:10]]
        lines.append(format_table(["License", "Count"], rows))
        lines.append("")

    return "\n".join(lines)


def build_top_repos(ranked_repos: list[dict]) -> str:
    """Build the top repositories table from ranked_repos.jsonl."""
    lines = ["## 2. Top Repositories\n"]
    if not ranked_repos:
        lines.append("_No ranked repos available._\n")
        return "\n".join(lines)

    headers = ["Rank", "Repo", "Stars", "Language", "Composite", "Description"]
    rows: list[list[str]] = []
    for i, rec in enumerate(ranked_repos[:30], 1):
        rid = rec.get("repo_id", "")
        url = rec.get("url", f"https://github.com/{rid}")
        stars = str(rec.get("stars", 0))
        lang = rec.get("language", "")
        score = f"{(rec.get('composite_score', 0.0) or 0.0):.3f}"
        desc = truncate((rec.get("description") or "").replace("|", "/"), 60)
        rows.append([str(i), f"[{rid}]({url})", stars, lang, score, desc])

    lines.append(format_table(headers, rows))
    lines.append("")
    return "\n".join(lines)


def build_deep_analysis_summaries(analyses_dir: str) -> str:
    """Build condensed summaries from per-repo analyses/*.md files."""
    lines = ["## 3. Deep Analysis Summaries\n"]
    if not os.path.isdir(analyses_dir):
        lines.append("_No deep analysis files found._\n")
        return "\n".join(lines)

    analysis_files = sorted(
        f for f in os.listdir(analyses_dir) if f.endswith("_analysis.md")
    )
    if not analysis_files:
        lines.append("_No per-repo analysis files found._\n")
        return "\n".join(lines)

    for fname in analysis_files:
        content = load_text(os.path.join(analyses_dir, fname))
        if not content:
            continue
        # Extract repo name from filename
        repo_name = fname.removesuffix("_analysis.md")
        lines.append(f"### {repo_name}\n")
        # Include the first ~600 chars as a condensed summary
        lines.append(truncate(content.strip(), 600))
        lines.append("")

    return "\n".join(lines)


def build_section(title: str, content: str) -> str:
    """Wrap phase content in a section. Return empty section note if no content."""
    lines = [f"{title}\n"]
    if content.strip():
        lines.append(content.strip())
    else:
        heading_text = title.lstrip("#").strip()
        lines.append(f"_{heading_text} not available._")
    lines.append("")
    return "\n".join(lines)


def build_appendix(repos: list[dict]) -> str:
    """Build the full repo database appendix table."""
    lines = ["## Appendix A: Full Repo Database\n"]
    if not repos:
        lines.append("_No repos in database._\n")
        return "\n".join(lines)

    sorted_repos = sorted(repos, key=lambda r: -(r.get("composite_score", 0) or 0))
    headers = ["Repo", "Stars", "Language", "Score", "Source", "Description"]
    rows: list[list[str]] = []
    for rec in sorted_repos:
        rid = rec.get("repo_id", "")
        url = rec.get("url", f"https://github.com/{rid}")
        stars = str(rec.get("stars", 0))
        lang = rec.get("language", "")
        score = f"{(rec.get('composite_score', 0.0) or 0.0):.3f}"
        src = rec.get("source", "")
        desc = truncate((rec.get("description") or "").replace("|", "/"), 60)
        rows.append([f"[{rid}]({url})", stars, lang, score, src, desc])

    lines.append(format_table(headers, rows))
    lines.append("")
    return "\n".join(lines)


def build_methodology() -> str:
    """Build the methodology appendix."""
    return """## Appendix B: Methodology

- **Phase 1: Intake** -- Parse deep-research output for GitHub URLs, paper references, and search keywords
- **Phase 2: Discovery** -- Multi-source search (GitHub repo search, Papers With Code, GitHub code search, direct URLs)
- **Phase 3: Scoring & Filtering** -- Composite score = relevance x 0.4 + quality x 0.35 + activity x 0.25; select top repos
- **Phase 4: Deep Dive** -- Shallow clone + deep code reading of top repos (architecture, algorithms, quality, reusability)
- **Phase 5: Cross-Repo Analysis** -- Comparison matrix, technique-to-code mapping, gap analysis
- **Phase 6: Integration Blueprint** -- Recommended architecture, reuse catalog, license compatibility, effort estimates
"""


# -- Main ---------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Assemble final GitHub research report from all phase outputs."
    )
    parser.add_argument(
        "--topic-dir", required=True,
        help="Path to the github-research-output/{slug}/ directory",
    )
    args = parser.parse_args()

    topic_dir = args.topic_dir.rstrip("/")
    if not os.path.isdir(topic_dir):
        print(f"[error] topic directory not found: {topic_dir}", file=sys.stderr)
        sys.exit(1)

    print(f"[compile] loading data from {topic_dir}", file=sys.stderr)

    # -- Load repo database --
    repo_db_path = os.path.join(topic_dir, "repo_db.jsonl")
    all_repos = load_jsonl(repo_db_path)
    print(f"  repo_db.jsonl: {len(all_repos)} repos", file=sys.stderr)

    # -- Load ranked repos --
    ranked_path = os.path.join(topic_dir, "phase3_filtering", "ranked_repos.jsonl")
    ranked_repos = load_jsonl(ranked_path)
    print(f"  ranked_repos.jsonl: {len(ranked_repos)} repos", file=sys.stderr)

    # -- Load phase text outputs --
    phase_paths = {
        "intake_summary":    os.path.join(topic_dir, "phase1_intake", "intake_summary.md"),
        "discovery_log":     os.path.join(topic_dir, "phase2_discovery", "discovery_log.md"),
        "filtering_report":  os.path.join(topic_dir, "phase3_filtering", "filtering_report.md"),
        "deep_dive_summary": os.path.join(topic_dir, "phase4_deep_dive", "deep_dive_summary.md"),
        "comparison_matrix": os.path.join(topic_dir, "phase5_analysis", "comparison_matrix.md"),
        "technique_map":     os.path.join(topic_dir, "phase5_analysis", "technique_map.md"),
        "analysis_report":   os.path.join(topic_dir, "phase5_analysis", "analysis_report.md"),
        "integration_plan":  os.path.join(topic_dir, "phase6_blueprint", "integration_plan.md"),
        "reuse_catalog":     os.path.join(topic_dir, "phase6_blueprint", "reuse_catalog.md"),
        "blueprint_summary": os.path.join(topic_dir, "phase6_blueprint", "blueprint_summary.md"),
    }

    phases: dict[str, str] = {}
    for key, path in phase_paths.items():
        phases[key] = load_text(path)
        status = "loaded" if phases[key] else "not found"
        print(f"  {key}: {status}", file=sys.stderr)

    analyses_dir = os.path.join(topic_dir, "phase4_deep_dive", "analyses")

    # -- Compute statistics --
    stats = compute_stats(all_repos)
    # Override total_filtered with actual ranked count if available
    if ranked_repos:
        stats["total_filtered"] = len(ranked_repos)
    # Count analysis files for total_analyzed
    if os.path.isdir(analyses_dir):
        analysis_count = sum(
            1 for f in os.listdir(analyses_dir) if f.endswith("_analysis.md")
        )
        if analysis_count > 0:
            stats["total_analyzed"] = analysis_count

    print(f"[compile] assembling report...", file=sys.stderr)

    # -- Assemble report --
    sections: list[str] = [
        build_header(topic_dir),
        "---\n",
        build_executive_summary(stats, phases["blueprint_summary"]),
        build_discovery_stats(stats),
        build_top_repos(ranked_repos if ranked_repos else all_repos),
        build_deep_analysis_summaries(analyses_dir),
        build_section("## 4. Cross-Repository Comparison", phases["comparison_matrix"]),
        build_section("## 5. Technique-to-Code Mapping", phases["technique_map"]),
        build_section("## 6. Integration Blueprint", phases["integration_plan"]),
        build_section("## 7. Reusable Components", phases["reuse_catalog"]),
        build_appendix(all_repos),
        build_methodology(),
    ]

    report = "\n".join(sections)

    # -- Write outputs --
    output_dir = os.path.join(topic_dir, "phase6_blueprint")
    os.makedirs(output_dir, exist_ok=True)

    report_path = os.path.join(output_dir, "final_report.md")
    with open(report_path, "w", encoding="utf-8") as f:
        f.write(report)

    stats_path = os.path.join(output_dir, "stats.json")
    with open(stats_path, "w", encoding="utf-8") as f:
        json.dump(stats, f, indent=2, ensure_ascii=False)

    # -- Summary --
    section_count = sum(1 for s in sections if s.strip().startswith("##"))
    print(
        f"\nReport compiled: {section_count} sections, "
        f"{stats['total_discovered']} repos, "
        f"{stats['total_analyzed']} analyzed",
        file=sys.stderr,
    )
    print(f"  -> {report_path}", file=sys.stderr)
    print(f"  -> {stats_path}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Extract and parse dependency files from a cloned repository.

Identifies Python, Node, Rust, Go, and system dependencies.
Self-contained: stdlib only (regex-based parsing, no toml/yaml libraries).

Usage:
    python extract_dependencies.py --repo-dir ./repos/owner_name --output deps.json
"""

import argparse
import json
import os
import re
import sys
from pathlib import Path


# ---------------------------------------------------------------------------
# ML framework detection mapping
# ---------------------------------------------------------------------------

ML_FRAMEWORK_MAP: dict[str, str] = {
    # PyTorch ecosystem
    "torch": "pytorch", "torchvision": "pytorch", "torchaudio": "pytorch",
    "pytorch-lightning": "pytorch", "lightning": "pytorch",
    # TensorFlow ecosystem
    "tensorflow": "tensorflow", "tensorflow-gpu": "tensorflow",
    "tf-nightly": "tensorflow", "tf-estimator-nightly": "tensorflow",
    "keras": "tensorflow",
    # JAX ecosystem
    "jax": "jax", "jaxlib": "jax", "flax": "jax", "optax": "jax",
    # HuggingFace ecosystem
    "transformers": "huggingface", "datasets": "huggingface",
    "tokenizers": "huggingface", "accelerate": "huggingface",
    "diffusers": "huggingface", "peft": "huggingface",
    # scikit-learn
    "scikit-learn": "scikit-learn", "sklearn": "scikit-learn",
    # Other ML tools
    "onnx": "onnx", "onnxruntime": "onnx",
    "tensorrt": "tensorrt", "triton": "triton",
    "deepspeed": "deepspeed", "fairscale": "fairscale",
}

SYSTEM_KEYWORDS: list[str] = [
    "cuda", "cudnn", "nvidia", "ffmpeg", "libsndfile", "sox",
    "opencv", "libgl", "libglib", "cmake", "gcc", "g++",
]


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _read_text(path: Path) -> str | None:
    """Read a file as UTF-8, returning None on failure."""
    try:
        return path.read_text(encoding="utf-8", errors="replace")
    except OSError:
        return None


def _strip_version(spec: str) -> str:
    """Extract bare package name from a version specifier string."""
    return re.split(r"[><=!~;\[\s@]", spec.strip())[0].strip().lower()


def _extract_quoted_strings(text: str) -> list[str]:
    """Extract all single- or double-quoted strings from text."""
    return re.findall(r"""['"]([^'"]+)['"]""", text)


# ---------------------------------------------------------------------------
# Python: requirements.txt
# ---------------------------------------------------------------------------

def parse_requirements_txt(repo_dir: Path) -> dict | None:
    """Parse requirements*.txt files."""
    candidates = [
        "requirements.txt", "requirements-dev.txt", "requirements_dev.txt",
        "requirements-test.txt", "requirements_test.txt",
        "requirements/base.txt", "requirements/main.txt",
    ]
    all_specs: list[str] = []
    source_files: list[str] = []

    for name in candidates:
        path = repo_dir / name
        if not path.is_file():
            continue
        text = _read_text(path)
        if text is None:
            continue
        source_files.append(name)
        for line in text.splitlines():
            line = line.strip()
            if not line or line.startswith("#") or line.startswith("-"):
                continue
            all_specs.append(line)

    if not all_specs:
        return None
    # Deduplicate while preserving order
    seen: set[str] = set()
    deduped: list[str] = []
    for spec in all_specs:
        if spec not in seen:
            seen.add(spec)
            deduped.append(spec)
    return {"requirements": deduped, "source_file": ", ".join(source_files)}


# ---------------------------------------------------------------------------
# Python: setup.py
# ---------------------------------------------------------------------------

def parse_setup_py(repo_dir: Path) -> dict | None:
    """Parse setup.py for install_requires and extras_require."""
    path = repo_dir / "setup.py"
    if not path.is_file():
        return None
    text = _read_text(path)
    if text is None:
        return None

    specs: list[str] = []

    # install_requires = [...]
    match = re.search(r"install_requires\s*=\s*\[([^\]]+)\]", text, re.DOTALL)
    if match:
        specs.extend(_extract_quoted_strings(match.group(1)))

    # extras_require = { ... }
    match = re.search(r"extras_require\s*=\s*\{([^}]+)\}", text, re.DOTALL)
    if match:
        specs.extend(_extract_quoted_strings(match.group(1)))

    if not specs:
        return None
    return {"requirements": specs, "source_file": "setup.py"}


# ---------------------------------------------------------------------------
# Python: pyproject.toml
# ---------------------------------------------------------------------------

def parse_pyproject_toml(repo_dir: Path) -> dict | None:
    """Parse pyproject.toml for dependencies (regex-based, no toml lib)."""
    path = repo_dir / "pyproject.toml"
    if not path.is_file():
        return None
    text = _read_text(path)
    if text is None:
        return None

    specs: list[str] = []

    # [project] dependencies = [...]
    match = re.search(r"dependencies\s*=\s*\[([^\]]*)\]", text, re.DOTALL)
    if match:
        specs.extend(_extract_quoted_strings(match.group(1)))

    # [project.optional-dependencies] section -- grab all arrays
    section_match = re.search(
        r"\[project\.optional-dependencies\](.*?)(?=\n\[|\Z)", text, re.DOTALL,
    )
    if section_match:
        for array_match in re.finditer(r"=\s*\[([^\]]*)\]", section_match.group(1)):
            specs.extend(_extract_quoted_strings(array_match.group(1)))

    if not specs:
        return None
    return {"requirements": specs, "source_file": "pyproject.toml"}


# ---------------------------------------------------------------------------
# Python: environment.yml / environment.yaml
# ---------------------------------------------------------------------------

def parse_environment_yml(repo_dir: Path) -> dict | None:
    """Parse environment.yml / environment.yaml for conda dependencies."""
    source_name: str | None = None
    for name in ("environment.yml", "environment.yaml"):
        path = repo_dir / name
        if path.is_file():
            source_name = name
            break
    if source_name is None:
        return None

    text = _read_text(repo_dir / source_name)
    if text is None:
        return None

    specs: list[str] = []
    in_deps = False
    in_pip = False

    for line in text.splitlines():
        stripped = line.strip()

        # Detect dependencies: section
        if re.match(r"^dependencies\s*:", stripped):
            in_deps = True
            continue

        if not in_deps:
            continue

        # End of section on non-indented, non-list line
        if stripped and not stripped.startswith("-") and not stripped.startswith("#"):
            if not in_pip:
                in_deps = False
                continue

        # pip sub-section
        if stripped == "- pip:" or stripped == "- pip":
            in_pip = True
            continue
        if in_pip:
            if stripped.startswith("- "):
                pkg = stripped[2:].strip()
                if pkg:
                    specs.append(pkg)
                continue
            if not stripped.startswith("-") and stripped:
                in_pip = False

        if stripped.startswith("- "):
            pkg = stripped[2:].strip()
            if pkg and pkg not in ("pip", "pip:"):
                specs.append(pkg)

    if not specs:
        return None
    return {"requirements": specs, "source_file": source_name}


# ---------------------------------------------------------------------------
# Node: package.json
# ---------------------------------------------------------------------------

def parse_package_json(repo_dir: Path) -> dict | None:
    """Parse package.json for dependencies and devDependencies."""
    path = repo_dir / "package.json"
    if not path.is_file():
        return None
    text = _read_text(path)
    if text is None:
        return None
    try:
        data = json.loads(text)
    except json.JSONDecodeError:
        return None

    deps = data.get("dependencies") or {}
    dev_deps = data.get("devDependencies") or {}
    if not deps and not dev_deps:
        return None

    result: dict = {"source_file": "package.json"}
    if deps:
        result["dependencies"] = deps
    if dev_deps:
        result["devDependencies"] = dev_deps
    return result


# ---------------------------------------------------------------------------
# Rust: Cargo.toml
# ---------------------------------------------------------------------------

def parse_cargo_toml(repo_dir: Path) -> dict | None:
    """Parse Cargo.toml for [dependencies] section."""
    path = repo_dir / "Cargo.toml"
    if not path.is_file():
        return None
    text = _read_text(path)
    if text is None:
        return None

    deps: dict[str, str] = {}
    match = re.search(
        r"\[dependencies\](.*?)(?=\n\[|\Z)", text, re.DOTALL,
    )
    if match:
        for line in match.group(1).splitlines():
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            # name = "version"
            m = re.match(r'^(\S+)\s*=\s*"([^"]*)"', line)
            if m:
                deps[m.group(1)] = m.group(2)
            else:
                # name = { version = "...", ... }
                m = re.match(r"^(\S+)\s*=\s*\{", line)
                if m:
                    ver = re.search(r'version\s*=\s*"([^"]*)"', line)
                    deps[m.group(1)] = ver.group(1) if ver else "*"

    if not deps:
        return None
    return {"dependencies": deps, "source_file": "Cargo.toml"}


# ---------------------------------------------------------------------------
# Go: go.mod
# ---------------------------------------------------------------------------

def parse_go_mod(repo_dir: Path) -> dict | None:
    """Parse go.mod for require block."""
    path = repo_dir / "go.mod"
    if not path.is_file():
        return None
    text = _read_text(path)
    if text is None:
        return None

    modules: list[str] = []
    in_require = False

    for line in text.splitlines():
        stripped = line.strip()
        if stripped.startswith("require ("):
            in_require = True
            continue
        if in_require:
            if stripped == ")":
                in_require = False
                continue
            parts = stripped.split()
            if parts and not parts[0].startswith("//"):
                modules.append(parts[0])
        elif stripped.startswith("require "):
            parts = stripped.split()
            if len(parts) >= 2:
                modules.append(parts[1])

    if not modules:
        return None
    return {"modules": modules, "source_file": "go.mod"}


# ---------------------------------------------------------------------------
# System dependency detection
# ---------------------------------------------------------------------------

def detect_system_deps(repo_dir: Path) -> list[str]:
    """Detect system-level dependencies from Dockerfile, README, setup.py."""
    found: set[str] = set()
    scan_files = [
        "Dockerfile", "docker-compose.yml", "docker-compose.yaml",
        "README.md", "README.rst", "readme.md",
        "setup.py", "Makefile", "CMakeLists.txt",
    ]

    for name in scan_files:
        path = repo_dir / name
        if not path.is_file():
            continue
        text = _read_text(path)
        if text is None:
            continue
        text_lower = text.lower()

        # Check for known system keywords
        for kw in SYSTEM_KEYWORDS:
            if kw in text_lower:
                found.add(kw)

        # Extract apt-get install packages from Dockerfiles
        if name.lower().startswith("dockerfile") or name.lower() == "dockerfile":
            for m in re.finditer(r"apt-get\s+install[^&\n]*", text):
                tokens = m.group(0).split()
                for tok in tokens:
                    tok = tok.strip().rstrip("\\")
                    if tok and not tok.startswith("-") and tok not in (
                        "apt-get", "install", "&&", "||", "RUN",
                    ):
                        found.add(tok)

    return sorted(found)


# ---------------------------------------------------------------------------
# ML framework detection
# ---------------------------------------------------------------------------

def detect_ml_frameworks(all_packages: list[str]) -> list[str]:
    """Identify ML frameworks from the combined package list."""
    frameworks: set[str] = set()
    for pkg in all_packages:
        pkg_lower = pkg.lower()
        if pkg_lower in ML_FRAMEWORK_MAP:
            frameworks.add(ML_FRAMEWORK_MAP[pkg_lower])
        # Handle tf-* prefix patterns
        elif pkg_lower.startswith("tf-") or pkg_lower.startswith("tensorflow-"):
            frameworks.add("tensorflow")
    return sorted(frameworks)


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main() -> None:
    parser = argparse.ArgumentParser(
        description="Extract and parse dependency files from a cloned repository",
    )
    parser.add_argument(
        "--repo-dir", required=True,
        help="Path to cloned repository directory",
    )
    parser.add_argument(
        "--output", required=True,
        help="Output JSON file path",
    )
    parser.add_argument(
        "--repo-id", default=None,
        help="Repository identifier (owner/name); inferred from dir name if omitted",
    )

    args = parser.parse_args()

    repo_dir = Path(args.repo_dir).resolve()
    if not repo_dir.is_dir():
        print(f"Error: repo directory not found: {repo_dir}", file=sys.stderr)
        sys.exit(1)

    # Infer repo_id from directory name if not provided
    repo_id: str = args.repo_id or ""
    if not repo_id:
        dir_name = repo_dir.name
        repo_id = dir_name.replace("_", "/", 1) if "_" in dir_name else dir_name

    print(f"Extracting dependencies from {repo_dir} ...", file=sys.stderr)

    # --- Python: merge results from all parsers ---
    python_info: dict | None = None
    for parser_fn in (parse_requirements_txt, parse_setup_py,
                      parse_pyproject_toml, parse_environment_yml):
        result = parser_fn(repo_dir)
        if result is not None:
            if python_info is None:
                python_info = result
            else:
                # Merge: append specs, combine source_file
                existing = set(python_info["requirements"])
                for spec in result["requirements"]:
                    if spec not in existing:
                        python_info["requirements"].append(spec)
                        existing.add(spec)
                python_info["source_file"] += f", {result['source_file']}"
    if python_info:
        print(f"  Python: {len(python_info['requirements'])} packages "
              f"from {python_info['source_file']}", file=sys.stderr)

    # --- Node ---
    node_info = parse_package_json(repo_dir)
    if node_info:
        n = len(node_info.get("dependencies", {})) + len(node_info.get("devDependencies", {}))
        print(f"  Node: {n} packages from {node_info['source_file']}", file=sys.stderr)

    # --- Rust ---
    rust_info = parse_cargo_toml(repo_dir)
    if rust_info:
        print(f"  Rust: {len(rust_info['dependencies'])} crates "
              f"from {rust_info['source_file']}", file=sys.stderr)

    # --- Go ---
    go_info = parse_go_mod(repo_dir)
    if go_info:
        print(f"  Go: {len(go_info['modules'])} modules "
              f"from {go_info['source_file']}", file=sys.stderr)

    # --- Collect all package names (flat, normalized) ---
    all_packages: set[str] = set()

    if python_info:
        for spec in python_info["requirements"]:
            name = _strip_version(spec)
            if name:
                all_packages.add(name)

    if node_info:
        for key in ("dependencies", "devDependencies"):
            for pkg_name in node_info.get(key, {}):
                all_packages.add(pkg_name.lower())

    if rust_info:
        for crate_name in rust_info["dependencies"]:
            all_packages.add(crate_name.lower())

    if go_info:
        for mod_path in go_info["modules"]:
            # Use last path segment as package name
            parts = mod_path.rstrip("/").split("/")
            all_packages.add(parts[-1].lower())

    all_packages_sorted = sorted(all_packages)

    # --- System dependencies ---
    system_deps = detect_system_deps(repo_dir)
    if system_deps:
        print(f"  System: {len(system_deps)} system dependencies detected",
              file=sys.stderr)

    # --- ML frameworks ---
    ml_frameworks = detect_ml_frameworks(all_packages_sorted)
    if ml_frameworks:
        print(f"  ML frameworks: {', '.join(ml_frameworks)}", file=sys.stderr)

    # --- Build output ---
    output: dict = {
        "repo_id": repo_id,
        "python": python_info,
        "node": node_info,
        "rust": rust_info,
        "go": go_info,
        "system": system_deps,
        "ml_frameworks": ml_frameworks,
        "all_packages": all_packages_sorted,
        "overlap_with": {},
    }

    output_path = Path(args.output)
    output_path.parent.mkdir(parents=True, exist_ok=True)
    with open(output_path, "w", encoding="utf-8") as f:
        json.dump(output, f, indent=2, ensure_ascii=False)
        f.write("\n")

    ecosystems = sum(1 for x in (python_info, node_info, rust_info, go_info)
                     if x is not None)
    print(
        f"Extracted {len(all_packages_sorted)} packages across "
        f"{ecosystems} ecosystem(s) -> {args.output}",
        file=sys.stderr,
    )


if __name__ == "__main__":
    main()