Data Scientist

#!/usr/bin/env python3
"""Track data science experiments: parameters, metrics, and notes in a local JSON log.

Maintains a structured experiment log file.  Each experiment records a name,
parameters, metrics, tags, and timestamps.  Supports listing, comparing, and
filtering experiments.

Usage:
    python experiment_tracker.py log --name "xgb_v2" --params '{"lr":0.1,"depth":6}' --metrics '{"f1":0.87,"auc":0.92}'
    python experiment_tracker.py list
    python experiment_tracker.py list --sort-by f1 --top 5
    python experiment_tracker.py compare --ids 1 3 5
    python experiment_tracker.py log --name "baseline" --params '{}' --metrics '{"f1":0.72}' --tags "baseline,prod" --json
"""

import argparse
import json
import os
import sys
from datetime import datetime


DEFAULT_LOG_FILE = "experiments.json"


def _load_experiments(path: str) -> list:
    if not os.path.exists(path):
        return []
    with open(path, "r") as f:
        return json.load(f)


def _save_experiments(experiments: list, path: str):
    with open(path, "w") as f:
        json.dump(experiments, f, indent=2)


def cmd_log(args):
    experiments = _load_experiments(args.log_file)

    try:
        params = json.loads(args.params) if args.params else {}
    except json.JSONDecodeError:
        print("Error: --params must be valid JSON.", file=sys.stderr)
        sys.exit(1)

    try:
        metrics = json.loads(args.metrics) if args.metrics else {}
    except json.JSONDecodeError:
        print("Error: --metrics must be valid JSON.", file=sys.stderr)
        sys.exit(1)

    tags = [t.strip() for t in args.tags.split(",")] if args.tags else []

    exp_id = len(experiments) + 1
    entry = {
        "id": exp_id,
        "name": args.name,
        "timestamp": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
        "parameters": params,
        "metrics": metrics,
        "tags": tags,
        "notes": args.notes or "",
    }

    experiments.append(entry)
    _save_experiments(experiments, args.log_file)

    if args.json:
        print(json.dumps(entry, indent=2))
    else:
        print(f"Experiment #{exp_id} logged: {args.name}")
        if params:
            print(f"  Params:  {json.dumps(params)}")
        if metrics:
            print(f"  Metrics: {json.dumps(metrics)}")
        if tags:
            print(f"  Tags:    {', '.join(tags)}")


def cmd_list(args):
    experiments = _load_experiments(args.log_file)
    if not experiments:
        print("No experiments logged yet.")
        return

    # Filter by tag
    if args.tag:
        experiments = [e for e in experiments if args.tag in e.get("tags", [])]

    # Sort
    if args.sort_by:
        metric_key = args.sort_by
        experiments = [e for e in experiments if metric_key in e.get("metrics", {})]
        experiments.sort(key=lambda e: e["metrics"][metric_key], reverse=True)

    # Top N
    if args.top:
        experiments = experiments[: args.top]

    if args.json:
        print(json.dumps(experiments, indent=2))
    else:
        print(f"{'ID':<5} {'Name':<25} {'Timestamp':<20} {'Metrics'}")
        print("-" * 80)
        for e in experiments:
            metrics_str = ", ".join(f"{k}={v}" for k, v in e.get("metrics", {}).items())
            print(f"{e['id']:<5} {e['name']:<25} {e['timestamp']:<20} {metrics_str}")


def cmd_compare(args):
    experiments = _load_experiments(args.log_file)
    if not experiments:
        print("No experiments logged yet.")
        return

    selected = [e for e in experiments if e["id"] in args.ids]
    if not selected:
        print(f"No experiments found with IDs: {args.ids}", file=sys.stderr)
        sys.exit(1)

    # Gather all metric keys
    all_metrics = set()
    for e in selected:
        all_metrics.update(e.get("metrics", {}).keys())
    all_metrics = sorted(all_metrics)

    # Gather all param keys
    all_params = set()
    for e in selected:
        all_params.update(e.get("parameters", {}).keys())
    all_params = sorted(all_params)

    if args.json:
        comparison = {
            "experiments": selected,
            "metric_keys": all_metrics,
            "param_keys": all_params,
        }
        # Find best per metric
        bests = {}
        for m in all_metrics:
            vals = [(e["id"], e["metrics"].get(m)) for e in selected if m in e.get("metrics", {})]
            if vals:
                best = max(vals, key=lambda x: x[1])
                bests[m] = {"experiment_id": best[0], "value": best[1]}
        comparison["best_per_metric"] = bests
        print(json.dumps(comparison, indent=2))
    else:
        # Header
        header = f"{'Metric':<20}"
        for e in selected:
            header += f" {'#' + str(e['id']) + ' ' + e['name']:<20}"
        print("Experiment Comparison")
        print("=" * 60)
        print(header)
        print("-" * len(header))

        # Parameters
        if all_params:
            print("\nParameters:")
            for p in all_params:
                row = f"  {p:<18}"
                for e in selected:
                    val = e.get("parameters", {}).get(p, "-")
                    row += f" {str(val):<20}"
                print(row)

        # Metrics
        if all_metrics:
            print("\nMetrics:")
            for m in all_metrics:
                values = []
                for e in selected:
                    val = e.get("metrics", {}).get(m)
                    values.append(val)
                best_val = max((v for v in values if v is not None), default=None)
                row = f"  {m:<18}"
                for val in values:
                    marker = " *" if val is not None and val == best_val else ""
                    row += f" {str(val if val is not None else '-'):<18}{marker}"
                print(row)
            print("\n  * = best value")


def main():
    parser = argparse.ArgumentParser(description="Track and compare data science experiments.")
    parser.add_argument("--log-file", default=DEFAULT_LOG_FILE, help=f"Path to experiment log (default: {DEFAULT_LOG_FILE})")
    parser.add_argument("--json", action="store_true", help="Output results as JSON")

    subparsers = parser.add_subparsers(dest="command", help="Command to run")

    # log
    log_parser = subparsers.add_parser("log", help="Log a new experiment")
    log_parser.add_argument("--name", required=True, help="Experiment name")
    log_parser.add_argument("--params", help="Parameters as JSON string")
    log_parser.add_argument("--metrics", help="Metrics as JSON string")
    log_parser.add_argument("--tags", help="Comma-separated tags")
    log_parser.add_argument("--notes", help="Free-text notes")

    # list
    list_parser = subparsers.add_parser("list", help="List experiments")
    list_parser.add_argument("--sort-by", help="Sort by metric name (descending)")
    list_parser.add_argument("--top", type=int, help="Show top N experiments")
    list_parser.add_argument("--tag", help="Filter by tag")

    # compare
    cmp_parser = subparsers.add_parser("compare", help="Compare experiments side-by-side")
    cmp_parser.add_argument("--ids", nargs="+", type=int, required=True, help="Experiment IDs to compare")

    args = parser.parse_args()

    if not args.command:
        parser.print_help()
        sys.exit(1)

    if args.command == "log":
        cmd_log(args)
    elif args.command == "list":
        cmd_list(args)
    elif args.command == "compare":
        cmd_compare(args)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Score and rank features for predictive modeling using standard-library-only methods.

Computes feature importance via variance, correlation with target, cardinality,
null rate, and information-theoretic measures.  Produces a ranked list with
composite scores to guide feature selection.

Usage:
    python feature_selector.py --file dataset.csv --target churn
    python feature_selector.py --file dataset.csv --target revenue --top 10 --json
    python feature_selector.py --file dataset.csv --target label --method all
"""

import argparse
import csv
import json
import math
import os
import sys
from collections import Counter


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

def _is_numeric(value: str) -> bool:
    try:
        float(value)
        return True
    except (ValueError, TypeError):
        return False


def _to_floats(values: list) -> list:
    return [float(v) for v in values if _is_numeric(str(v)) and str(v).strip()]


def _mean(vals: list) -> float:
    return sum(vals) / len(vals) if vals else 0.0


def _std(vals: list) -> float:
    if len(vals) < 2:
        return 0.0
    m = _mean(vals)
    return math.sqrt(sum((x - m) ** 2 for x in vals) / (len(vals) - 1))


def _correlation(x: list, y: list) -> float:
    """Pearson correlation coefficient."""
    n = min(len(x), len(y))
    if n < 3:
        return 0.0
    mx, my = _mean(x[:n]), _mean(y[:n])
    sx, sy = _std(x[:n]), _std(y[:n])
    if sx == 0 or sy == 0:
        return 0.0
    cov = sum((x[i] - mx) * (y[i] - my) for i in range(n)) / (n - 1)
    return cov / (sx * sy)


def _entropy(values: list) -> float:
    """Shannon entropy in bits."""
    counts = Counter(values)
    total = len(values)
    if total == 0:
        return 0.0
    ent = 0.0
    for c in counts.values():
        p = c / total
        if p > 0:
            ent -= p * math.log2(p)
    return ent


def _mutual_information(feature_vals: list, target_vals: list) -> float:
    """Approximate mutual information using discrete bins."""
    n = len(feature_vals)
    if n == 0:
        return 0.0

    # Bin numeric features into 10 bins
    joint = Counter(zip(feature_vals, target_vals))
    f_counts = Counter(feature_vals)
    t_counts = Counter(target_vals)

    mi = 0.0
    for (f, t), joint_count in joint.items():
        p_joint = joint_count / n
        p_f = f_counts[f] / n
        p_t = t_counts[t] / n
        if p_joint > 0 and p_f > 0 and p_t > 0:
            mi += p_joint * math.log2(p_joint / (p_f * p_t))
    return mi


def _bin_numeric(values: list, bins: int = 10) -> list:
    """Bin numeric values into discrete categories."""
    floats = _to_floats(values)
    if not floats:
        return values  # non-numeric, return as-is
    mn, mx = min(floats), max(floats)
    if mn == mx:
        return ["bin_0"] * len(values)
    step = (mx - mn) / bins
    result = []
    for v in values:
        if _is_numeric(str(v)):
            idx = min(int((float(v) - mn) / step), bins - 1)
            result.append(f"bin_{idx}")
        else:
            result.append("NA")
    return result


# ---------------------------------------------------------------------------
# Feature scoring
# ---------------------------------------------------------------------------

def score_features(data: list, target_col: str, method: str = "all") -> list:
    if not data:
        return []

    columns = [c for c in data[0].keys() if c != target_col]
    target_values = [row.get(target_col, "") for row in data]
    target_numeric = _to_floats(target_values)
    target_is_numeric = len(target_numeric) > len(target_values) * 0.8

    results = []

    for col in columns:
        raw_values = [row.get(col, "") for row in data]
        total = len(raw_values)
        non_null = [v for v in raw_values if v is not None and str(v).strip()]
        null_count = total - len(non_null)
        null_rate = null_count / total if total > 0 else 0

        score_components = {}

        # 1. Null rate score (lower nulls = better)
        null_score = max(0, 1.0 - null_rate)
        score_components["null_completeness"] = round(null_score, 4)

        # 2. Variance / cardinality score
        numeric_vals = _to_floats(raw_values)
        is_numeric = len(numeric_vals) > len(non_null) * 0.8 if non_null else False

        if is_numeric and numeric_vals:
            std = _std(numeric_vals)
            mean_abs = abs(_mean(numeric_vals))
            cv = std / mean_abs if mean_abs > 0 else std
            var_score = min(1.0, cv)  # Cap at 1.0
            score_components["variance"] = round(var_score, 4)
        else:
            unique = len(set(str(v) for v in non_null))
            card_ratio = unique / len(non_null) if non_null else 0
            # Penalize very low (constant) and very high (unique ID) cardinality
            if card_ratio > 0.95 and len(non_null) > 50:
                var_score = 0.1  # Likely a unique ID
            elif card_ratio < 0.01:
                var_score = 0.1  # Nearly constant
            else:
                var_score = min(1.0, card_ratio * 5)
            score_components["cardinality"] = round(var_score, 4)

        # 3. Correlation with target (numeric features vs numeric target)
        if is_numeric and target_is_numeric and method in ("all", "correlation"):
            # Align lengths
            paired = []
            for row in data:
                fv, tv = row.get(col, ""), row.get(target_col, "")
                if _is_numeric(str(fv)) and _is_numeric(str(tv)):
                    paired.append((float(fv), float(tv)))
            if len(paired) > 5:
                fx, fy = zip(*paired)
                corr = abs(_correlation(list(fx), list(fy)))
                score_components["abs_correlation"] = round(corr, 4)

        # 4. Mutual information (works for both numeric and categorical)
        if method in ("all", "mutual_info"):
            feat_binned = _bin_numeric(raw_values) if is_numeric else [str(v) for v in raw_values]
            tgt_binned = _bin_numeric(target_values) if target_is_numeric else [str(v) for v in target_values]
            mi = _mutual_information(feat_binned, tgt_binned)
            # Normalize by target entropy
            t_ent = _entropy(tgt_binned)
            mi_normalized = mi / t_ent if t_ent > 0 else 0
            score_components["mutual_info"] = round(min(1.0, mi_normalized), 4)

        # Composite score (weighted average)
        weights = {
            "null_completeness": 0.15,
            "variance": 0.20,
            "cardinality": 0.20,
            "abs_correlation": 0.35,
            "mutual_info": 0.30,
        }
        total_weight = 0
        weighted_sum = 0
        for key, value in score_components.items():
            w = weights.get(key, 0.1)
            weighted_sum += value * w
            total_weight += w
        composite = weighted_sum / total_weight if total_weight > 0 else 0

        results.append({
            "feature": col,
            "composite_score": round(composite, 4),
            "data_type": "numeric" if is_numeric else "categorical",
            "null_rate": round(null_rate, 4),
            "unique_values": len(set(str(v) for v in non_null)),
            "scores": score_components,
        })

    results.sort(key=lambda x: x["composite_score"], reverse=True)
    return results


def main():
    parser = argparse.ArgumentParser(description="Score and rank features for predictive modeling.")
    parser.add_argument("--file", required=True, help="Path to CSV data file")
    parser.add_argument("--target", required=True, help="Target column name")
    parser.add_argument("--top", type=int, help="Show only top N features")
    parser.add_argument("--method", choices=["all", "correlation", "mutual_info"], default="all", help="Scoring method (default: all)")
    parser.add_argument("--json", action="store_true", help="Output as JSON")
    args = parser.parse_args()

    if not os.path.exists(args.file):
        print(f"Error: File not found: {args.file}", file=sys.stderr)
        sys.exit(1)

    with open(args.file, "r", newline="") as f:
        data = list(csv.DictReader(f))

    if not data:
        print("Error: No data rows found.", file=sys.stderr)
        sys.exit(1)

    if args.target not in data[0]:
        print(f"Error: Target column '{args.target}' not found. Available: {', '.join(data[0].keys())}", file=sys.stderr)
        sys.exit(1)

    results = score_features(data, args.target, args.method)
    if args.top:
        results = results[: args.top]

    if args.json:
        print(json.dumps({"target": args.target, "method": args.method, "features": results}, indent=2))
    else:
        print(f"Feature Importance Ranking (target: {args.target})")
        print("=" * 70)
        print(f"{'Rank':<6} {'Feature':<25} {'Score':<8} {'Type':<12} {'Nulls':<8} {'Unique'}")
        print("-" * 70)
        for i, r in enumerate(results, 1):
            print(f"{i:<6} {r['feature']:<25} {r['composite_score']:<8} {r['data_type']:<12} {r['null_rate']:<8} {r['unique_values']}")
        print()
        if results:
            print("Top feature details:")
            top = results[0]
            for k, v in top["scores"].items():
                print(f"  {k}: {v}")

    sys.exit(0)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Run statistical hypothesis tests on CSV data using only the standard library.

Supports two-sample t-test (Welch's), paired t-test, chi-square test of
independence, and proportion z-test.  Computes test statistics, p-values,
confidence intervals, and effect sizes.

Usage:
    python hypothesis_tester.py ttest --file data.csv --col-a group_a --col-b group_b
    python hypothesis_tester.py proportion --successes-a 120 --trials-a 1000 --successes-b 145 --trials-b 1000
    python hypothesis_tester.py chi-square --file contingency.csv
    python hypothesis_tester.py ttest --file data.csv --col-a before --col-b after --paired --json
"""

import argparse
import csv
import json
import math
import os
import sys


# ---------------------------------------------------------------------------
# Statistical helpers (standard library only)
# ---------------------------------------------------------------------------

def _mean(values: list) -> float:
    return sum(values) / len(values)


def _variance(values: list, ddof: int = 1) -> float:
    m = _mean(values)
    return sum((x - m) ** 2 for x in values) / (len(values) - ddof)


def _std(values: list, ddof: int = 1) -> float:
    return math.sqrt(_variance(values, ddof))


def _normal_cdf(x: float) -> float:
    """Approximate standard normal CDF using Abramowitz & Stegun."""
    sign = 1 if x >= 0 else -1
    x = abs(x)
    t = 1.0 / (1.0 + 0.2316419 * x)
    d = 0.3989422804014327  # 1/sqrt(2*pi)
    poly = t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))))
    cdf = 1.0 - d * math.exp(-0.5 * x * x) * poly
    return 0.5 + sign * (cdf - 0.5)


def _t_cdf(t_val: float, df: float) -> float:
    """Approximate t-distribution CDF using normal approximation for df > 30,
    otherwise use a simple beta-function based approximation."""
    if df > 30:
        return _normal_cdf(t_val)
    # Use regularized incomplete beta function approximation
    x = df / (df + t_val * t_val)
    # Simple approximation via normal with correction
    g = math.lgamma((df + 1) / 2) - math.lgamma(df / 2)
    correction = math.exp(g) / math.sqrt(df * math.pi)
    # For small df, use a Cornish-Fisher-style approximation
    z = t_val * (1 - 1 / (4 * df))
    return _normal_cdf(z)


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------

def welch_ttest(a: list, b: list, alpha: float = 0.05) -> dict:
    """Two-sample Welch's t-test (unequal variances)."""
    n_a, n_b = len(a), len(b)
    mean_a, mean_b = _mean(a), _mean(b)
    var_a, var_b = _variance(a), _variance(b)
    se = math.sqrt(var_a / n_a + var_b / n_b)
    if se == 0:
        return {"error": "Standard error is zero; groups may be identical."}
    t_stat = (mean_a - mean_b) / se

    # Welch-Satterthwaite degrees of freedom
    num = (var_a / n_a + var_b / n_b) ** 2
    denom = (var_a / n_a) ** 2 / (n_a - 1) + (var_b / n_b) ** 2 / (n_b - 1)
    df = num / denom if denom > 0 else n_a + n_b - 2

    p_value = 2 * (1 - _t_cdf(abs(t_stat), df))

    # Cohen's d
    pooled_std = math.sqrt((var_a + var_b) / 2)
    cohens_d = (mean_a - mean_b) / pooled_std if pooled_std > 0 else 0

    # 95% CI for difference
    z = 1.96
    ci_low = (mean_a - mean_b) - z * se
    ci_high = (mean_a - mean_b) + z * se

    effect_label = "negligible"
    d_abs = abs(cohens_d)
    if d_abs >= 0.8:
        effect_label = "large"
    elif d_abs >= 0.5:
        effect_label = "medium"
    elif d_abs >= 0.2:
        effect_label = "small"

    return {
        "test": "welch_ttest",
        "group_a": {"n": n_a, "mean": round(mean_a, 4), "std": round(_std(a), 4)},
        "group_b": {"n": n_b, "mean": round(mean_b, 4), "std": round(_std(b), 4)},
        "t_statistic": round(t_stat, 4),
        "degrees_of_freedom": round(df, 2),
        "p_value": round(p_value, 6),
        "significant": p_value < alpha,
        "alpha": alpha,
        "cohens_d": round(cohens_d, 4),
        "effect_size": effect_label,
        "ci_95": [round(ci_low, 4), round(ci_high, 4)],
        "interpretation": f"The difference is {'statistically significant' if p_value < alpha else 'not statistically significant'} (p={round(p_value, 4)}) with a {effect_label} effect size (d={round(cohens_d, 2)}).",
    }


def paired_ttest(a: list, b: list, alpha: float = 0.05) -> dict:
    """Paired t-test for dependent samples."""
    if len(a) != len(b):
        return {"error": "Paired t-test requires equal-length samples."}
    diffs = [x - y for x, y in zip(a, b)]
    n = len(diffs)
    mean_d = _mean(diffs)
    std_d = _std(diffs)
    se = std_d / math.sqrt(n) if n > 0 else 0
    if se == 0:
        return {"error": "Standard error is zero."}
    t_stat = mean_d / se
    df = n - 1
    p_value = 2 * (1 - _t_cdf(abs(t_stat), df))
    cohens_d = mean_d / std_d if std_d > 0 else 0

    return {
        "test": "paired_ttest",
        "n_pairs": n,
        "mean_difference": round(mean_d, 4),
        "std_difference": round(std_d, 4),
        "t_statistic": round(t_stat, 4),
        "degrees_of_freedom": df,
        "p_value": round(p_value, 6),
        "significant": p_value < alpha,
        "alpha": alpha,
        "cohens_d": round(cohens_d, 4),
    }


def proportion_ztest(succ_a: int, n_a: int, succ_b: int, n_b: int, alpha: float = 0.05) -> dict:
    """Two-proportion z-test."""
    p_a = succ_a / n_a
    p_b = succ_b / n_b
    p_pool = (succ_a + succ_b) / (n_a + n_b)
    se = math.sqrt(p_pool * (1 - p_pool) * (1 / n_a + 1 / n_b))
    if se == 0:
        return {"error": "Standard error is zero."}
    z = (p_a - p_b) / se
    p_value = 2 * (1 - _normal_cdf(abs(z)))
    lift = (p_b - p_a) / p_a if p_a > 0 else 0

    return {
        "test": "proportion_ztest",
        "group_a": {"successes": succ_a, "trials": n_a, "rate": round(p_a, 4)},
        "group_b": {"successes": succ_b, "trials": n_b, "rate": round(p_b, 4)},
        "z_statistic": round(z, 4),
        "p_value": round(p_value, 6),
        "significant": p_value < alpha,
        "alpha": alpha,
        "lift": round(lift, 4),
        "interpretation": f"Group B rate ({round(p_b, 4)}) vs Group A ({round(p_a, 4)}): {'+' if lift >= 0 else ''}{round(lift * 100, 1)}% lift. {'Significant' if p_value < alpha else 'Not significant'} (p={round(p_value, 4)}).",
    }


def chi_square_test(table: list, alpha: float = 0.05) -> dict:
    """Chi-square test of independence from a contingency table (list of lists)."""
    rows = len(table)
    cols = len(table[0]) if rows > 0 else 0
    row_totals = [sum(r) for r in table]
    col_totals = [sum(table[r][c] for r in range(rows)) for c in range(cols)]
    grand_total = sum(row_totals)

    if grand_total == 0:
        return {"error": "Table totals are zero."}

    chi2 = 0.0
    for r in range(rows):
        for c in range(cols):
            expected = row_totals[r] * col_totals[c] / grand_total
            if expected > 0:
                chi2 += (table[r][c] - expected) ** 2 / expected

    df = (rows - 1) * (cols - 1)
    # Approximate p-value using Wilson-Hilferty normal approximation
    if df > 0:
        z = (chi2 / df) ** (1 / 3) - (1 - 2 / (9 * df))
        z /= math.sqrt(2 / (9 * df)) if df > 0 else 1
        p_value = 1 - _normal_cdf(z)
    else:
        p_value = 1.0

    # Cramer's V
    min_dim = min(rows, cols) - 1
    cramers_v = math.sqrt(chi2 / (grand_total * min_dim)) if grand_total > 0 and min_dim > 0 else 0

    return {
        "test": "chi_square",
        "chi2_statistic": round(chi2, 4),
        "degrees_of_freedom": df,
        "p_value": round(max(p_value, 0), 6),
        "significant": p_value < alpha,
        "alpha": alpha,
        "cramers_v": round(cramers_v, 4),
    }


# ---------------------------------------------------------------------------
# Data loading
# ---------------------------------------------------------------------------

def _load_columns(file_path: str, col_a: str, col_b: str) -> tuple:
    with open(file_path, "r", newline="") as f:
        reader = csv.DictReader(f)
        a_vals, b_vals = [], []
        for row in reader:
            va, vb = row.get(col_a), row.get(col_b)
            if va is not None and va.strip():
                try:
                    a_vals.append(float(va))
                except ValueError:
                    pass
            if vb is not None and vb.strip():
                try:
                    b_vals.append(float(vb))
                except ValueError:
                    pass
    return a_vals, b_vals


def _load_contingency(file_path: str) -> list:
    with open(file_path, "r", newline="") as f:
        reader = csv.reader(f)
        next(reader, None)  # skip header
        table = []
        for row in reader:
            table.append([int(float(v)) for v in row if v.strip()])
    return table


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():
    parser = argparse.ArgumentParser(description="Run statistical hypothesis tests on data.")
    parser.add_argument("--json", action="store_true", help="Output results as JSON")
    parser.add_argument("--alpha", type=float, default=0.05, help="Significance level (default: 0.05)")

    sub = parser.add_subparsers(dest="test_type", help="Test to run")

    # ttest
    tt = sub.add_parser("ttest", help="Two-sample or paired t-test")
    tt.add_argument("--file", required=True, help="CSV file with data columns")
    tt.add_argument("--col-a", required=True, help="Column name for group A")
    tt.add_argument("--col-b", required=True, help="Column name for group B")
    tt.add_argument("--paired", action="store_true", help="Run paired t-test instead of independent")

    # proportion
    pr = sub.add_parser("proportion", help="Two-proportion z-test")
    pr.add_argument("--successes-a", type=int, required=True)
    pr.add_argument("--trials-a", type=int, required=True)
    pr.add_argument("--successes-b", type=int, required=True)
    pr.add_argument("--trials-b", type=int, required=True)

    # chi-square
    ch = sub.add_parser("chi-square", help="Chi-square test of independence")
    ch.add_argument("--file", required=True, help="CSV file with contingency table (numeric cells, first row is header)")

    args = parser.parse_args()
    if not args.test_type:
        parser.print_help()
        sys.exit(1)

    result = {}

    if args.test_type == "ttest":
        if not os.path.exists(args.file):
            print(f"Error: File not found: {args.file}", file=sys.stderr)
            sys.exit(1)
        a, b = _load_columns(args.file, args.col_a, args.col_b)
        if len(a) < 2 or len(b) < 2:
            print("Error: Each group needs at least 2 values.", file=sys.stderr)
            sys.exit(1)
        if args.paired:
            result = paired_ttest(a, b, args.alpha)
        else:
            result = welch_ttest(a, b, args.alpha)

    elif args.test_type == "proportion":
        result = proportion_ztest(args.successes_a, args.trials_a, args.successes_b, args.trials_b, args.alpha)

    elif args.test_type == "chi-square":
        if not os.path.exists(args.file):
            print(f"Error: File not found: {args.file}", file=sys.stderr)
            sys.exit(1)
        table = _load_contingency(args.file)
        if len(table) < 2 or any(len(r) < 2 for r in table):
            print("Error: Contingency table must be at least 2x2.", file=sys.stderr)
            sys.exit(1)
        result = chi_square_test(table, args.alpha)

    if "error" in result:
        print(f"Error: {result['error']}", file=sys.stderr)
        sys.exit(1)

    if args.json:
        print(json.dumps(result, indent=2))
    else:
        print(f"Hypothesis Test: {result.get('test', args.test_type)}")
        print("=" * 50)
        for k, v in result.items():
            if k == "test":
                continue
            if isinstance(v, dict):
                print(f"  {k}:")
                for kk, vv in v.items():
                    print(f"    {kk}: {vv}")
            elif isinstance(v, list):
                print(f"  {k}: [{', '.join(str(x) for x in v)}]")
            else:
                print(f"  {k}: {v}")

    sys.exit(0)


if __name__ == "__main__":
    main()