Notebook Ml Architect

"""
Evaluation Block for ML Notebooks

Comprehensive evaluation utilities for classification and regression tasks.
Copy relevant functions to your notebook's evaluation section.

Contents:
    - evaluate_classifier: Full classification metrics and visualizations
    - evaluate_regressor: Full regression metrics and visualizations
    - plot_learning_curves: Training/validation curves
    - plot_feature_importance: Feature importance visualization
"""

from typing import Any

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd


# =============================================================================
# Classification Evaluation
# =============================================================================


def evaluate_classifier(
    y_true: np.ndarray,
    y_pred: np.ndarray,
    y_prob: np.ndarray | None = None,
    class_names: list[str] | None = None,
    figsize: tuple[int, int] = (12, 5),
) -> dict[str, float]:
    """
    Comprehensive classification evaluation with metrics and visualizations.

    Args:
        y_true: True labels
        y_pred: Predicted labels
        y_prob: Predicted probabilities (for ROC/AUC)
        class_names: Names for each class
        figsize: Figure size for plots

    Returns:
        Dictionary of computed metrics
    """
    from sklearn.metrics import (
        accuracy_score,
        classification_report,
        confusion_matrix,
        f1_score,
        precision_score,
        recall_score,
        roc_auc_score,
        roc_curve,
    )

    # Compute metrics
    metrics = {
        "accuracy": accuracy_score(y_true, y_pred),
        "precision_weighted": precision_score(
            y_true, y_pred, average="weighted", zero_division=0
        ),
        "recall_weighted": recall_score(
            y_true, y_pred, average="weighted", zero_division=0
        ),
        "f1_weighted": f1_score(y_true, y_pred, average="weighted", zero_division=0),
        "precision_macro": precision_score(
            y_true, y_pred, average="macro", zero_division=0
        ),
        "recall_macro": recall_score(y_true, y_pred, average="macro", zero_division=0),
        "f1_macro": f1_score(y_true, y_pred, average="macro", zero_division=0),
    }

    # ROC AUC for binary classification
    if y_prob is not None:
        try:
            if len(np.unique(y_true)) == 2:
                # Binary classification
                if y_prob.ndim == 2:
                    y_prob_pos = y_prob[:, 1]
                else:
                    y_prob_pos = y_prob
                metrics["roc_auc"] = roc_auc_score(y_true, y_prob_pos)
            else:
                # Multi-class
                metrics["roc_auc_ovr"] = roc_auc_score(
                    y_true, y_prob, multi_class="ovr"
                )
        except ValueError:
            pass

    # Print classification report
    print("Classification Report:")
    print("=" * 60)
    print(
        classification_report(y_true, y_pred, target_names=class_names, zero_division=0)
    )

    # Print summary metrics
    print("\nSummary Metrics:")
    print("-" * 40)
    for name, value in metrics.items():
        print(f"  {name}: {value:.4f}")

    # Create visualizations
    fig, axes = plt.subplots(
        1,
        2 if y_prob is not None and len(np.unique(y_true)) == 2 else 1,
        figsize=figsize,
    )

    if not isinstance(axes, np.ndarray):
        axes = [axes]

    # Confusion Matrix
    import seaborn as sns

    cm = confusion_matrix(y_true, y_pred)
    sns.heatmap(
        cm,
        annot=True,
        fmt="d",
        cmap="Blues",
        xticklabels=class_names,
        yticklabels=class_names,
        ax=axes[0],
    )
    axes[0].set_xlabel("Predicted")
    axes[0].set_ylabel("Actual")
    axes[0].set_title("Confusion Matrix")

    # ROC Curve (binary only)
    if y_prob is not None and len(np.unique(y_true)) == 2 and len(axes) > 1:
        if y_prob.ndim == 2:
            y_prob_pos = y_prob[:, 1]
        else:
            y_prob_pos = y_prob

        fpr, tpr, _ = roc_curve(y_true, y_prob_pos)
        axes[1].plot(fpr, tpr, label=f"AUC = {metrics.get('roc_auc', 0):.4f}")
        axes[1].plot([0, 1], [0, 1], "k--", label="Random")
        axes[1].set_xlabel("False Positive Rate")
        axes[1].set_ylabel("True Positive Rate")
        axes[1].set_title("ROC Curve")
        axes[1].legend()

    plt.tight_layout()
    plt.show()

    return metrics


# =============================================================================
# Regression Evaluation
# =============================================================================


def evaluate_regressor(
    y_true: np.ndarray,
    y_pred: np.ndarray,
    figsize: tuple[int, int] = (12, 5),
) -> dict[str, float]:
    """
    Comprehensive regression evaluation with metrics and visualizations.

    Args:
        y_true: True values
        y_pred: Predicted values
        figsize: Figure size for plots

    Returns:
        Dictionary of computed metrics
    """
    from sklearn.metrics import (
        mean_absolute_error,
        mean_absolute_percentage_error,
        mean_squared_error,
        r2_score,
    )

    # Compute metrics
    metrics = {
        "r2": r2_score(y_true, y_pred),
        "mae": mean_absolute_error(y_true, y_pred),
        "mse": mean_squared_error(y_true, y_pred),
        "rmse": np.sqrt(mean_squared_error(y_true, y_pred)),
        "mape": mean_absolute_percentage_error(y_true, y_pred) * 100,
    }

    # Print summary
    print("Regression Metrics:")
    print("=" * 60)
    for name, value in metrics.items():
        print(f"  {name}: {value:.4f}")

    # Create visualizations
    fig, axes = plt.subplots(1, 2, figsize=figsize)

    # Actual vs Predicted
    axes[0].scatter(y_true, y_pred, alpha=0.5)
    min_val = min(y_true.min(), y_pred.min())
    max_val = max(y_true.max(), y_pred.max())
    axes[0].plot([min_val, max_val], [min_val, max_val], "r--", label="Perfect")
    axes[0].set_xlabel("Actual")
    axes[0].set_ylabel("Predicted")
    axes[0].set_title(f"Actual vs Predicted (R² = {metrics['r2']:.4f})")
    axes[0].legend()

    # Residuals
    residuals = y_true - y_pred
    axes[1].hist(residuals, bins=30, edgecolor="black", alpha=0.7)
    axes[1].axvline(x=0, color="r", linestyle="--")
    axes[1].set_xlabel("Residual")
    axes[1].set_ylabel("Frequency")
    axes[1].set_title(f"Residual Distribution (MAE = {metrics['mae']:.4f})")

    plt.tight_layout()
    plt.show()

    return metrics


# =============================================================================
# Learning Curves
# =============================================================================


def plot_learning_curves(
    train_scores: list[float],
    val_scores: list[float],
    train_losses: list[float] | None = None,
    val_losses: list[float] | None = None,
    metric_name: str = "Accuracy",
    figsize: tuple[int, int] = (12, 5),
) -> None:
    """
    Plot training and validation learning curves.

    Args:
        train_scores: Training scores per epoch
        val_scores: Validation scores per epoch
        train_losses: Training losses per epoch (optional)
        val_losses: Validation losses per epoch (optional)
        metric_name: Name of the metric being plotted
        figsize: Figure size
    """
    epochs = range(1, len(train_scores) + 1)

    if train_losses is not None and val_losses is not None:
        fig, axes = plt.subplots(1, 2, figsize=figsize)

        # Scores
        axes[0].plot(epochs, train_scores, "b-", label="Training")
        axes[0].plot(epochs, val_scores, "r-", label="Validation")
        axes[0].set_xlabel("Epoch")
        axes[0].set_ylabel(metric_name)
        axes[0].set_title(f"{metric_name} vs Epoch")
        axes[0].legend()
        axes[0].grid(True, alpha=0.3)

        # Losses
        axes[1].plot(epochs, train_losses, "b-", label="Training")
        axes[1].plot(epochs, val_losses, "r-", label="Validation")
        axes[1].set_xlabel("Epoch")
        axes[1].set_ylabel("Loss")
        axes[1].set_title("Loss vs Epoch")
        axes[1].legend()
        axes[1].grid(True, alpha=0.3)

    else:
        fig, ax = plt.subplots(figsize=(figsize[0] // 2, figsize[1]))
        ax.plot(epochs, train_scores, "b-", label="Training")
        ax.plot(epochs, val_scores, "r-", label="Validation")
        ax.set_xlabel("Epoch")
        ax.set_ylabel(metric_name)
        ax.set_title(f"{metric_name} vs Epoch")
        ax.legend()
        ax.grid(True, alpha=0.3)

    plt.tight_layout()
    plt.show()


# =============================================================================
# Feature Importance
# =============================================================================


def plot_feature_importance(
    feature_names: list[str],
    importances: np.ndarray,
    top_n: int = 20,
    figsize: tuple[int, int] = (10, 8),
) -> pd.DataFrame:
    """
    Plot feature importance.

    Args:
        feature_names: List of feature names
        importances: Array of importance values
        top_n: Number of top features to show
        figsize: Figure size

    Returns:
        DataFrame with feature importances
    """
    import seaborn as sns

    # Create DataFrame
    importance_df = pd.DataFrame(
        {"feature": feature_names, "importance": importances}
    ).sort_values("importance", ascending=False)

    # Plot
    plt.figure(figsize=figsize)
    sns.barplot(
        data=importance_df.head(top_n),
        x="importance",
        y="feature",
        palette="viridis",
    )
    plt.xlabel("Importance")
    plt.ylabel("Feature")
    plt.title(f"Top {top_n} Feature Importances")
    plt.tight_layout()
    plt.show()

    return importance_df


# =============================================================================
# Cross-Validation Summary
# =============================================================================


def print_cv_summary(cv_results: dict[str, Any]) -> pd.DataFrame:
    """
    Print cross-validation results summary.

    Args:
        cv_results: Results from cross_validate() or similar

    Returns:
        Summary DataFrame
    """
    summary = []
    for key, values in cv_results.items():
        if key.startswith("test_") or key.startswith("train_"):
            summary.append(
                {
                    "metric": key,
                    "mean": np.mean(values),
                    "std": np.std(values),
                    "min": np.min(values),
                    "max": np.max(values),
                }
            )

    df = pd.DataFrame(summary)
    print("Cross-Validation Results:")
    print("=" * 60)
    print(df.to_string(index=False))
    return df

"""
Reproducibility Header for ML Notebooks

Copy this code to the first code cell of your notebook to ensure reproducible results.
Handles seeding for: random, numpy, torch, tensorflow.

Usage:
    1. Copy this entire file content to your first code cell
    2. Adjust SEED value as needed
    3. Add any additional library version prints
"""

import os
import random
import sys
from datetime import datetime

import numpy as np

# =============================================================================
# Configuration
# =============================================================================

SEED = 42
PROJECT_ROOT = os.path.dirname(os.path.abspath("__file__"))


# =============================================================================
# Random Seed Setting
# =============================================================================


def set_seeds(seed: int = SEED) -> None:
    """
    Set random seeds for reproducibility across all common ML libraries.

    Args:
        seed: Random seed value (default: 42)
    """
    # Python's random module
    random.seed(seed)

    # NumPy
    np.random.seed(seed)

    # Python hash seed (for dict ordering in Python 3.7+)
    os.environ["PYTHONHASHSEED"] = str(seed)

    # PyTorch (if available)
    try:
        import torch

        torch.manual_seed(seed)
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)  # For multi-GPU

        # Deterministic operations (may impact performance)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

        # For PyTorch 1.8+
        if hasattr(torch, "use_deterministic_algorithms"):
            try:
                torch.use_deterministic_algorithms(True)
            except Exception:
                pass  # Some ops don't have deterministic implementations
    except ImportError:
        pass

    # TensorFlow (if available)
    try:
        import tensorflow as tf

        tf.random.set_seed(seed)

        # Disable GPU non-determinism
        os.environ["TF_DETERMINISTIC_OPS"] = "1"
        os.environ["TF_CUDNN_DETERMINISTIC"] = "1"
    except ImportError:
        pass

    print(f"Random seeds set to: {seed}")


# =============================================================================
# Environment Capture
# =============================================================================


def print_environment() -> dict:
    """
    Print and return environment information for reproducibility.

    Returns:
        Dictionary with environment details
    """
    env_info = {
        "timestamp": datetime.now().isoformat(),
        "python_version": sys.version,
        "platform": sys.platform,
        "seed": SEED,
        "packages": {},
    }

    print("=" * 60)
    print("Environment Information")
    print("=" * 60)
    print(f"Timestamp: {env_info['timestamp']}")
    print(f"Python: {env_info['python_version']}")
    print(f"Platform: {env_info['platform']}")
    print(f"Seed: {env_info['seed']}")
    print()

    # Core packages
    packages = [
        "numpy",
        "pandas",
        "sklearn",
        "scipy",
        "matplotlib",
        "seaborn",
        "torch",
        "tensorflow",
        "keras",
        "xgboost",
        "lightgbm",
        "catboost",
    ]

    print("Package Versions:")
    for pkg_name in packages:
        try:
            pkg = __import__(pkg_name)
            version = getattr(pkg, "__version__", "unknown")
            env_info["packages"][pkg_name] = version
            print(f"  {pkg_name}: {version}")
        except ImportError:
            pass

    print("=" * 60)
    return env_info


# =============================================================================
# Initialize
# =============================================================================

# Set seeds immediately
set_seeds(SEED)

# Print environment info
ENV_INFO = print_environment()


# =============================================================================
# Optional: GPU Information
# =============================================================================


def print_gpu_info() -> None:
    """Print GPU information if available."""
    print("\nGPU Information:")

    # PyTorch GPU
    try:
        import torch

        if torch.cuda.is_available():
            print(f"  PyTorch CUDA: {torch.version.cuda}")
            print(f"  GPU Count: {torch.cuda.device_count()}")
            for i in range(torch.cuda.device_count()):
                print(f"  GPU {i}: {torch.cuda.get_device_name(i)}")
        else:
            print("  PyTorch: No CUDA available")
    except ImportError:
        pass

    # TensorFlow GPU
    try:
        import tensorflow as tf

        gpus = tf.config.list_physical_devices("GPU")
        if gpus:
            print(f"  TensorFlow GPUs: {len(gpus)}")
            for gpu in gpus:
                print(f"    {gpu.name}")
        else:
            print("  TensorFlow: No GPUs available")
    except ImportError:
        pass


# Uncomment to print GPU info:
# print_gpu_info()

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification Model\n",
    "\n",
    "**Problem**: [TODO: Problem description]\n",
    "**Dataset**: [TODO: Dataset name]\n",
    "**Author**: [TODO: Your name]\n",
    "**Date**: [TODO: Date]\n",
    "\n",
    "## Objective\n",
    "[TODO: Describe the classification objective]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": ["parameters"]
   },
   "outputs": [],
   "source": [
    "# Parameters (for papermill parameterization)\n",
    "SEED = 42\n",
    "TEST_SIZE = 0.2\n",
    "TARGET_COL = 'target'\n",
    "DATA_PATH = 'data/raw/dataset.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard library\n",
    "import os\n",
    "import sys\n",
    "import random\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "\n",
    "# Data handling\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# ML\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score, precision_score, recall_score, f1_score,\n",
    "    confusion_matrix, classification_report, roc_auc_score, roc_curve\n",
    ")\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.dummy import DummyClassifier\n",
    "\n",
    "# Set seeds for reproducibility\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "os.environ['PYTHONHASHSEED'] = str(SEED)\n",
    "\n",
    "# Display settings\n",
    "pd.set_option('display.max_columns', 50)\n",
    "plt.style.use('seaborn-v0_8-whitegrid')\n",
    "%matplotlib inline\n",
    "\n",
    "# Environment info\n",
    "print(f\"Timestamp: {datetime.now().isoformat()}\")\n",
    "print(f\"Python: {sys.version}\")\n",
    "print(f\"NumPy: {np.__version__}\")\n",
    "print(f\"Pandas: {pd.__version__}\")\n",
    "print(f\"Seed: {SEED}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(DATA_PATH)\n",
    "print(f\"Loaded {len(df):,} rows, {len(df.columns)} columns\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Quick EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Basic info\n",
    "print(df.info())\n",
    "print(\"\\nMissing values:\")\n",
    "print(df.isnull().sum()[df.isnull().sum() > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Target distribution\n",
    "print(f\"\\nTarget distribution ({TARGET_COL}):\")\n",
    "print(df[TARGET_COL].value_counts())\n",
    "print(f\"\\nClass balance:\")\n",
    "print(df[TARGET_COL].value_counts(normalize=True).round(3))\n",
    "\n",
    "plt.figure(figsize=(8, 5))\n",
    "df[TARGET_COL].value_counts().plot(kind='bar', edgecolor='black')\n",
    "plt.title(f'Target Distribution: {TARGET_COL}')\n",
    "plt.ylabel('Count')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a copy for processing\n",
    "df_processed = df.copy()\n",
    "\n",
    "# TODO: Handle missing values\n",
    "# Example: df_processed = df_processed.dropna()\n",
    "# Example: df_processed['col'] = df_processed['col'].fillna(df_processed['col'].median())\n",
    "\n",
    "# TODO: Remove duplicates if needed\n",
    "# df_processed = df_processed.drop_duplicates()\n",
    "\n",
    "print(f\"After preprocessing: {len(df_processed):,} rows\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Create new features\n",
    "# Example: df_processed['feature_ratio'] = df_processed['a'] / (df_processed['b'] + 1)\n",
    "\n",
    "# Encode categorical variables (if any)\n",
    "categorical_cols = df_processed.select_dtypes(include=['object']).columns.tolist()\n",
    "if TARGET_COL in categorical_cols:\n",
    "    categorical_cols.remove(TARGET_COL)\n",
    "\n",
    "print(f\"Categorical columns to encode: {categorical_cols}\")\n",
    "\n",
    "# One-hot encode (for low cardinality)\n",
    "if categorical_cols:\n",
    "    df_processed = pd.get_dummies(df_processed, columns=categorical_cols, drop_first=True)\n",
    "\n",
    "print(f\"Final shape: {df_processed.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Train/Test Split\n",
    "\n",
    "**CRITICAL**: All preprocessing that learns from data (scaling, encoding, imputation) must be fit on training data only!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define features and target\n",
    "FEATURE_COLS = [col for col in df_processed.columns if col != TARGET_COL]\n",
    "\n",
    "X = df_processed[FEATURE_COLS]\n",
    "y = df_processed[TARGET_COL]\n",
    "\n",
    "# Encode target if categorical\n",
    "if y.dtype == 'object':\n",
    "    le = LabelEncoder()\n",
    "    y = le.fit_transform(y)\n",
    "    print(f\"Target classes: {le.classes_}\")\n",
    "\n",
    "# Split BEFORE any scaling\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y,\n",
    "    test_size=TEST_SIZE,\n",
    "    random_state=SEED,\n",
    "    stratify=y  # Maintain class distribution\n",
    ")\n",
    "\n",
    "print(f\"Train size: {len(X_train):,} ({len(X_train)/len(X)*100:.1f}%)\")\n",
    "print(f\"Test size: {len(X_test):,} ({len(X_test)/len(X)*100:.1f}%)\")\n",
    "print(f\"\\nTrain target distribution:\")\n",
    "print(pd.Series(y_train).value_counts(normalize=True).round(3))\n",
    "print(f\"\\nTest target distribution:\")\n",
    "print(pd.Series(y_test).value_counts(normalize=True).round(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scale features (fit on train only!)\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)  # Transform only, no fit!\n",
    "\n",
    "print(\"Scaling complete. Train mean ~0, std ~1\")\n",
    "print(f\"Train mean: {X_train_scaled.mean():.6f}\")\n",
    "print(f\"Train std: {X_train_scaled.std():.6f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Baseline model\n",
    "baseline = DummyClassifier(strategy='most_frequent', random_state=SEED)\n",
    "baseline.fit(X_train_scaled, y_train)\n",
    "baseline_pred = baseline.predict(X_test_scaled)\n",
    "baseline_acc = accuracy_score(y_test, baseline_pred)\n",
    "print(f\"Baseline accuracy (most frequent): {baseline_acc:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Primary model\n",
    "MODEL_PARAMS = {\n",
    "    'n_estimators': 100,\n",
    "    'max_depth': 10,\n",
    "    'min_samples_split': 5,\n",
    "    'min_samples_leaf': 2,\n",
    "    'random_state': SEED,\n",
    "    'n_jobs': -1\n",
    "}\n",
    "\n",
    "model = RandomForestClassifier(**MODEL_PARAMS)\n",
    "print(f\"Model: {model.__class__.__name__}\")\n",
    "print(f\"Parameters: {MODEL_PARAMS}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "# Train model\n",
    "start_time = time.time()\n",
    "model.fit(X_train_scaled, y_train)\n",
    "train_time = time.time() - start_time\n",
    "print(f\"Training time: {train_time:.2f} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cross-validation on training data\n",
    "cv_scores = cross_val_score(model, X_train_scaled, y_train, cv=5, scoring='accuracy')\n",
    "print(f\"CV Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std()*2:.4f})\")\n",
    "print(f\"CV Scores: {cv_scores.round(4)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predictions\n",
    "y_pred = model.predict(X_test_scaled)\n",
    "y_prob = model.predict_proba(X_test_scaled)\n",
    "\n",
    "# For binary classification, get probability of positive class\n",
    "if y_prob.shape[1] == 2:\n",
    "    y_prob_pos = y_prob[:, 1]\n",
    "else:\n",
    "    y_prob_pos = y_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classification report\n",
    "print(\"Classification Report:\")\n",
    "print(\"=\" * 60)\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summary metrics\n",
    "metrics = {\n",
    "    'accuracy': accuracy_score(y_test, y_pred),\n",
    "    'precision_weighted': precision_score(y_test, y_pred, average='weighted'),\n",
    "    'recall_weighted': recall_score(y_test, y_pred, average='weighted'),\n",
    "    'f1_weighted': f1_score(y_test, y_pred, average='weighted'),\n",
    "}\n",
    "\n",
    "# Add ROC AUC for binary classification\n",
    "if len(np.unique(y_test)) == 2:\n",
    "    metrics['roc_auc'] = roc_auc_score(y_test, y_prob_pos)\n",
    "\n",
    "print(\"\\nSummary Metrics:\")\n",
    "print(\"-\" * 40)\n",
    "for name, value in metrics.items():\n",
    "    print(f\"  {name}: {value:.4f}\")\n",
    "\n",
    "print(f\"\\nImprovement over baseline: {(metrics['accuracy'] - baseline_acc)*100:.2f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion Matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n",
    "plt.xlabel('Predicted')\n",
    "plt.ylabel('Actual')\n",
    "plt.title('Confusion Matrix')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ROC Curve (binary classification only)\n",
    "if len(np.unique(y_test)) == 2:\n",
    "    fpr, tpr, thresholds = roc_curve(y_test, y_prob_pos)\n",
    "    \n",
    "    plt.figure(figsize=(8, 6))\n",
    "    plt.plot(fpr, tpr, label=f'Model (AUC = {metrics[\"roc_auc\"]:.4f})')\n",
    "    plt.plot([0, 1], [0, 1], 'k--', label='Random')\n",
    "    plt.xlabel('False Positive Rate')\n",
    "    plt.ylabel('True Positive Rate')\n",
    "    plt.title('ROC Curve')\n",
    "    plt.legend()\n",
    "    plt.grid(True, alpha=0.3)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 10. Error Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Feature importance\n",
    "importance_df = pd.DataFrame({\n",
    "    'feature': FEATURE_COLS,\n",
    "    'importance': model.feature_importances_\n",
    "}).sort_values('importance', ascending=False)\n",
    "\n",
    "plt.figure(figsize=(10, 8))\n",
    "sns.barplot(data=importance_df.head(20), x='importance', y='feature')\n",
    "plt.title('Top 20 Feature Importances')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"Top 10 features:\")\n",
    "print(importance_df.head(10).to_string(index=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Analyze misclassifications\n",
    "error_mask = y_pred != y_test\n",
    "error_count = error_mask.sum()\n",
    "error_rate = error_count / len(y_test)\n",
    "\n",
    "print(f\"Total errors: {error_count} ({error_rate*100:.2f}%)\")\n",
    "print(f\"\\nErrors by predicted class:\")\n",
    "print(pd.Series(y_pred[error_mask]).value_counts())\n",
    "print(f\"\\nErrors by actual class:\")\n",
    "print(pd.Series(y_test[error_mask]).value_counts())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 11. Save Artifacts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "import json\n",
    "\n",
    "# Create artifacts directory\n",
    "ARTIFACTS_DIR = Path('artifacts')\n",
    "ARTIFACTS_DIR.mkdir(exist_ok=True)\n",
    "\n",
    "timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')\n",
    "\n",
    "# Save model\n",
    "model_path = ARTIFACTS_DIR / f'model_{timestamp}.joblib'\n",
    "joblib.dump(model, model_path)\n",
    "print(f\"Model saved: {model_path}\")\n",
    "\n",
    "# Save scaler\n",
    "scaler_path = ARTIFACTS_DIR / f'scaler_{timestamp}.joblib'\n",
    "joblib.dump(scaler, scaler_path)\n",
    "print(f\"Scaler saved: {scaler_path}\")\n",
    "\n",
    "# Save metadata\n",
    "metadata = {\n",
    "    'timestamp': timestamp,\n",
    "    'model_type': model.__class__.__name__,\n",
    "    'model_params': MODEL_PARAMS,\n",
    "    'feature_cols': FEATURE_COLS,\n",
    "    'target_col': TARGET_COL,\n",
    "    'metrics': metrics,\n",
    "    'train_size': len(X_train),\n",
    "    'test_size': len(X_test),\n",
    "    'seed': SEED,\n",
    "    'baseline_accuracy': baseline_acc,\n",
    "    'cv_mean': float(cv_scores.mean()),\n",
    "    'cv_std': float(cv_scores.std())\n",
    "}\n",
    "\n",
    "metadata_path = ARTIFACTS_DIR / f'metadata_{timestamp}.json'\n",
    "with open(metadata_path, 'w') as f:\n",
    "    json.dump(metadata, f, indent=2)\n",
    "print(f\"Metadata saved: {metadata_path}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 12. Conclusions\n",
    "\n",
    "### Results Summary\n",
    "- **Model**: [TODO: Model type]\n",
    "- **Test Accuracy**: [TODO: X.XX]\n",
    "- **Baseline**: [TODO: X.XX]\n",
    "- **Improvement**: [TODO: X.XX%]\n",
    "\n",
    "### Key Findings\n",
    "1. [TODO: Most important features]\n",
    "2. [TODO: Model performance observations]\n",
    "3. [TODO: Error patterns]\n",
    "\n",
    "### Limitations\n",
    "- [TODO: Data limitations]\n",
    "- [TODO: Model limitations]\n",
    "\n",
    "### Next Steps\n",
    "1. [TODO: Hyperparameter tuning]\n",
    "2. [TODO: Try other models]\n",
    "3. [TODO: Feature engineering]\n",
    "4. [TODO: Deployment considerations]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exploratory Data Analysis\n",
    "\n",
    "**Dataset**: [TODO: Dataset name]\n",
    "**Author**: [TODO: Your name]\n",
    "**Date**: [TODO: Date]\n",
    "\n",
    "## Objective\n",
    "[TODO: Describe the analysis objective]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard library\n",
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "# Data handling\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Configuration\n",
    "SEED = 42\n",
    "np.random.seed(SEED)\n",
    "\n",
    "# Display settings\n",
    "pd.set_option('display.max_columns', 50)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "plt.style.use('seaborn-v0_8-whitegrid')\n",
    "%matplotlib inline\n",
    "\n",
    "# Print versions\n",
    "print(f\"Python: {sys.version}\")\n",
    "print(f\"NumPy: {np.__version__}\")\n",
    "print(f\"Pandas: {pd.__version__}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Update path to your data\n",
    "DATA_PATH = Path('data/raw')\n",
    "df = pd.read_csv(DATA_PATH / 'dataset.csv')\n",
    "\n",
    "print(f\"Loaded {len(df):,} rows, {len(df.columns)} columns\")\n",
    "print(f\"Memory usage: {df.memory_usage(deep=True).sum() / 1e6:.2f} MB\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Basic Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data types and non-null counts\n",
    "print(df.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# First few rows\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Last few rows\n",
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Random sample\n",
    "df.sample(5, random_state=SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Descriptive statistics for numeric columns\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Descriptive statistics for categorical columns\n",
    "df.describe(include=['object', 'category'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Missing Values Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Missing value summary\n",
    "missing = df.isnull().sum()\n",
    "missing_pct = (missing / len(df) * 100).round(2)\n",
    "missing_df = pd.DataFrame({\n",
    "    'missing_count': missing,\n",
    "    'missing_pct': missing_pct\n",
    "}).query('missing_count > 0').sort_values('missing_pct', ascending=False)\n",
    "\n",
    "if len(missing_df) > 0:\n",
    "    print(f\"Columns with missing values: {len(missing_df)}\")\n",
    "    display(missing_df)\n",
    "else:\n",
    "    print(\"No missing values found!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Visualize missing values\n",
    "if len(missing_df) > 0:\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    sns.barplot(data=missing_df.reset_index(), x='index', y='missing_pct')\n",
    "    plt.xticks(rotation=45, ha='right')\n",
    "    plt.xlabel('Column')\n",
    "    plt.ylabel('Missing %')\n",
    "    plt.title('Missing Values by Column')\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Univariate Analysis\n",
    "\n",
    "### 5.1 Numeric Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identify numeric columns\n",
    "numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()\n",
    "print(f\"Numeric columns ({len(numeric_cols)}): {numeric_cols}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Histograms for numeric columns\n",
    "n_cols = min(len(numeric_cols), 12)  # Limit to 12 columns\n",
    "if n_cols > 0:\n",
    "    fig, axes = plt.subplots(\n",
    "        nrows=(n_cols + 2) // 3, \n",
    "        ncols=3, \n",
    "        figsize=(15, 4 * ((n_cols + 2) // 3))\n",
    "    )\n",
    "    axes = axes.flatten()\n",
    "    \n",
    "    for idx, col in enumerate(numeric_cols[:n_cols]):\n",
    "        df[col].hist(ax=axes[idx], bins=30, edgecolor='black', alpha=0.7)\n",
    "        axes[idx].set_title(col)\n",
    "        axes[idx].set_xlabel('')\n",
    "    \n",
    "    # Hide unused subplots\n",
    "    for idx in range(n_cols, len(axes)):\n",
    "        axes[idx].set_visible(False)\n",
    "    \n",
    "    plt.suptitle('Numeric Feature Distributions', y=1.02, fontsize=14)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Box plots for outlier detection\n",
    "if len(numeric_cols) > 0:\n",
    "    n_cols = min(len(numeric_cols), 12)\n",
    "    fig, axes = plt.subplots(\n",
    "        nrows=(n_cols + 2) // 3, \n",
    "        ncols=3, \n",
    "        figsize=(15, 4 * ((n_cols + 2) // 3))\n",
    "    )\n",
    "    axes = axes.flatten()\n",
    "    \n",
    "    for idx, col in enumerate(numeric_cols[:n_cols]):\n",
    "        sns.boxplot(data=df, y=col, ax=axes[idx])\n",
    "        axes[idx].set_title(col)\n",
    "    \n",
    "    for idx in range(n_cols, len(axes)):\n",
    "        axes[idx].set_visible(False)\n",
    "    \n",
    "    plt.suptitle('Numeric Feature Box Plots (Outlier Detection)', y=1.02, fontsize=14)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5.2 Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Identify categorical columns\n",
    "categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist()\n",
    "print(f\"Categorical columns ({len(categorical_cols)}): {categorical_cols}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Value counts for categorical columns\n",
    "for col in categorical_cols:\n",
    "    print(f\"\\n{col} ({df[col].nunique()} unique values):\")\n",
    "    print(df[col].value_counts().head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Bar plots for categorical columns (low cardinality only)\n",
    "low_cardinality = [col for col in categorical_cols if df[col].nunique() <= 10]\n",
    "\n",
    "if len(low_cardinality) > 0:\n",
    "    n_cols = min(len(low_cardinality), 6)\n",
    "    fig, axes = plt.subplots(\n",
    "        nrows=(n_cols + 1) // 2, \n",
    "        ncols=2, \n",
    "        figsize=(14, 4 * ((n_cols + 1) // 2))\n",
    "    )\n",
    "    axes = axes.flatten()\n",
    "    \n",
    "    for idx, col in enumerate(low_cardinality[:n_cols]):\n",
    "        df[col].value_counts().plot(kind='bar', ax=axes[idx], edgecolor='black')\n",
    "        axes[idx].set_title(col)\n",
    "        axes[idx].set_xlabel('')\n",
    "        plt.setp(axes[idx].xaxis.get_majorticklabels(), rotation=45, ha='right')\n",
    "    \n",
    "    for idx in range(n_cols, len(axes)):\n",
    "        axes[idx].set_visible(False)\n",
    "    \n",
    "    plt.suptitle('Categorical Feature Distributions', y=1.02, fontsize=14)\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Bivariate Analysis\n",
    "\n",
    "### 6.1 Target Variable Analysis\n",
    "\n",
    "[TODO: Update TARGET_COL to your target variable]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: Set your target variable\n",
    "TARGET_COL = 'target'  # Change this\n",
    "\n",
    "if TARGET_COL in df.columns:\n",
    "    print(f\"Target variable: {TARGET_COL}\")\n",
    "    print(f\"Type: {df[TARGET_COL].dtype}\")\n",
    "    print(f\"\\nValue counts:\")\n",
    "    print(df[TARGET_COL].value_counts())\n",
    "    \n",
    "    # Plot target distribution\n",
    "    plt.figure(figsize=(8, 5))\n",
    "    if df[TARGET_COL].dtype in ['object', 'category'] or df[TARGET_COL].nunique() <= 10:\n",
    "        df[TARGET_COL].value_counts().plot(kind='bar', edgecolor='black')\n",
    "        plt.ylabel('Count')\n",
    "    else:\n",
    "        df[TARGET_COL].hist(bins=30, edgecolor='black')\n",
    "        plt.ylabel('Frequency')\n",
    "    plt.title(f'Target Distribution: {TARGET_COL}')\n",
    "    plt.xlabel(TARGET_COL)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "else:\n",
    "    print(f\"Target column '{TARGET_COL}' not found in dataset\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6.2 Feature vs Target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Numeric features vs target (if target is categorical)\n",
    "if TARGET_COL in df.columns and df[TARGET_COL].nunique() <= 10:\n",
    "    for col in numeric_cols[:6]:  # Limit to first 6\n",
    "        if col != TARGET_COL:\n",
    "            plt.figure(figsize=(10, 5))\n",
    "            sns.boxplot(data=df, x=TARGET_COL, y=col)\n",
    "            plt.title(f'{col} by {TARGET_COL}')\n",
    "            plt.tight_layout()\n",
    "            plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Correlation Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Correlation matrix for numeric columns\n",
    "if len(numeric_cols) > 1:\n",
    "    corr_matrix = df[numeric_cols].corr()\n",
    "    \n",
    "    plt.figure(figsize=(12, 10))\n",
    "    mask = np.triu(np.ones_like(corr_matrix, dtype=bool))\n",
    "    sns.heatmap(\n",
    "        corr_matrix, \n",
    "        mask=mask,\n",
    "        annot=True if len(numeric_cols) <= 15 else False,\n",
    "        fmt='.2f',\n",
    "        cmap='coolwarm',\n",
    "        center=0,\n",
    "        square=True,\n",
    "        linewidths=0.5\n",
    "    )\n",
    "    plt.title('Feature Correlation Matrix')\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Highly correlated features\n",
    "if len(numeric_cols) > 1:\n",
    "    high_corr = []\n",
    "    for i in range(len(corr_matrix.columns)):\n",
    "        for j in range(i + 1, len(corr_matrix.columns)):\n",
    "            if abs(corr_matrix.iloc[i, j]) > 0.7:\n",
    "                high_corr.append({\n",
    "                    'feature_1': corr_matrix.columns[i],\n",
    "                    'feature_2': corr_matrix.columns[j],\n",
    "                    'correlation': corr_matrix.iloc[i, j]\n",
    "                })\n",
    "    \n",
    "    if high_corr:\n",
    "        print(\"Highly correlated feature pairs (|r| > 0.7):\")\n",
    "        display(pd.DataFrame(high_corr).sort_values('correlation', key=abs, ascending=False))\n",
    "    else:\n",
    "        print(\"No highly correlated feature pairs found.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Correlation with target\n",
    "if TARGET_COL in df.columns and TARGET_COL in numeric_cols:\n",
    "    target_corr = corr_matrix[TARGET_COL].drop(TARGET_COL).sort_values(key=abs, ascending=False)\n",
    "    \n",
    "    plt.figure(figsize=(10, 6))\n",
    "    target_corr.plot(kind='barh')\n",
    "    plt.xlabel('Correlation with Target')\n",
    "    plt.title(f'Feature Correlations with {TARGET_COL}')\n",
    "    plt.tight_layout()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Key Findings Summary\n",
    "\n",
    "### Data Overview\n",
    "- **Rows**: [TODO]\n",
    "- **Columns**: [TODO]\n",
    "- **Numeric features**: [TODO]\n",
    "- **Categorical features**: [TODO]\n",
    "\n",
    "### Missing Values\n",
    "[TODO: Summarize missing value findings]\n",
    "\n",
    "### Target Variable\n",
    "[TODO: Describe target distribution, class imbalance if applicable]\n",
    "\n",
    "### Key Correlations\n",
    "[TODO: List important correlations]\n",
    "\n",
    "### Potential Issues\n",
    "- [TODO: List any data quality issues]\n",
    "- [TODO: List outliers]\n",
    "- [TODO: List class imbalance]\n",
    "\n",
    "### Recommendations for Modeling\n",
    "1. [TODO: Preprocessing suggestions]\n",
    "2. [TODO: Feature engineering ideas]\n",
    "3. [TODO: Modeling approach suggestions]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save processed data (optional)\n",
    "# df.to_csv(DATA_PATH / 'processed' / 'dataset_cleaned.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Experiment\n",
    "\n",
    "**Experiment ID**: [Auto-generated]\n",
    "**Hypothesis**: [TODO: What are you testing?]\n",
    "**Author**: [TODO: Your name]\n",
    "**Date**: [TODO: Date]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Experiment Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": ["parameters"]
   },
   "outputs": [],
   "source": [
    "# =============================================================================\n",
    "# EXPERIMENT PARAMETERS\n",
    "# These can be overridden by papermill for automated runs\n",
    "# =============================================================================\n",
    "\n",
    "# Experiment identification\n",
    "EXPERIMENT_NAME = 'default_experiment'\n",
    "RUN_NAME = None  # Auto-generated if None\n",
    "\n",
    "# Data parameters\n",
    "DATA_PATH = 'data/raw/dataset.csv'\n",
    "TARGET_COL = 'target'\n",
    "TEST_SIZE = 0.2\n",
    "\n",
    "# Model parameters\n",
    "MODEL_TYPE = 'random_forest'  # Options: random_forest, xgboost, logistic\n",
    "N_ESTIMATORS = 100\n",
    "MAX_DEPTH = 10\n",
    "LEARNING_RATE = 0.1  # For gradient boosting models\n",
    "\n",
    "# Training parameters\n",
    "SEED = 42\n",
    "CV_FOLDS = 5\n",
    "\n",
    "# Output\n",
    "SAVE_MODEL = True\n",
    "ARTIFACTS_DIR = 'artifacts'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Setup and Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard library\n",
    "import os\n",
    "import sys\n",
    "import random\n",
    "import json\n",
    "import logging\n",
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "\n",
    "# Data handling\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# Visualization\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# ML\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import (\n",
    "    accuracy_score, precision_score, recall_score, f1_score,\n",
    "    confusion_matrix, classification_report, roc_auc_score\n",
    ")\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# Optional: XGBoost\n",
    "try:\n",
    "    import xgboost as xgb\n",
    "    HAS_XGB = True\n",
    "except ImportError:\n",
    "    HAS_XGB = False\n",
    "\n",
    "# Optional: MLflow tracking\n",
    "try:\n",
    "    import mlflow\n",
    "    import mlflow.sklearn\n",
    "    HAS_MLFLOW = True\n",
    "except ImportError:\n",
    "    HAS_MLFLOW = False\n",
    "\n",
    "print(f\"MLflow available: {HAS_MLFLOW}\")\n",
    "print(f\"XGBoost available: {HAS_XGB}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =============================================================================\n",
    "# REPRODUCIBILITY SETUP\n",
    "# =============================================================================\n",
    "\n",
    "def set_seeds(seed):\n",
    "    \"\"\"Set random seeds for reproducibility.\"\"\"\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    \n",
    "    # PyTorch (if available)\n",
    "    try:\n",
    "        import torch\n",
    "        torch.manual_seed(seed)\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "    except ImportError:\n",
    "        pass\n",
    "\n",
    "set_seeds(SEED)\n",
    "print(f\"Seeds set to: {SEED}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# =============================================================================\n",
    "# EXPERIMENT LOGGING SETUP\n",
    "# =============================================================================\n",
    "\n",
    "# Generate experiment ID\n",
    "TIMESTAMP = datetime.now().strftime('%Y%m%d_%H%M%S')\n",
    "EXPERIMENT_ID = f\"{EXPERIMENT_NAME}_{TIMESTAMP}\"\n",
    "\n",
    "if RUN_NAME is None:\n",
    "    RUN_NAME = f\"run_{TIMESTAMP}\"\n",
    "\n",
    "# Setup logging\n",
    "logging.basicConfig(\n",
    "    level=logging.INFO,\n",
    "    format='%(asctime)s - %(levelname)s - %(message)s'\n",
    ")\n",
    "logger = logging.getLogger(__name__)\n",
    "\n",
    "# Create artifacts directory\n",
    "artifacts_path = Path(ARTIFACTS_DIR) / EXPERIMENT_ID\n",
    "artifacts_path.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "# Initialize experiment log\n",
    "experiment_log = {\n",
    "    'experiment_id': EXPERIMENT_ID,\n",
    "    'run_name': RUN_NAME,\n",
    "    'timestamp': TIMESTAMP,\n",
    "    'parameters': {\n",
    "        'data_path': DATA_PATH,\n",
    "        'target_col': TARGET_COL,\n",
    "        'test_size': TEST_SIZE,\n",
    "        'model_type': MODEL_TYPE,\n",
    "        'n_estimators': N_ESTIMATORS,\n",
    "        'max_depth': MAX_DEPTH,\n",
    "        'learning_rate': LEARNING_RATE,\n",
    "        'seed': SEED,\n",
    "        'cv_folds': CV_FOLDS,\n",
    "    },\n",
    "    'environment': {\n",
    "        'python_version': sys.version,\n",
    "        'numpy_version': np.__version__,\n",
    "        'pandas_version': pd.__version__,\n",
    "    },\n",
    "    'metrics': {},\n",
    "    'artifacts': [],\n",
    "}\n",
    "\n",
    "print(f\"Experiment ID: {EXPERIMENT_ID}\")\n",
    "print(f\"Run Name: {RUN_NAME}\")\n",
    "print(f\"Artifacts Path: {artifacts_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optional: Initialize MLflow tracking\n",
    "if HAS_MLFLOW:\n",
    "    mlflow.set_experiment(EXPERIMENT_NAME)\n",
    "    mlflow.start_run(run_name=RUN_NAME)\n",
    "    \n",
    "    # Log parameters\n",
    "    mlflow.log_params(experiment_log['parameters'])\n",
    "    print(\"MLflow run started\")\n",
    "else:\n",
    "    print(\"MLflow not available, using local logging only\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Data Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.info(f\"Loading data from {DATA_PATH}\")\n",
    "\n",
    "df = pd.read_csv(DATA_PATH)\n",
    "\n",
    "experiment_log['data'] = {\n",
    "    'n_rows': len(df),\n",
    "    'n_cols': len(df.columns),\n",
    "    'columns': df.columns.tolist(),\n",
    "}\n",
    "\n",
    "print(f\"Loaded {len(df):,} rows, {len(df.columns)} columns\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Preprocessing Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_data(df, target_col):\n",
    "    \"\"\"Preprocess the dataset.\"\"\"\n",
    "    df_processed = df.copy()\n",
    "    \n",
    "    # TODO: Add your preprocessing steps\n",
    "    # Example: Handle missing values\n",
    "    # df_processed = df_processed.dropna()\n",
    "    \n",
    "    # Example: Encode categoricals\n",
    "    categorical_cols = df_processed.select_dtypes(include=['object']).columns.tolist()\n",
    "    if target_col in categorical_cols:\n",
    "        categorical_cols.remove(target_col)\n",
    "    \n",
    "    if categorical_cols:\n",
    "        df_processed = pd.get_dummies(df_processed, columns=categorical_cols, drop_first=True)\n",
    "    \n",
    "    return df_processed\n",
    "\n",
    "df_processed = preprocess_data(df, TARGET_COL)\n",
    "print(f\"After preprocessing: {df_processed.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prepare features and target\n",
    "FEATURE_COLS = [col for col in df_processed.columns if col != TARGET_COL]\n",
    "\n",
    "X = df_processed[FEATURE_COLS]\n",
    "y = df_processed[TARGET_COL]\n",
    "\n",
    "# Split data\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y,\n",
    "    test_size=TEST_SIZE,\n",
    "    random_state=SEED,\n",
    "    stratify=y\n",
    ")\n",
    "\n",
    "# Scale features\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "experiment_log['data']['n_features'] = len(FEATURE_COLS)\n",
    "experiment_log['data']['train_size'] = len(X_train)\n",
    "experiment_log['data']['test_size'] = len(X_test)\n",
    "\n",
    "print(f\"Features: {len(FEATURE_COLS)}\")\n",
    "print(f\"Train: {len(X_train)}, Test: {len(X_test)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Model Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(model_type, **kwargs):\n",
    "    \"\"\"Factory function to create models.\"\"\"\n",
    "    if model_type == 'random_forest':\n",
    "        return RandomForestClassifier(\n",
    "            n_estimators=kwargs.get('n_estimators', 100),\n",
    "            max_depth=kwargs.get('max_depth', 10),\n",
    "            random_state=kwargs.get('seed', 42),\n",
    "            n_jobs=-1\n",
    "        )\n",
    "    elif model_type == 'logistic':\n",
    "        return LogisticRegression(\n",
    "            max_iter=1000,\n",
    "            random_state=kwargs.get('seed', 42)\n",
    "        )\n",
    "    elif model_type == 'xgboost' and HAS_XGB:\n",
    "        return xgb.XGBClassifier(\n",
    "            n_estimators=kwargs.get('n_estimators', 100),\n",
    "            max_depth=kwargs.get('max_depth', 10),\n",
    "            learning_rate=kwargs.get('learning_rate', 0.1),\n",
    "            random_state=kwargs.get('seed', 42),\n",
    "            use_label_encoder=False,\n",
    "            eval_metric='logloss'\n",
    "        )\n",
    "    else:\n",
    "        raise ValueError(f\"Unknown model type: {model_type}\")\n",
    "\n",
    "model = get_model(\n",
    "    MODEL_TYPE,\n",
    "    n_estimators=N_ESTIMATORS,\n",
    "    max_depth=MAX_DEPTH,\n",
    "    learning_rate=LEARNING_RATE,\n",
    "    seed=SEED\n",
    ")\n",
    "\n",
    "print(f\"Model: {model.__class__.__name__}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Training with Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "logger.info(\"Starting training...\")\n",
    "start_time = time.time()\n",
    "\n",
    "# Cross-validation\n",
    "cv_scores = cross_val_score(model, X_train_scaled, y_train, cv=CV_FOLDS, scoring='accuracy')\n",
    "\n",
    "# Fit final model\n",
    "model.fit(X_train_scaled, y_train)\n",
    "\n",
    "train_time = time.time() - start_time\n",
    "\n",
    "# Log CV metrics\n",
    "experiment_log['metrics']['cv_accuracy_mean'] = float(cv_scores.mean())\n",
    "experiment_log['metrics']['cv_accuracy_std'] = float(cv_scores.std())\n",
    "experiment_log['metrics']['train_time_seconds'] = train_time\n",
    "\n",
    "if HAS_MLFLOW:\n",
    "    mlflow.log_metric('cv_accuracy_mean', cv_scores.mean())\n",
    "    mlflow.log_metric('cv_accuracy_std', cv_scores.std())\n",
    "    mlflow.log_metric('train_time_seconds', train_time)\n",
    "\n",
    "print(f\"Training completed in {train_time:.2f}s\")\n",
    "print(f\"CV Accuracy: {cv_scores.mean():.4f} (+/- {cv_scores.std()*2:.4f})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 7. Evaluation with Metrics Logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Predictions\n",
    "y_pred = model.predict(X_test_scaled)\n",
    "y_prob = model.predict_proba(X_test_scaled)\n",
    "\n",
    "# Calculate metrics\n",
    "test_metrics = {\n",
    "    'test_accuracy': accuracy_score(y_test, y_pred),\n",
    "    'test_precision': precision_score(y_test, y_pred, average='weighted'),\n",
    "    'test_recall': recall_score(y_test, y_pred, average='weighted'),\n",
    "    'test_f1': f1_score(y_test, y_pred, average='weighted'),\n",
    "}\n",
    "\n",
    "# Add ROC AUC for binary classification\n",
    "if len(np.unique(y_test)) == 2:\n",
    "    test_metrics['test_roc_auc'] = roc_auc_score(y_test, y_prob[:, 1])\n",
    "\n",
    "# Log metrics\n",
    "experiment_log['metrics'].update(test_metrics)\n",
    "\n",
    "if HAS_MLFLOW:\n",
    "    for name, value in test_metrics.items():\n",
    "        mlflow.log_metric(name, value)\n",
    "\n",
    "print(\"\\nTest Metrics:\")\n",
    "print(\"=\" * 40)\n",
    "for name, value in test_metrics.items():\n",
    "    print(f\"  {name}: {value:.4f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Classification report\n",
    "print(\"\\nClassification Report:\")\n",
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Confusion matrix plot\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(8, 6))\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=ax)\n",
    "ax.set_xlabel('Predicted')\n",
    "ax.set_ylabel('Actual')\n",
    "ax.set_title(f'Confusion Matrix - {EXPERIMENT_ID}')\n",
    "plt.tight_layout()\n",
    "\n",
    "# Save figure\n",
    "cm_path = artifacts_path / 'confusion_matrix.png'\n",
    "plt.savefig(cm_path, dpi=150, bbox_inches='tight')\n",
    "experiment_log['artifacts'].append(str(cm_path))\n",
    "\n",
    "if HAS_MLFLOW:\n",
    "    mlflow.log_artifact(str(cm_path))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 8. Artifact Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import joblib\n",
    "\n",
    "if SAVE_MODEL:\n",
    "    # Save model\n",
    "    model_path = artifacts_path / 'model.joblib'\n",
    "    joblib.dump(model, model_path)\n",
    "    experiment_log['artifacts'].append(str(model_path))\n",
    "    print(f\"Model saved: {model_path}\")\n",
    "    \n",
    "    # Save scaler\n",
    "    scaler_path = artifacts_path / 'scaler.joblib'\n",
    "    joblib.dump(scaler, scaler_path)\n",
    "    experiment_log['artifacts'].append(str(scaler_path))\n",
    "    print(f\"Scaler saved: {scaler_path}\")\n",
    "    \n",
    "    # Log to MLflow\n",
    "    if HAS_MLFLOW:\n",
    "        mlflow.sklearn.log_model(model, 'model')\n",
    "        mlflow.log_artifact(str(scaler_path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save experiment log\n",
    "log_path = artifacts_path / 'experiment_log.json'\n",
    "with open(log_path, 'w') as f:\n",
    "    json.dump(experiment_log, f, indent=2, default=str)\n",
    "print(f\"Experiment log saved: {log_path}\")\n",
    "\n",
    "if HAS_MLFLOW:\n",
    "    mlflow.log_artifact(str(log_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9. Experiment Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# End MLflow run\n",
    "if HAS_MLFLOW:\n",
    "    mlflow.end_run()\n",
    "    print(\"MLflow run ended\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print experiment summary\n",
    "print(\"\\n\" + \"=\" * 60)\n",
    "print(\"EXPERIMENT SUMMARY\")\n",
    "print(\"=\" * 60)\n",
    "print(f\"\\nExperiment ID: {EXPERIMENT_ID}\")\n",
    "print(f\"Model: {model.__class__.__name__}\")\n",
    "print(f\"\\nKey Parameters:\")\n",
    "print(f\"  - n_estimators: {N_ESTIMATORS}\")\n",
    "print(f\"  - max_depth: {MAX_DEPTH}\")\n",
    "print(f\"  - seed: {SEED}\")\n",
    "print(f\"\\nData:\")\n",
    "print(f\"  - Train samples: {experiment_log['data']['train_size']}\")\n",
    "print(f\"  - Test samples: {experiment_log['data']['test_size']}\")\n",
    "print(f\"  - Features: {experiment_log['data']['n_features']}\")\n",
    "print(f\"\\nResults:\")\n",
    "print(f\"  - CV Accuracy: {experiment_log['metrics']['cv_accuracy_mean']:.4f} (+/- {experiment_log['metrics']['cv_accuracy_std']*2:.4f})\")\n",
    "print(f\"  - Test Accuracy: {experiment_log['metrics']['test_accuracy']:.4f}\")\n",
    "print(f\"  - Test F1: {experiment_log['metrics']['test_f1']:.4f}\")\n",
    "print(f\"\\nArtifacts saved to: {artifacts_path}\")\n",
    "print(\"=\" * 60)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

#!/usr/bin/env python3
"""
Analyze Jupyter notebook structure and detect anti-patterns.

Usage:
    python analyze_notebook.py <notebook.ipynb> [--output json|text]

Outputs:
    - Cell counts by type
    - Import statements
    - Function/class definitions
    - Detected issues and anti-patterns
    - Section structure
"""

import argparse
import json
import re
import sys
from pathlib import Path
from typing import Any

try:
    import nbformat
except ImportError:
    print("Error: nbformat not installed. Run: pip install nbformat")
    sys.exit(1)


def analyze_notebook(notebook_path: str) -> dict[str, Any]:
    """Parse notebook and extract structure information."""
    path = Path(notebook_path)
    if not path.exists():
        raise FileNotFoundError(f"Notebook not found: {notebook_path}")

    nb = nbformat.read(path, as_version=4)

    analysis = {
        "path": str(path),
        "metadata": {
            "kernel": nb.metadata.get("kernelspec", {}).get("display_name", "Unknown"),
            "language": nb.metadata.get("kernelspec", {}).get("language", "Unknown"),
            "nbformat": f"{nb.nbformat}.{nb.nbformat_minor}",
        },
        "cell_counts": {"code": 0, "markdown": 0, "raw": 0, "total": 0},
        "execution_info": {
            "max_execution_count": 0,
            "cells_with_output": 0,
            "execution_order_issues": [],
        },
        "imports": [],
        "functions": [],
        "classes": [],
        "sections": [],
        "issues": [],
    }

    last_execution_count = 0
    import_pattern = re.compile(r"^(?:from\s+(\S+)\s+)?import\s+(\S+)", re.MULTILINE)
    function_pattern = re.compile(r"^def\s+(\w+)\s*\(", re.MULTILINE)
    class_pattern = re.compile(r"^class\s+(\w+)\s*[\(:]", re.MULTILINE)
    seed_pattern = re.compile(
        r"(?:np\.random\.seed|random\.seed|torch\.manual_seed|tf\.random\.set_seed)\s*\(",
        re.MULTILINE,
    )
    absolute_path_pattern = re.compile(r"['\"](?:/[^'\"]+|[A-Z]:\\[^'\"]+)['\"]")

    has_seed = False
    has_train_test_split = False
    imports_after_code = False
    first_code_cell = True

    for idx, cell in enumerate(nb.cells):
        cell_type = cell.cell_type
        analysis["cell_counts"][cell_type] = (
            analysis["cell_counts"].get(cell_type, 0) + 1
        )
        analysis["cell_counts"]["total"] += 1

        if cell_type == "markdown":
            # Extract section headers
            source = cell.source
            for line in source.split("\n"):
                if line.startswith("#"):
                    header_level = len(line) - len(line.lstrip("#"))
                    header_text = line.lstrip("#").strip()
                    if header_text:
                        analysis["sections"].append(
                            {
                                "level": header_level,
                                "title": header_text,
                                "cell_index": idx,
                            }
                        )

        elif cell_type == "code":
            source = cell.source
            execution_count = cell.get("execution_count")

            # Check execution order
            if execution_count is not None:
                analysis["execution_info"]["max_execution_count"] = max(
                    analysis["execution_info"]["max_execution_count"], execution_count
                )
                if execution_count < last_execution_count:
                    analysis["execution_info"]["execution_order_issues"].append(
                        {
                            "cell_index": idx,
                            "execution_count": execution_count,
                            "expected_after": last_execution_count,
                        }
                    )
                last_execution_count = execution_count

            # Check for outputs
            if cell.get("outputs"):
                analysis["execution_info"]["cells_with_output"] += 1

            # Extract imports
            for match in import_pattern.finditer(source):
                module = match.group(1) or match.group(2)
                if module:
                    module_name = module.split(".")[0]
                    if module_name not in analysis["imports"]:
                        analysis["imports"].append(module_name)

                    # Check if imports come after code
                    if not first_code_cell and module_name not in [
                        "warnings",
                        "logging",
                    ]:
                        imports_after_code = True

            # Extract functions
            for match in function_pattern.finditer(source):
                analysis["functions"].append(
                    {"name": match.group(1), "cell_index": idx}
                )

            # Extract classes
            for match in class_pattern.finditer(source):
                analysis["classes"].append({"name": match.group(1), "cell_index": idx})

            # Check for seeds
            if seed_pattern.search(source):
                has_seed = True

            # Check for train_test_split
            if "train_test_split" in source:
                has_train_test_split = True

            # Check for absolute paths
            if absolute_path_pattern.search(source):
                analysis["issues"].append(
                    {
                        "severity": "HIGH",
                        "type": "absolute_path",
                        "cell_index": idx,
                        "message": "Absolute file path detected. Use relative paths for portability.",
                    }
                )

            first_code_cell = False

    # Post-analysis checks

    # Check for missing seeds
    if not has_seed and analysis["cell_counts"]["code"] > 3:
        analysis["issues"].append(
            {
                "severity": "HIGH",
                "type": "missing_seed",
                "message": "No random seed setting found. Add np.random.seed() for reproducibility.",
            }
        )

    # Check for missing train/test split
    ml_imports = {"sklearn", "torch", "tensorflow", "keras", "xgboost", "lightgbm"}
    if ml_imports.intersection(set(analysis["imports"])) and not has_train_test_split:
        analysis["issues"].append(
            {
                "severity": "MEDIUM",
                "type": "no_train_test_split",
                "message": "ML imports found but no train_test_split detected. Ensure proper data splitting.",
            }
        )

    # Check for imports after code
    if imports_after_code:
        analysis["issues"].append(
            {
                "severity": "LOW",
                "type": "scattered_imports",
                "message": "Imports found after code cells. Move all imports to the beginning.",
            }
        )

    # Check execution order
    if analysis["execution_info"]["execution_order_issues"]:
        analysis["issues"].append(
            {
                "severity": "HIGH",
                "type": "execution_order",
                "message": f"Cells executed out of order. Found {len(analysis['execution_info']['execution_order_issues'])} issues.",
            }
        )

    # Check for no functions (lack of modularization)
    if analysis["cell_counts"]["code"] > 10 and len(analysis["functions"]) == 0:
        analysis["issues"].append(
            {
                "severity": "MEDIUM",
                "type": "no_modularization",
                "message": "No functions defined in a notebook with 10+ code cells. Consider refactoring.",
            }
        )

    # Check filename
    if "Untitled" in path.name:
        analysis["issues"].append(
            {
                "severity": "LOW",
                "type": "default_name",
                "message": "Notebook has default name. Use descriptive naming.",
            }
        )

    if "Copy" in path.name:
        analysis["issues"].append(
            {
                "severity": "LOW",
                "type": "copy_name",
                "message": "Notebook appears to be a copy. Rename appropriately.",
            }
        )

    return analysis


def format_text_output(analysis: dict[str, Any]) -> str:
    """Format analysis results as human-readable text."""
    lines = []
    lines.append(f"Notebook Analysis: {analysis['path']}")
    lines.append("=" * 60)

    # Metadata
    lines.append("\nMetadata:")
    lines.append(f"  Kernel: {analysis['metadata']['kernel']}")
    lines.append(f"  Language: {analysis['metadata']['language']}")
    lines.append(f"  Format: {analysis['metadata']['nbformat']}")

    # Cell counts
    lines.append("\nCell Counts:")
    for cell_type, count in analysis["cell_counts"].items():
        lines.append(f"  {cell_type}: {count}")

    # Execution info
    lines.append("\nExecution Info:")
    lines.append(
        f"  Max execution count: {analysis['execution_info']['max_execution_count']}"
    )
    lines.append(
        f"  Cells with output: {analysis['execution_info']['cells_with_output']}"
    )

    # Imports
    if analysis["imports"]:
        lines.append(f"\nImports ({len(analysis['imports'])}):")
        lines.append(f"  {', '.join(sorted(analysis['imports']))}")

    # Functions
    if analysis["functions"]:
        lines.append(f"\nFunctions ({len(analysis['functions'])}):")
        for func in analysis["functions"]:
            lines.append(f"  - {func['name']} (cell {func['cell_index']})")

    # Classes
    if analysis["classes"]:
        lines.append(f"\nClasses ({len(analysis['classes'])}):")
        for cls in analysis["classes"]:
            lines.append(f"  - {cls['name']} (cell {cls['cell_index']})")

    # Sections
    if analysis["sections"]:
        lines.append(f"\nSections ({len(analysis['sections'])}):")
        for section in analysis["sections"]:
            indent = "  " * section["level"]
            lines.append(f"{indent}{section['title']}")

    # Issues
    if analysis["issues"]:
        lines.append(f"\nIssues Found ({len(analysis['issues'])}):")
        # Sort by severity
        severity_order = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3}
        sorted_issues = sorted(
            analysis["issues"], key=lambda x: severity_order.get(x["severity"], 4)
        )
        for issue in sorted_issues:
            lines.append(f"  [{issue['severity']}] {issue['type']}")
            lines.append(f"    {issue['message']}")
    else:
        lines.append("\nNo issues found!")

    return "\n".join(lines)


def main():
    parser = argparse.ArgumentParser(description="Analyze Jupyter notebook structure")
    parser.add_argument("notebook", help="Path to notebook file")
    parser.add_argument(
        "--output",
        "-o",
        choices=["json", "text"],
        default="text",
        help="Output format (default: text)",
    )

    args = parser.parse_args()

    try:
        analysis = analyze_notebook(args.notebook)

        if args.output == "json":
            print(json.dumps(analysis, indent=2))
        else:
            print(format_text_output(analysis))

    except FileNotFoundError as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Error analyzing notebook: {e}", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()