Data Analyst

export default async function data_analyst(input) {
  console.log("🧠 Running skill: data-analyst");
  
  // TODO: implement actual logic for this skill
  return {
    message: "Skill 'data-analyst' executed successfully!",
    input
  };
}

{
  "name": "@ai-labs-claude-skills/data-analyst",
  "version": "1.0.0",
  "description": "Claude AI skill: data-analyst",
  "main": "index.js",
  "files": [
    "."
  ],
  "license": "MIT",
  "author": "AI Labs"
}

pandas>=2.0.0
numpy>=1.24.0
scikit-learn>=1.3.0
plotly>=5.18.0
dash>=2.14.0
dash-bootstrap-components>=1.5.0

#!/usr/bin/env python3
"""
Analyze missing values in a dataset and suggest appropriate imputation strategies.
"""

import pandas as pd
import numpy as np
import json
import sys
from pathlib import Path


def detect_column_type(series):
    """Detect the semantic type of a column for better imputation."""
    # Remove missing values for analysis
    clean_series = series.dropna()

    if len(clean_series) == 0:
        return 'unknown'

    # Check if numeric
    if pd.api.types.is_numeric_dtype(series):
        # Check if it's likely categorical (few unique values)
        unique_ratio = len(clean_series.unique()) / len(clean_series)
        if unique_ratio < 0.05 and len(clean_series.unique()) < 20:
            return 'categorical_numeric'

        # Check if it's an ID column (monotonic or very high cardinality)
        if clean_series.is_monotonic_increasing or unique_ratio > 0.95:
            return 'id'

        # Check distribution characteristics
        skewness = clean_series.skew()
        if abs(skewness) < 0.5:
            return 'numeric_normal'
        else:
            return 'numeric_skewed'

    # Check if datetime
    elif pd.api.types.is_datetime64_any_dtype(series):
        return 'datetime'

    # Check if categorical/object
    elif pd.api.types.is_object_dtype(series) or pd.api.types.is_categorical_dtype(series):
        unique_ratio = len(clean_series.unique()) / len(clean_series)

        if unique_ratio > 0.9:
            return 'text_unique'
        elif unique_ratio < 0.1:
            return 'categorical_low_cardinality'
        else:
            return 'categorical_medium_cardinality'

    return 'unknown'


def suggest_imputation_strategy(series, col_type):
    """Suggest the best imputation strategy based on column type and data characteristics."""
    missing_pct = (series.isna().sum() / len(series)) * 100

    strategies = {
        'method': None,
        'reasoning': '',
        'alternative': None
    }

    # High missing rate - consider dropping or special handling
    if missing_pct > 50:
        strategies['method'] = 'drop_or_flag'
        strategies['reasoning'] = f'{missing_pct:.1f}% missing - consider dropping column or creating missing indicator'
        return strategies

    # Strategy based on column type
    if col_type == 'numeric_normal':
        strategies['method'] = 'mean'
        strategies['reasoning'] = 'Normally distributed numeric data - mean imputation appropriate'
        strategies['alternative'] = 'median'

    elif col_type == 'numeric_skewed':
        strategies['method'] = 'median'
        strategies['reasoning'] = 'Skewed numeric data - median more robust than mean'
        strategies['alternative'] = 'knn'

    elif col_type == 'categorical_numeric':
        strategies['method'] = 'mode'
        strategies['reasoning'] = 'Numeric data with categorical nature - mode imputation'
        strategies['alternative'] = 'most_frequent'

    elif col_type in ['categorical_low_cardinality', 'categorical_medium_cardinality']:
        strategies['method'] = 'mode'
        strategies['reasoning'] = 'Categorical data - mode (most frequent) imputation'
        strategies['alternative'] = 'constant:Unknown'

    elif col_type == 'text_unique':
        strategies['method'] = 'constant'
        strategies['reasoning'] = 'High cardinality text - impute with constant value'
        strategies['alternative'] = 'drop_rows'

    elif col_type == 'datetime':
        strategies['method'] = 'forward_fill'
        strategies['reasoning'] = 'DateTime data - forward fill (carry last observation forward)'
        strategies['alternative'] = 'interpolate'

    elif col_type == 'id':
        strategies['method'] = 'drop_rows'
        strategies['reasoning'] = 'ID column - cannot impute, drop rows with missing IDs'
        strategies['alternative'] = 'generate_sequence'

    else:
        strategies['method'] = 'knn'
        strategies['reasoning'] = 'Unknown type - use KNN imputation based on similar rows'
        strategies['alternative'] = 'drop_rows'

    return strategies


def analyze_missing_values(filepath, output_json=None):
    """
    Analyze missing values in a CSV file and generate a comprehensive report.

    Args:
        filepath: Path to the CSV file
        output_json: Optional path to save analysis results as JSON

    Returns:
        Dictionary containing analysis results
    """
    # Load data
    df = pd.read_csv(filepath)

    # Calculate overall statistics
    total_rows = len(df)
    total_cells = df.size
    missing_cells = df.isna().sum().sum()
    missing_pct = (missing_cells / total_cells) * 100

    # Analyze each column
    column_analysis = {}

    for col in df.columns:
        missing_count = df[col].isna().sum()
        missing_pct_col = (missing_count / total_rows) * 100

        if missing_count > 0:
            col_type = detect_column_type(df[col])
            strategy = suggest_imputation_strategy(df[col], col_type)

            # Calculate statistics for non-missing values
            clean_data = df[col].dropna()
            stats = {}

            if pd.api.types.is_numeric_dtype(df[col]):
                stats = {
                    'mean': float(clean_data.mean()),
                    'median': float(clean_data.median()),
                    'std': float(clean_data.std()),
                    'min': float(clean_data.min()),
                    'max': float(clean_data.max())
                }
            elif pd.api.types.is_object_dtype(df[col]) or pd.api.types.is_categorical_dtype(df[col]):
                value_counts = clean_data.value_counts().head(5)
                stats = {
                    'unique_values': int(clean_data.nunique()),
                    'most_common': value_counts.to_dict()
                }

            column_analysis[col] = {
                'missing_count': int(missing_count),
                'missing_percentage': float(missing_pct_col),
                'data_type': str(df[col].dtype),
                'detected_type': col_type,
                'total_values': int(total_rows),
                'non_missing_values': int(total_rows - missing_count),
                'statistics': stats,
                'imputation_strategy': strategy
            }

    # Build complete report
    report = {
        'file': str(filepath),
        'total_rows': int(total_rows),
        'total_columns': int(len(df.columns)),
        'total_cells': int(total_cells),
        'missing_cells': int(missing_cells),
        'missing_percentage': float(missing_pct),
        'columns_with_missing': len(column_analysis),
        'column_analysis': column_analysis
    }

    # Save to JSON if requested
    if output_json:
        with open(output_json, 'w') as f:
            json.dump(report, f, indent=2)
        print(f"Analysis saved to: {output_json}")

    return report


def print_report(report):
    """Print a formatted report to console."""
    print("\n" + "="*80)
    print("MISSING VALUES ANALYSIS REPORT")
    print("="*80)
    print(f"\nFile: {report['file']}")
    print(f"Dimensions: {report['total_rows']} rows × {report['total_columns']} columns")
    print(f"\nOverall Missing Data:")
    print(f"  - Missing cells: {report['missing_cells']:,} / {report['total_cells']:,}")
    print(f"  - Missing percentage: {report['missing_percentage']:.2f}%")
    print(f"  - Columns with missing values: {report['columns_with_missing']}")

    if report['columns_with_missing'] > 0:
        print("\n" + "-"*80)
        print("COLUMN-BY-COLUMN ANALYSIS")
        print("-"*80)

        for col, analysis in report['column_analysis'].items():
            print(f"\n📊 Column: {col}")
            print(f"   Type: {analysis['data_type']} (detected as: {analysis['detected_type']})")
            print(f"   Missing: {analysis['missing_count']:,} / {analysis['total_values']:,} ({analysis['missing_percentage']:.2f}%)")

            # Print statistics
            if analysis['statistics']:
                print(f"   Statistics:")
                for key, value in analysis['statistics'].items():
                    if isinstance(value, float):
                        print(f"     - {key}: {value:.4f}")
                    elif isinstance(value, dict) and key == 'most_common':
                        print(f"     - {key}:")
                        for val, count in value.items():
                            print(f"       • {val}: {count}")
                    else:
                        print(f"     - {key}: {value}")

            # Print imputation strategy
            strategy = analysis['imputation_strategy']
            print(f"   ✅ Recommended: {strategy['method']}")
            print(f"      Reason: {strategy['reasoning']}")
            if strategy['alternative']:
                print(f"      Alternative: {strategy['alternative']}")
    else:
        print("\n✅ No missing values found in the dataset!")

    print("\n" + "="*80 + "\n")


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python analyze_missing_values.py <csv_file> [output_json]")
        sys.exit(1)

    filepath = sys.argv[1]
    output_json = sys.argv[2] if len(sys.argv) > 2 else None

    if not Path(filepath).exists():
        print(f"Error: File not found: {filepath}")
        sys.exit(1)

    report = analyze_missing_values(filepath, output_json)
    print_report(report)

#!/usr/bin/env python3
"""
Create an interactive Plotly Dash dashboard for data visualization and trend analysis.
"""

import pandas as pd
import numpy as np
import sys
import json
from pathlib import Path
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import dash
from dash import dcc, html, Input, Output, callback
import dash_bootstrap_components as dbc


def detect_time_column(df):
    """Detect potential time/date columns in the dataframe."""
    time_cols = []
    for col in df.columns:
        # Check if column is datetime type
        if pd.api.types.is_datetime64_any_dtype(df[col]):
            time_cols.append(col)
        # Try to parse as datetime
        elif pd.api.types.is_object_dtype(df[col]):
            try:
                pd.to_datetime(df[col])
                time_cols.append(col)
            except:
                pass
    return time_cols


def detect_numeric_columns(df):
    """Get all numeric columns suitable for visualization."""
    return [col for col in df.columns if pd.api.types.is_numeric_dtype(df[col])]


def detect_categorical_columns(df, max_categories=20):
    """Get categorical columns with reasonable cardinality."""
    cat_cols = []
    for col in df.columns:
        if pd.api.types.is_object_dtype(df[col]) or pd.api.types.is_categorical_dtype(df[col]):
            if df[col].nunique() <= max_categories:
                cat_cols.append(col)
    return cat_cols


def create_time_series_plot(df, time_col, numeric_cols):
    """Create time series visualization for trend analysis."""
    # Convert time column to datetime
    df[time_col] = pd.to_datetime(df[time_col])
    df = df.sort_values(time_col)

    # Create subplot for each numeric column
    n_cols = len(numeric_cols)
    fig = make_subplots(
        rows=min(n_cols, 4),
        cols=1,
        subplot_titles=[f"{col} Over Time" for col in numeric_cols[:4]],
        vertical_spacing=0.1
    )

    for idx, col in enumerate(numeric_cols[:4]):  # Limit to 4 for readability
        row = idx + 1
        fig.add_trace(
            go.Scatter(
                x=df[time_col],
                y=df[col],
                mode='lines+markers',
                name=col,
                line=dict(width=2),
                marker=dict(size=4)
            ),
            row=row,
            col=1
        )

    fig.update_layout(
        height=300 * min(n_cols, 4),
        showlegend=True,
        title_text="Time Series Trends",
        hovermode='x unified'
    )

    fig.update_xaxes(title_text="Date", row=min(n_cols, 4), col=1)

    return fig


def create_distribution_plots(df, numeric_cols):
    """Create distribution plots (histograms and box plots)."""
    n_cols = len(numeric_cols)
    cols_per_row = 2
    n_rows = (n_cols + cols_per_row - 1) // cols_per_row

    fig = make_subplots(
        rows=n_rows,
        cols=cols_per_row,
        subplot_titles=[col for col in numeric_cols],
        vertical_spacing=0.1,
        horizontal_spacing=0.1
    )

    for idx, col in enumerate(numeric_cols):
        row = (idx // cols_per_row) + 1
        col_pos = (idx % cols_per_row) + 1

        fig.add_trace(
            go.Histogram(
                x=df[col],
                name=col,
                nbinsx=30,
                marker_color='lightblue',
                opacity=0.7
            ),
            row=row,
            col=col_pos
        )

    fig.update_layout(
        height=300 * n_rows,
        showlegend=False,
        title_text="Distribution Analysis"
    )

    return fig


def create_correlation_heatmap(df, numeric_cols):
    """Create correlation heatmap for numeric columns."""
    if len(numeric_cols) < 2:
        return None

    corr_matrix = df[numeric_cols].corr()

    fig = go.Figure(data=go.Heatmap(
        z=corr_matrix.values,
        x=corr_matrix.columns,
        y=corr_matrix.columns,
        colorscale='RdBu',
        zmid=0,
        text=corr_matrix.values.round(2),
        texttemplate='%{text}',
        textfont={"size": 10},
        colorbar=dict(title="Correlation")
    ))

    fig.update_layout(
        title="Correlation Heatmap",
        height=max(400, len(numeric_cols) * 40),
        xaxis={'side': 'bottom'}
    )

    return fig


def create_categorical_analysis(df, cat_cols, numeric_col=None):
    """Create categorical analysis plots."""
    if not cat_cols:
        return None

    n_cols = len(cat_cols)
    fig = make_subplots(
        rows=n_cols,
        cols=1,
        subplot_titles=[f"Distribution of {col}" for col in cat_cols],
        vertical_spacing=0.1
    )

    for idx, col in enumerate(cat_cols):
        row = idx + 1
        value_counts = df[col].value_counts().head(10)  # Top 10 categories

        fig.add_trace(
            go.Bar(
                x=value_counts.index,
                y=value_counts.values,
                name=col,
                marker_color='steelblue'
            ),
            row=row,
            col=1
        )

    fig.update_layout(
        height=300 * n_cols,
        showlegend=False,
        title_text="Categorical Variable Analysis"
    )

    return fig


def create_scatter_matrix(df, numeric_cols):
    """Create scatter plot matrix for relationships between variables."""
    if len(numeric_cols) < 2:
        return None

    # Limit to 5 columns for readability
    cols_to_plot = numeric_cols[:5]

    fig = px.scatter_matrix(
        df,
        dimensions=cols_to_plot,
        title="Scatter Plot Matrix - Variable Relationships"
    )

    fig.update_traces(diagonal_visible=False, showupperhalf=False)
    fig.update_layout(height=800)

    return fig


def create_summary_statistics_table(df):
    """Create a summary statistics table."""
    numeric_cols = detect_numeric_columns(df)

    if not numeric_cols:
        return None

    stats_df = df[numeric_cols].describe().round(2).T
    stats_df['missing'] = df[numeric_cols].isna().sum()
    stats_df['missing_pct'] = (stats_df['missing'] / len(df) * 100).round(2)

    fig = go.Figure(data=[go.Table(
        header=dict(
            values=['Variable'] + list(stats_df.columns),
            fill_color='paleturquoise',
            align='left',
            font=dict(size=12, color='black')
        ),
        cells=dict(
            values=[stats_df.index] + [stats_df[col] for col in stats_df.columns],
            fill_color='lavender',
            align='left',
            font=dict(size=11)
        )
    )])

    fig.update_layout(
        title="Summary Statistics",
        height=min(600, 50 + len(numeric_cols) * 30)
    )

    return fig


def build_dashboard(filepath, output_dir=None, port=8050):
    """
    Build and launch an interactive Plotly Dash dashboard.

    Args:
        filepath: Path to the CSV file
        output_dir: Directory to save dashboard files (optional)
        port: Port to run the dashboard server (default: 8050)
    """
    # Load data
    df = pd.read_csv(filepath)

    # Detect column types
    numeric_cols = detect_numeric_columns(df)
    cat_cols = detect_categorical_columns(df)
    time_cols = detect_time_column(df)

    print(f"\nDataset loaded: {len(df)} rows, {len(df.columns)} columns")
    print(f"Numeric columns: {len(numeric_cols)}")
    print(f"Categorical columns: {len(cat_cols)}")
    print(f"Time columns: {len(time_cols)}")

    # Create visualizations
    plots = {}

    # Summary statistics
    plots['summary'] = create_summary_statistics_table(df)

    # Time series (if time column exists)
    if time_cols and numeric_cols:
        plots['timeseries'] = create_time_series_plot(df, time_cols[0], numeric_cols)

    # Distribution plots
    if numeric_cols:
        plots['distributions'] = create_distribution_plots(df, numeric_cols[:6])

    # Correlation heatmap
    if len(numeric_cols) >= 2:
        plots['correlation'] = create_correlation_heatmap(df, numeric_cols)

    # Categorical analysis
    if cat_cols:
        plots['categorical'] = create_categorical_analysis(df, cat_cols[:5])

    # Scatter matrix
    if len(numeric_cols) >= 2:
        plots['scatter_matrix'] = create_scatter_matrix(df, numeric_cols)

    # Initialize Dash app
    app = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP])

    # Create layout
    app.layout = dbc.Container([
        dbc.Row([
            dbc.Col([
                html.H1("Data Analysis Dashboard", className="text-center mb-4"),
                html.Hr()
            ])
        ]),

        # Dataset info
        dbc.Row([
            dbc.Col([
                dbc.Card([
                    dbc.CardBody([
                        html.H4("Dataset Overview", className="card-title"),
                        html.P(f"File: {Path(filepath).name}"),
                        html.P(f"Rows: {len(df):,} | Columns: {len(df.columns)}"),
                        html.P(f"Numeric: {len(numeric_cols)} | Categorical: {len(cat_cols)} | Time: {len(time_cols)}")
                    ])
                ], className="mb-4")
            ])
        ]),

        # Summary statistics
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['summary']) if plots.get('summary') else html.Div()
            ])
        ], className="mb-4"),

        # Time series
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['timeseries']) if plots.get('timeseries') else html.Div()
            ])
        ], className="mb-4") if plots.get('timeseries') else html.Div(),

        # Distributions
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['distributions']) if plots.get('distributions') else html.Div()
            ])
        ], className="mb-4"),

        # Correlation heatmap
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['correlation']) if plots.get('correlation') else html.Div()
            ])
        ], className="mb-4") if plots.get('correlation') else html.Div(),

        # Categorical analysis
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['categorical']) if plots.get('categorical') else html.Div()
            ])
        ], className="mb-4") if plots.get('categorical') else html.Div(),

        # Scatter matrix
        dbc.Row([
            dbc.Col([
                dcc.Graph(figure=plots['scatter_matrix']) if plots.get('scatter_matrix') else html.Div()
            ])
        ], className="mb-4") if plots.get('scatter_matrix') else html.Div(),

        # Footer
        dbc.Row([
            dbc.Col([
                html.Hr(),
                html.P("Interactive Data Analysis Dashboard | Generated with Plotly Dash",
                       className="text-center text-muted")
            ])
        ])
    ], fluid=True)

    # Save standalone HTML if output_dir specified
    if output_dir:
        output_dir = Path(output_dir)
        output_dir.mkdir(parents=True, exist_ok=True)

        # Save individual plots as HTML
        for plot_name, fig in plots.items():
            if fig:
                html_file = output_dir / f"{plot_name}.html"
                fig.write_html(html_file)
                print(f"Saved: {html_file}")

        print(f"\nStatic visualizations saved to: {output_dir}")

    # Run server
    print(f"\n{'='*80}")
    print(f"Starting dashboard server...")
    print(f"Access the dashboard at: http://127.0.0.1:{port}")
    print(f"Press Ctrl+C to stop the server")
    print(f"{'='*80}\n")

    app.run_server(debug=False, port=port)


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python create_dashboard.py <csv_file> [output_dir] [port]")
        print("\nExample:")
        print("  python create_dashboard.py data.csv")
        print("  python create_dashboard.py data.csv ./visualizations 8050")
        sys.exit(1)

    filepath = sys.argv[1]
    output_dir = sys.argv[2] if len(sys.argv) > 2 else None
    port = int(sys.argv[3]) if len(sys.argv) > 3 else 8050

    if not Path(filepath).exists():
        print(f"Error: File not found: {filepath}")
        sys.exit(1)

    build_dashboard(filepath, output_dir, port)

#!/usr/bin/env python3
"""
Automatically impute missing values in a dataset based on analysis.
"""

import pandas as pd
import numpy as np
import json
import sys
from pathlib import Path
from sklearn.impute import KNNImputer
from sklearn.preprocessing import LabelEncoder


def impute_column(df, col, method, params=None):
    """
    Impute missing values in a specific column using the specified method.

    Args:
        df: DataFrame
        col: Column name
        method: Imputation method
        params: Additional parameters for the method

    Returns:
        Series with imputed values
    """
    series = df[col].copy()
    params = params or {}

    if method == 'mean':
        return series.fillna(series.mean())

    elif method == 'median':
        return series.fillna(series.median())

    elif method == 'mode':
        mode_value = series.mode()
        if len(mode_value) > 0:
            return series.fillna(mode_value[0])
        return series

    elif method == 'most_frequent':
        # Same as mode but explicitly for categorical
        most_frequent = series.value_counts().idxmax() if len(series.value_counts()) > 0 else None
        return series.fillna(most_frequent)

    elif method == 'constant':
        fill_value = params.get('fill_value', 'Unknown')
        return series.fillna(fill_value)

    elif method == 'forward_fill':
        return series.fillna(method='ffill')

    elif method == 'backward_fill':
        return series.fillna(method='bfill')

    elif method == 'interpolate':
        if pd.api.types.is_numeric_dtype(series):
            return series.interpolate(method='linear')
        else:
            return series.fillna(method='ffill')  # Fallback for non-numeric

    elif method == 'knn':
        # KNN imputation requires numeric data
        # For mixed datasets, we'll use KNN on numeric columns only
        return series  # Will be handled by batch KNN imputation

    elif method == 'drop_rows':
        # Mark for row deletion (handled at DataFrame level)
        return series

    elif method == 'drop_or_flag':
        # For columns with too many missing values
        # Option to create a missing indicator instead of dropping
        if params.get('create_indicator', False):
            return series
        return series

    else:
        print(f"Warning: Unknown imputation method '{method}' for column '{col}'. Skipping.")
        return series


def batch_knn_imputation(df, columns_for_knn, n_neighbors=5):
    """
    Perform KNN imputation on multiple numeric columns simultaneously.

    Args:
        df: DataFrame
        columns_for_knn: List of columns to impute using KNN
        n_neighbors: Number of neighbors for KNN

    Returns:
        DataFrame with KNN-imputed values
    """
    if not columns_for_knn:
        return df

    # Select only numeric columns for KNN
    numeric_cols = [col for col in columns_for_knn if pd.api.types.is_numeric_dtype(df[col])]

    if not numeric_cols:
        return df

    # Perform KNN imputation
    imputer = KNNImputer(n_neighbors=n_neighbors)
    df[numeric_cols] = imputer.fit_transform(df[numeric_cols])

    return df


def impute_missing_values(filepath, analysis_json=None, output_file=None, create_missing_indicators=False):
    """
    Automatically impute missing values based on analysis results.

    Args:
        filepath: Path to the CSV file
        analysis_json: Path to analysis JSON (optional, will analyze if not provided)
        output_file: Path to save imputed data (optional)
        create_missing_indicators: Whether to create binary columns indicating missingness

    Returns:
        DataFrame with imputed values and imputation report
    """
    # Load data
    df = pd.read_csv(filepath)
    original_rows = len(df)

    # Load or generate analysis
    if analysis_json and Path(analysis_json).exists():
        with open(analysis_json, 'r') as f:
            analysis = json.load(f)
    else:
        # Import and run analysis
        from analyze_missing_values import analyze_missing_values
        analysis = analyze_missing_values(filepath)

    # Track imputation actions
    imputation_log = {}
    columns_to_drop = []
    rows_to_drop_mask = pd.Series([False] * len(df))
    knn_columns = []

    # Create missing indicators if requested
    if create_missing_indicators:
        for col in analysis['column_analysis'].keys():
            indicator_col = f'{col}_was_missing'
            df[indicator_col] = df[col].isna().astype(int)

    # Process each column with missing values
    for col, col_analysis in analysis['column_analysis'].items():
        strategy = col_analysis['imputation_strategy']
        method = strategy['method']

        imputation_log[col] = {
            'method': method,
            'missing_before': col_analysis['missing_count'],
            'reasoning': strategy['reasoning']
        }

        if method == 'drop_or_flag':
            if col_analysis['missing_percentage'] > 70:
                columns_to_drop.append(col)
                imputation_log[col]['action'] = f"Dropped column ({col_analysis['missing_percentage']:.1f}% missing)"
            else:
                # Keep column, user can decide
                imputation_log[col]['action'] = f"Flagged for review ({col_analysis['missing_percentage']:.1f}% missing)"

        elif method == 'drop_rows':
            rows_to_drop_mask |= df[col].isna()
            imputation_log[col]['action'] = "Marked rows for deletion"

        elif method == 'knn':
            knn_columns.append(col)
            imputation_log[col]['action'] = "Queued for KNN imputation"

        elif method.startswith('constant:'):
            # Extract the constant value
            fill_value = method.split(':', 1)[1]
            df[col] = impute_column(df, col, 'constant', {'fill_value': fill_value})
            imputation_log[col]['action'] = f"Filled with constant: {fill_value}"
            imputation_log[col]['missing_after'] = df[col].isna().sum()

        else:
            # Standard imputation methods
            df[col] = impute_column(df, col, method)
            imputation_log[col]['action'] = f"Imputed using {method}"
            imputation_log[col]['missing_after'] = df[col].isna().sum()

    # Perform batch KNN imputation
    if knn_columns:
        print(f"Performing KNN imputation on {len(knn_columns)} columns...")
        df = batch_knn_imputation(df, knn_columns)
        for col in knn_columns:
            imputation_log[col]['action'] = "Imputed using KNN"
            imputation_log[col]['missing_after'] = df[col].isna().sum()

    # Drop columns marked for deletion
    if columns_to_drop:
        df = df.drop(columns=columns_to_drop)
        print(f"Dropped {len(columns_to_drop)} columns with excessive missing values")

    # Drop rows marked for deletion
    if rows_to_drop_mask.any():
        rows_to_drop = rows_to_drop_mask.sum()
        df = df[~rows_to_drop_mask].reset_index(drop=True)
        print(f"Dropped {rows_to_drop} rows with missing critical values")

    # Generate report
    report = {
        'input_file': str(filepath),
        'original_rows': original_rows,
        'final_rows': len(df),
        'rows_dropped': original_rows - len(df),
        'columns_dropped': len(columns_to_drop),
        'columns_imputed': len([c for c in imputation_log if 'missing_after' in imputation_log[c]]),
        'imputation_log': imputation_log
    }

    # Save imputed data
    if output_file:
        df.to_csv(output_file, index=False)
        print(f"\nImputed data saved to: {output_file}")
        report['output_file'] = str(output_file)

    return df, report


def print_imputation_report(report):
    """Print a formatted imputation report."""
    print("\n" + "="*80)
    print("IMPUTATION REPORT")
    print("="*80)
    print(f"\nInput file: {report['input_file']}")
    if 'output_file' in report:
        print(f"Output file: {report['output_file']}")

    print(f"\nData dimensions:")
    print(f"  - Original rows: {report['original_rows']:,}")
    print(f"  - Final rows: {report['final_rows']:,}")
    if report['rows_dropped'] > 0:
        print(f"  - Rows dropped: {report['rows_dropped']:,}")

    print(f"\nImputation summary:")
    print(f"  - Columns imputed: {report['columns_imputed']}")
    if report['columns_dropped'] > 0:
        print(f"  - Columns dropped: {report['columns_dropped']}")

    print("\n" + "-"*80)
    print("COLUMN-BY-COLUMN IMPUTATION LOG")
    print("-"*80)

    for col, log in report['imputation_log'].items():
        print(f"\n📊 Column: {col}")
        print(f"   Method: {log['method']}")
        print(f"   Missing before: {log['missing_before']}")
        if 'missing_after' in log:
            print(f"   Missing after: {log['missing_after']}")
        print(f"   Action: {log['action']}")
        print(f"   Reasoning: {log['reasoning']}")

    print("\n" + "="*80 + "\n")


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python impute_missing_values.py <csv_file> [analysis_json] [output_file]")
        print("\nExample:")
        print("  python impute_missing_values.py data.csv")
        print("  python impute_missing_values.py data.csv analysis.json data_imputed.csv")
        sys.exit(1)

    filepath = sys.argv[1]
    analysis_json = sys.argv[2] if len(sys.argv) > 2 else None
    output_file = sys.argv[3] if len(sys.argv) > 3 else filepath.replace('.csv', '_imputed.csv')

    if not Path(filepath).exists():
        print(f"Error: File not found: {filepath}")
        sys.exit(1)

    print("Starting automatic imputation...")
    df_imputed, report = impute_missing_values(
        filepath,
        analysis_json=analysis_json,
        output_file=output_file,
        create_missing_indicators=False
    )

    print_imputation_report(report)

    # Save report as JSON
    report_json = output_file.replace('.csv', '_report.json')
    with open(report_json, 'w') as f:
        json.dump(report, f, indent=2)
    print(f"Detailed report saved to: {report_json}")