Statistical Analysis

# A/B Test Configuration Template
# Statistical testing framework configuration

# Experiment Metadata
experiment:
  name: "checkout_flow_optimization"
  description: "Test new checkout flow vs current"
  owner: "product-team"
  start_date: "2024-01-01"
  end_date: "2024-01-14"

# Hypothesis
hypothesis:
  null: "New checkout flow has no effect on conversion rate"
  alternative: "New checkout flow increases conversion rate"
  type: "two-tailed"  # one-tailed, two-tailed

# Test Configuration
test_config:
  # Statistical parameters
  significance_level: 0.05  # Alpha (Type I error rate)
  power: 0.80  # 1 - Beta (Type II error rate)
  minimum_detectable_effect: 0.05  # 5% relative change

  # Sample size calculation
  baseline_conversion_rate: 0.10  # 10% current conversion
  expected_improvement: 0.02  # 2 percentage points

  # Test type
  test_type: "proportions"  # proportions, means, rates
  statistical_test: "chi_square"  # chi_square, t_test, z_test

# Variants
variants:
  - name: "control"
    description: "Current checkout flow"
    allocation: 0.50  # 50% of traffic

  - name: "treatment"
    description: "New streamlined checkout"
    allocation: 0.50  # 50% of traffic

# Metrics
metrics:
  primary:
    - name: "conversion_rate"
      description: "Proportion of users who complete purchase"
      type: "proportion"
      success_event: "purchase_completed"
      denominator_event: "checkout_started"

  secondary:
    - name: "average_order_value"
      description: "Average purchase amount"
      type: "continuous"

    - name: "time_to_purchase"
      description: "Time from checkout start to completion"
      type: "continuous"
      unit: "seconds"

  guardrail:
    - name: "error_rate"
      description: "Rate of checkout errors"
      type: "proportion"
      threshold_type: "upper"
      threshold: 0.02  # Alert if errors > 2%

# Segmentation
segmentation:
  dimensions:
    - "device_type"  # mobile, desktop, tablet
    - "user_type"    # new, returning
    - "country"
  analyze_segments: true

# Analysis Settings
analysis:
  # Correction for multiple comparisons
  multiple_comparison_correction: "bonferroni"  # bonferroni, holm, fdr

  # Confidence intervals
  confidence_level: 0.95

  # Bayesian analysis (optional)
  bayesian:
    enabled: true
    prior_type: "beta"
    prior_alpha: 1
    prior_beta: 1

  # Sequential analysis
  sequential:
    enabled: true
    check_frequency: "daily"
    spending_function: "obrien_fleming"

# Stopping Rules
stopping_rules:
  # Early stopping for success
  early_success:
    enabled: true
    threshold: 0.01  # Stop if p-value < 0.01

  # Early stopping for futility
  futility:
    enabled: true
    threshold: 0.50  # Stop if effect unlikely

  # Safety stopping
  safety:
    enabled: true
    metric: "error_rate"
    threshold: 0.05  # Stop if errors > 5%

# Reporting
reporting:
  format: "html"
  include_visualizations: true
  generate_summary: true
  recipients:
    - "product@company.com"
    - "data-science@company.com"

#!/usr/bin/env python3
"""
Statistical Hypothesis Testing Framework
Comprehensive statistical tests for data science
"""

import numpy as np
from scipy import stats
from typing import Dict, Tuple, Optional, List
from dataclasses import dataclass
from enum import Enum


class TestType(Enum):
    ONE_TAILED_LEFT = "one-tailed-left"
    ONE_TAILED_RIGHT = "one-tailed-right"
    TWO_TAILED = "two-tailed"


@dataclass
class TestResult:
    test_name: str
    statistic: float
    p_value: float
    significance_level: float
    is_significant: bool
    conclusion: str
    effect_size: Optional[float] = None
    confidence_interval: Optional[Tuple[float, float]] = None


class HypothesisTester:
    """Statistical hypothesis testing framework."""

    def __init__(self, alpha: float = 0.05):
        self.alpha = alpha

    def t_test_independent(self, group1: np.ndarray, group2: np.ndarray,
                          test_type: TestType = TestType.TWO_TAILED) -> TestResult:
        """
        Independent samples t-test.
        Compare means of two independent groups.
        """
        t_stat, p_value = stats.ttest_ind(group1, group2)

        # Adjust p-value for one-tailed tests
        if test_type == TestType.ONE_TAILED_LEFT:
            p_value = p_value / 2 if t_stat < 0 else 1 - p_value / 2
        elif test_type == TestType.ONE_TAILED_RIGHT:
            p_value = p_value / 2 if t_stat > 0 else 1 - p_value / 2

        # Cohen's d effect size
        pooled_std = np.sqrt((np.var(group1) + np.var(group2)) / 2)
        cohens_d = (np.mean(group1) - np.mean(group2)) / pooled_std

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="Independent Samples t-test",
            statistic=t_stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=cohens_d,
            conclusion=self._generate_conclusion(is_significant, "means")
        )

    def t_test_paired(self, before: np.ndarray, after: np.ndarray,
                     test_type: TestType = TestType.TWO_TAILED) -> TestResult:
        """
        Paired samples t-test.
        Compare means of related samples (before/after).
        """
        t_stat, p_value = stats.ttest_rel(before, after)

        if test_type == TestType.ONE_TAILED_LEFT:
            p_value = p_value / 2 if t_stat < 0 else 1 - p_value / 2
        elif test_type == TestType.ONE_TAILED_RIGHT:
            p_value = p_value / 2 if t_stat > 0 else 1 - p_value / 2

        # Effect size for paired data
        diff = before - after
        cohens_d = np.mean(diff) / np.std(diff)

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="Paired Samples t-test",
            statistic=t_stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=cohens_d,
            conclusion=self._generate_conclusion(is_significant, "means")
        )

    def chi_square_test(self, observed: np.ndarray) -> TestResult:
        """
        Chi-square test for independence.
        Test association between categorical variables.
        """
        chi2, p_value, dof, expected = stats.chi2_contingency(observed)

        # Cramer's V effect size
        n = observed.sum()
        min_dim = min(observed.shape[0] - 1, observed.shape[1] - 1)
        cramers_v = np.sqrt(chi2 / (n * min_dim)) if min_dim > 0 else 0

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="Chi-square Test of Independence",
            statistic=chi2,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=cramers_v,
            conclusion=self._generate_conclusion(is_significant, "association")
        )

    def anova_one_way(self, *groups) -> TestResult:
        """
        One-way ANOVA.
        Compare means across multiple groups.
        """
        f_stat, p_value = stats.f_oneway(*groups)

        # Eta-squared effect size
        all_data = np.concatenate(groups)
        grand_mean = np.mean(all_data)
        ss_between = sum(len(g) * (np.mean(g) - grand_mean)**2 for g in groups)
        ss_total = np.sum((all_data - grand_mean)**2)
        eta_squared = ss_between / ss_total if ss_total > 0 else 0

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="One-way ANOVA",
            statistic=f_stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=eta_squared,
            conclusion=self._generate_conclusion(is_significant, "group means")
        )

    def mann_whitney_u(self, group1: np.ndarray, group2: np.ndarray) -> TestResult:
        """
        Mann-Whitney U test (non-parametric alternative to t-test).
        Compare distributions of two independent groups.
        """
        u_stat, p_value = stats.mannwhitneyu(group1, group2, alternative='two-sided')

        # Effect size (rank-biserial correlation)
        n1, n2 = len(group1), len(group2)
        effect_size = 1 - (2 * u_stat) / (n1 * n2)

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="Mann-Whitney U Test",
            statistic=u_stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=effect_size,
            conclusion=self._generate_conclusion(is_significant, "distributions")
        )

    def proportion_z_test(self, successes1: int, n1: int,
                         successes2: int, n2: int) -> TestResult:
        """
        Two-proportion z-test.
        Compare proportions between two groups (A/B testing).
        """
        p1 = successes1 / n1
        p2 = successes2 / n2
        p_pooled = (successes1 + successes2) / (n1 + n2)

        se = np.sqrt(p_pooled * (1 - p_pooled) * (1/n1 + 1/n2))
        z_stat = (p1 - p2) / se if se > 0 else 0
        p_value = 2 * (1 - stats.norm.cdf(abs(z_stat)))

        # 95% CI for difference
        se_diff = np.sqrt(p1*(1-p1)/n1 + p2*(1-p2)/n2)
        ci = (p1 - p2 - 1.96*se_diff, p1 - p2 + 1.96*se_diff)

        is_significant = p_value < self.alpha

        return TestResult(
            test_name="Two-Proportion Z-test",
            statistic=z_stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            effect_size=p1 - p2,
            confidence_interval=ci,
            conclusion=self._generate_conclusion(is_significant, "proportions")
        )

    def normality_test(self, data: np.ndarray) -> TestResult:
        """
        Shapiro-Wilk test for normality.
        Check if data follows normal distribution.
        """
        # Use Shapiro-Wilk for n < 5000, D'Agostino-Pearson otherwise
        if len(data) < 5000:
            stat, p_value = stats.shapiro(data)
            test_name = "Shapiro-Wilk Normality Test"
        else:
            stat, p_value = stats.normaltest(data)
            test_name = "D'Agostino-Pearson Normality Test"

        is_significant = p_value < self.alpha
        # Note: for normality, significant = NOT normal

        conclusion = (
            "Data is NOT normally distributed (reject normality)"
            if is_significant else
            "Data appears normally distributed (fail to reject normality)"
        )

        return TestResult(
            test_name=test_name,
            statistic=stat,
            p_value=p_value,
            significance_level=self.alpha,
            is_significant=is_significant,
            conclusion=conclusion
        )

    def _generate_conclusion(self, is_significant: bool, comparison_type: str) -> str:
        """Generate human-readable conclusion."""
        if is_significant:
            return f"Statistically significant difference in {comparison_type} (p < {self.alpha})"
        else:
            return f"No statistically significant difference in {comparison_type} (p >= {self.alpha})"


def calculate_sample_size(baseline_rate: float, mde: float,
                         alpha: float = 0.05, power: float = 0.8) -> int:
    """
    Calculate required sample size per group for A/B test.

    Args:
        baseline_rate: Current conversion rate
        mde: Minimum detectable effect (absolute)
        alpha: Significance level
        power: Statistical power
    """
    from scipy.stats import norm

    p1 = baseline_rate
    p2 = baseline_rate + mde
    p_avg = (p1 + p2) / 2

    z_alpha = norm.ppf(1 - alpha / 2)
    z_beta = norm.ppf(power)

    n = (2 * p_avg * (1 - p_avg) * (z_alpha + z_beta)**2) / (p2 - p1)**2

    return int(np.ceil(n))


def main():
    """Demo statistical tests."""
    print("Statistical Hypothesis Testing Demo")
    print("=" * 50)

    tester = HypothesisTester(alpha=0.05)

    # Example: A/B Test (proportion comparison)
    print("\n1. A/B Test Example:")
    print("-" * 30)

    # Control: 100 conversions out of 1000
    # Treatment: 120 conversions out of 1000
    result = tester.proportion_z_test(100, 1000, 120, 1000)
    print(f"Test: {result.test_name}")
    print(f"Z-statistic: {result.statistic:.4f}")
    print(f"P-value: {result.p_value:.4f}")
    print(f"Effect size: {result.effect_size:.4f}")
    print(f"95% CI: [{result.confidence_interval[0]:.4f}, {result.confidence_interval[1]:.4f}]")
    print(f"Conclusion: {result.conclusion}")

    # Example: Comparing two groups
    print("\n2. T-test Example:")
    print("-" * 30)

    group_a = np.random.normal(100, 15, 50)
    group_b = np.random.normal(110, 15, 50)

    result = tester.t_test_independent(group_a, group_b)
    print(f"Test: {result.test_name}")
    print(f"T-statistic: {result.statistic:.4f}")
    print(f"P-value: {result.p_value:.4f}")
    print(f"Cohen's d: {result.effect_size:.4f}")
    print(f"Conclusion: {result.conclusion}")

    # Sample size calculation
    print("\n3. Sample Size Calculation:")
    print("-" * 30)
    n = calculate_sample_size(baseline_rate=0.10, mde=0.02)
    print(f"Required sample size per group: {n:,}")
    print(f"(For detecting 2% absolute lift from 10% baseline)")


if __name__ == '__main__':
    main()