Planning Disaster Recovery

#!/bin/bash
#
# Database Failover Chaos Test
#
# Purpose: Simulate database primary failure to validate failover mechanisms,
# replica promotion, and application resilience during database outages.
#
# Requirements:
# - PostgreSQL/MySQL client tools (psql, mysql)
# - SSH access to database servers (for direct testing)
# - jq for JSON parsing
# - Prometheus endpoint (optional, for metrics)
#
# Usage:
#   ./db-failover-test.sh --mode postgres --primary-host db1.example.com --secondary-host db2.example.com
#   ./db-failover-test.sh --mode mysql --primary-host mysql-primary --secondary-host mysql-replica
#   ./db-failover-test.sh --mode aws-rds --db-cluster prod-cluster --region us-east-1
#

set -euo pipefail

# Default configuration
MODE="${MODE:-postgres}"
DRY_RUN="${DRY_RUN:-false}"
TIMEOUT="${TIMEOUT:-60}"  # 1 minute
PROMETHEUS_URL="${PROMETHEUS_URL:-}"
LOG_FILE="/tmp/db-failover-test-$(date +%Y%m%d-%H%M%S).log"

# Database credentials (prefer environment variables or secrets manager)
DB_USER="${DB_USER:-postgres}"
DB_PASSWORD="${DB_PASSWORD:-}"
DB_NAME="${DB_NAME:-postgres}"
DB_PORT="${DB_PORT:-5432}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging function
log() {
    local level="$1"
    shift
    local message="$*"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${timestamp} [${level}] ${message}" | tee -a "$LOG_FILE"
}

log_info() { log "${BLUE}INFO${NC}" "$@"; }
log_success() { log "${GREEN}SUCCESS${NC}" "$@"; }
log_warning() { log "${YELLOW}WARNING${NC}" "$@"; }
log_error() { log "${RED}ERROR${NC}" "$@"; }

# Usage information
usage() {
    cat <<EOF
Usage: $0 [OPTIONS]

Chaos engineering test to simulate database failover and validate replica promotion.

OPTIONS:
    --mode MODE              Database mode: 'postgres', 'mysql', or 'aws-rds' (default: postgres)
    --primary-host HOST      Primary database host
    --secondary-host HOST    Secondary/replica database host
    --db-cluster CLUSTER     RDS cluster identifier (AWS RDS mode)
    --region REGION          AWS region (AWS RDS mode)
    --db-user USER           Database user (default: postgres)
    --db-password PASS       Database password (or use DB_PASSWORD env var)
    --db-name NAME           Database name (default: postgres)
    --db-port PORT           Database port (default: 5432 for postgres, 3306 for mysql)
    --timeout SECONDS        Maximum failover time (default: 60)
    --failure-method METHOD  How to fail primary: 'stop-service', 'kill-process', 'network' (default: stop-service)
    --dry-run                Show what would be done without executing
    --prometheus-url URL     Prometheus endpoint for metrics collection
    --skip-confirmation      Skip safety confirmation prompt
    --help                   Show this help message

EXAMPLES:
    # PostgreSQL with streaming replication
    $0 --mode postgres --primary-host db1.internal --secondary-host db2.internal

    # MySQL with async replication
    $0 --mode mysql --primary-host mysql-primary --secondary-host mysql-replica --db-port 3306

    # AWS RDS Multi-AZ
    $0 --mode aws-rds --db-cluster prod-aurora --region us-east-1

ENVIRONMENT VARIABLES:
    DB_USER              Database username
    DB_PASSWORD          Database password
    DRY_RUN              Set to 'true' to enable dry-run mode
    PROMETHEUS_URL       Prometheus endpoint URL
    AWS_PROFILE          AWS profile to use (AWS RDS mode)

EOF
    exit 1
}

# Parse arguments
SKIP_CONFIRMATION=false
PRIMARY_HOST=""
SECONDARY_HOST=""
DB_CLUSTER=""
REGION=""
FAILURE_METHOD="stop-service"

while [[ $# -gt 0 ]]; do
    case $1 in
        --mode)
            MODE="$2"
            shift 2
            ;;
        --primary-host)
            PRIMARY_HOST="$2"
            shift 2
            ;;
        --secondary-host)
            SECONDARY_HOST="$2"
            shift 2
            ;;
        --db-cluster)
            DB_CLUSTER="$2"
            shift 2
            ;;
        --region)
            REGION="$2"
            shift 2
            ;;
        --db-user)
            DB_USER="$2"
            shift 2
            ;;
        --db-password)
            DB_PASSWORD="$2"
            shift 2
            ;;
        --db-name)
            DB_NAME="$2"
            shift 2
            ;;
        --db-port)
            DB_PORT="$2"
            shift 2
            ;;
        --timeout)
            TIMEOUT="$2"
            shift 2
            ;;
        --failure-method)
            FAILURE_METHOD="$2"
            shift 2
            ;;
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --prometheus-url)
            PROMETHEUS_URL="$2"
            shift 2
            ;;
        --skip-confirmation)
            SKIP_CONFIRMATION=true
            shift
            ;;
        --help)
            usage
            ;;
        *)
            log_error "Unknown option: $1"
            usage
            ;;
    esac
done

# Set MySQL default port if needed
if [[ "$MODE" == "mysql" && "$DB_PORT" == "5432" ]]; then
    DB_PORT=3306
fi

# Validate required parameters
validate_params() {
    if [[ "$MODE" == "aws-rds" ]]; then
        if [[ -z "$DB_CLUSTER" || -z "$REGION" ]]; then
            log_error "AWS RDS mode requires --db-cluster and --region"
            exit 1
        fi
        if ! command -v aws &> /dev/null; then
            log_error "AWS CLI is required for AWS RDS mode"
            exit 1
        fi
    else
        if [[ -z "$PRIMARY_HOST" || -z "$SECONDARY_HOST" ]]; then
            log_error "Direct mode requires --primary-host and --secondary-host"
            exit 1
        fi

        if [[ "$MODE" == "postgres" ]]; then
            if ! command -v psql &> /dev/null; then
                log_error "psql is required for PostgreSQL mode"
                exit 1
            fi
        elif [[ "$MODE" == "mysql" ]]; then
            if ! command -v mysql &> /dev/null; then
                log_error "mysql is required for MySQL mode"
                exit 1
            fi
        else
            log_error "Invalid mode: $MODE. Must be 'postgres', 'mysql', or 'aws-rds'"
            exit 1
        fi
    fi

    if ! command -v jq &> /dev/null; then
        log_error "jq is required but not installed"
        exit 1
    fi
}

# Safety confirmation
confirm_execution() {
    if [[ "$SKIP_CONFIRMATION" == "true" ]]; then
        return 0
    fi

    echo ""
    log_warning "═══════════════════════════════════════════════════════════"
    log_warning "  CHAOS ENGINEERING TEST - DATABASE FAILOVER SIMULATION"
    log_warning "═══════════════════════════════════════════════════════════"
    echo ""
    log_warning "This test will:"
    log_warning "  1. Simulate primary database failure"
    log_warning "  2. Force secondary replica promotion"
    log_warning "  3. Measure failover time and data consistency"
    log_warning "  4. Restore primary database"
    echo ""
    log_warning "Configuration:"
    log_warning "  Mode: $MODE"
    if [[ "$MODE" == "aws-rds" ]]; then
        log_warning "  RDS Cluster: $DB_CLUSTER"
        log_warning "  Region: $REGION"
    else
        log_warning "  Primary Host: $PRIMARY_HOST"
        log_warning "  Secondary Host: $SECONDARY_HOST"
        log_warning "  Failure Method: $FAILURE_METHOD"
    fi
    log_warning "  Timeout: ${TIMEOUT}s"
    log_warning "  Dry Run: $DRY_RUN"
    echo ""
    log_warning "═══════════════════════════════════════════════════════════"
    echo ""

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "Running in DRY-RUN mode - no changes will be made"
        return 0
    fi

    read -p "Do you want to proceed? (yes/no): " -r
    echo
    if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
        log_info "Test cancelled by user"
        exit 0
    fi
}

# Collect Prometheus metrics
collect_metric() {
    local metric_name="$1"
    local query="$2"

    if [[ -z "$PROMETHEUS_URL" ]]; then
        return 0
    fi

    local result=$(curl -s "${PROMETHEUS_URL}/api/v1/query?query=${query}" | jq -r '.data.result[0].value[1]' 2>/dev/null || echo "0")
    echo "$result"
}

# PostgreSQL: Check if database is in recovery mode
postgres_check_recovery() {
    local host="$1"
    local pgpassword_export=""

    if [[ -n "$DB_PASSWORD" ]]; then
        pgpassword_export="PGPASSWORD='$DB_PASSWORD'"
    fi

    local recovery_status=$(eval "$pgpassword_export psql -h $host -U $DB_USER -d $DB_NAME -p $DB_PORT -t -c 'SELECT pg_is_in_recovery();'" 2>/dev/null | tr -d ' ')

    echo "$recovery_status"
}

# PostgreSQL: Get replication lag
postgres_get_lag() {
    local host="$1"
    local pgpassword_export=""

    if [[ -n "$DB_PASSWORD" ]]; then
        pgpassword_export="PGPASSWORD='$DB_PASSWORD'"
    fi

    local lag=$(eval "$pgpassword_export psql -h $host -U $DB_USER -d $DB_NAME -p $DB_PORT -t -c \"SELECT EXTRACT(EPOCH FROM (NOW() - pg_last_xact_replay_timestamp()));\"" 2>/dev/null | tr -d ' ')

    echo "${lag:-0}"
}

# MySQL: Check replication status
mysql_check_replication() {
    local host="$1"
    local mysql_cmd="mysql -h $host -u $DB_USER -p$DB_PASSWORD -P $DB_PORT -e"

    if [[ -z "$DB_PASSWORD" ]]; then
        mysql_cmd="mysql -h $host -u $DB_USER -P $DB_PORT -e"
    fi

    local slave_status=$($mysql_cmd "SHOW SLAVE STATUS\G" 2>/dev/null | grep "Slave_IO_Running" | awk '{print $2}')

    echo "$slave_status"
}

# MySQL: Get replication lag
mysql_get_lag() {
    local host="$1"
    local mysql_cmd="mysql -h $host -u $DB_USER -p$DB_PASSWORD -P $DB_PORT -e"

    if [[ -z "$DB_PASSWORD" ]]; then
        mysql_cmd="mysql -h $host -u $DB_USER -P $DB_PORT -e"
    fi

    local lag=$($mysql_cmd "SHOW SLAVE STATUS\G" 2>/dev/null | grep "Seconds_Behind_Master" | awk '{print $2}')

    echo "${lag:-0}"
}

# Get baseline metrics
get_baseline_metrics() {
    log_info "Collecting baseline metrics..."

    if [[ "$MODE" == "postgres" ]]; then
        BASELINE_RECOVERY_STATUS=$(postgres_check_recovery "$PRIMARY_HOST")
        BASELINE_LAG=$(postgres_get_lag "$SECONDARY_HOST")
        log_info "Primary recovery status: $BASELINE_RECOVERY_STATUS (should be 'f' for primary)"
        log_info "Secondary replication lag: ${BASELINE_LAG}s"
    elif [[ "$MODE" == "mysql" ]]; then
        BASELINE_REPLICATION=$(mysql_check_replication "$SECONDARY_HOST")
        BASELINE_LAG=$(mysql_get_lag "$SECONDARY_HOST")
        log_info "Replication status: $BASELINE_REPLICATION (should be 'Yes')"
        log_info "Replication lag: ${BASELINE_LAG}s"
    fi

    BASELINE_DB_CONNECTIONS=$(collect_metric "db_connections" 'pg_stat_database_numbackends{datname="'$DB_NAME'"}')
    BASELINE_ERROR_RATE=$(collect_metric "error_rate" 'rate(db_errors_total[1m])')

    log_info "Baseline DB Connections: $BASELINE_DB_CONNECTIONS"
    log_info "Baseline Error Rate: $BASELINE_ERROR_RATE"
}

# Fail primary database
fail_primary() {
    local host="$PRIMARY_HOST"

    log_info "Simulating primary database failure using method: $FAILURE_METHOD"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would fail primary database: $host"
        return 0
    fi

    case $FAILURE_METHOD in
        stop-service)
            if [[ "$MODE" == "postgres" ]]; then
                ssh "$host" "sudo systemctl stop postgresql" 2>/dev/null || log_warning "Failed to stop PostgreSQL service"
            elif [[ "$MODE" == "mysql" ]]; then
                ssh "$host" "sudo systemctl stop mysql" 2>/dev/null || ssh "$host" "sudo systemctl stop mysqld" 2>/dev/null || log_warning "Failed to stop MySQL service"
            fi
            ;;
        kill-process)
            if [[ "$MODE" == "postgres" ]]; then
                ssh "$host" "sudo pkill -9 postgres" 2>/dev/null || log_warning "Failed to kill PostgreSQL process"
            elif [[ "$MODE" == "mysql" ]]; then
                ssh "$host" "sudo pkill -9 mysqld" 2>/dev/null || log_warning "Failed to kill MySQL process"
            fi
            ;;
        network)
            ssh "$host" "sudo iptables -A INPUT -p tcp --dport $DB_PORT -j DROP" 2>/dev/null || log_warning "Failed to add iptables rule"
            ssh "$host" "sudo iptables -A OUTPUT -p tcp --sport $DB_PORT -j DROP" 2>/dev/null || log_warning "Failed to add iptables rule"
            ;;
        *)
            log_error "Unknown failure method: $FAILURE_METHOD"
            return 1
            ;;
    esac

    log_success "Primary database failed"
}

# AWS RDS: Trigger failover
aws_rds_failover() {
    log_info "Triggering AWS RDS failover for cluster: $DB_CLUSTER"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would trigger RDS failover"
        return 0
    fi

    aws rds failover-db-cluster \
        --db-cluster-identifier "$DB_CLUSTER" \
        --region "$REGION"

    log_success "Failover initiated"
}

# Measure failover time
measure_failover() {
    local secondary_host="$SECONDARY_HOST"
    local start_time=$(date +%s)
    local failover_time=0
    local max_wait="$TIMEOUT"

    log_info "Measuring failover time (timeout: ${max_wait}s)..."
    log_info "Monitoring secondary: $secondary_host"

    while true; do
        local elapsed=$(($(date +%s) - start_time))

        if [[ $elapsed -ge $max_wait ]]; then
            log_error "Failover timeout reached (${max_wait}s)"
            return 1
        fi

        local is_primary=false

        if [[ "$MODE" == "postgres" ]]; then
            local recovery_status=$(postgres_check_recovery "$secondary_host")
            if [[ "$recovery_status" == "f" ]]; then
                is_primary=true
            fi
        elif [[ "$MODE" == "mysql" ]]; then
            # For MySQL, check if secondary is accepting writes
            local mysql_cmd="mysql -h $secondary_host -u $DB_USER -p$DB_PASSWORD -P $DB_PORT -e"
            if [[ -z "$DB_PASSWORD" ]]; then
                mysql_cmd="mysql -h $secondary_host -u $DB_USER -P $DB_PORT -e"
            fi

            if $mysql_cmd "SELECT 1" &>/dev/null; then
                local read_only=$($mysql_cmd "SHOW VARIABLES LIKE 'read_only'\G" 2>/dev/null | grep "Value" | awk '{print $2}')
                if [[ "$read_only" == "OFF" ]]; then
                    is_primary=true
                fi
            fi
        fi

        if [[ "$is_primary" == "true" ]]; then
            failover_time=$elapsed
            log_success "Secondary promoted to primary after ${failover_time}s"
            break
        fi

        log_info "Waiting for promotion... (${elapsed}s elapsed)"
        sleep 2
    done

    echo "$failover_time"
}

# Verify failover success
verify_failover() {
    local secondary_host="$SECONDARY_HOST"

    log_info "Verifying failover success..."

    if [[ "$MODE" == "postgres" ]]; then
        local recovery_status=$(postgres_check_recovery "$secondary_host")
        if [[ "$recovery_status" != "f" ]]; then
            log_error "Secondary is still in recovery mode"
            return 1
        fi
        log_success "Secondary is now primary (recovery mode: false)"

        # Test write operation
        local pgpassword_export=""
        if [[ -n "$DB_PASSWORD" ]]; then
            pgpassword_export="PGPASSWORD='$DB_PASSWORD'"
        fi

        if eval "$pgpassword_export psql -h $secondary_host -U $DB_USER -d $DB_NAME -p $DB_PORT -c 'CREATE TABLE IF NOT EXISTS chaos_test (id INT);'" &>/dev/null; then
            eval "$pgpassword_export psql -h $secondary_host -U $DB_USER -d $DB_NAME -p $DB_PORT -c 'DROP TABLE chaos_test;'" &>/dev/null
            log_success "Write test successful"
        else
            log_warning "Write test failed"
            return 1
        fi

    elif [[ "$MODE" == "mysql" ]]; then
        local mysql_cmd="mysql -h $secondary_host -u $DB_USER -p$DB_PASSWORD -P $DB_PORT -e"
        if [[ -z "$DB_PASSWORD" ]]; then
            mysql_cmd="mysql -h $secondary_host -u $DB_USER -P $DB_PORT -e"
        fi

        local read_only=$($mysql_cmd "SHOW VARIABLES LIKE 'read_only'\G" 2>/dev/null | grep "Value" | awk '{print $2}')
        if [[ "$read_only" != "OFF" ]]; then
            log_error "Secondary is still in read-only mode"
            return 1
        fi
        log_success "Secondary is now writable (read_only: OFF)"

        # Test write operation
        if $mysql_cmd "CREATE TABLE IF NOT EXISTS chaos_test (id INT);" &>/dev/null; then
            $mysql_cmd "DROP TABLE chaos_test;" &>/dev/null
            log_success "Write test successful"
        else
            log_warning "Write test failed"
            return 1
        fi
    fi

    log_success "Failover verification complete"
    return 0
}

# Restore primary database
restore_primary() {
    local host="$PRIMARY_HOST"

    log_info "Restoring primary database: $host"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would restore primary database"
        return 0
    fi

    case $FAILURE_METHOD in
        stop-service)
            if [[ "$MODE" == "postgres" ]]; then
                ssh "$host" "sudo systemctl start postgresql" 2>/dev/null || log_warning "Failed to start PostgreSQL service"
            elif [[ "$MODE" == "mysql" ]]; then
                ssh "$host" "sudo systemctl start mysql" 2>/dev/null || ssh "$host" "sudo systemctl start mysqld" 2>/dev/null || log_warning "Failed to start MySQL service"
            fi
            ;;
        kill-process)
            if [[ "$MODE" == "postgres" ]]; then
                ssh "$host" "sudo systemctl start postgresql" 2>/dev/null || log_warning "Failed to start PostgreSQL service"
            elif [[ "$MODE" == "mysql" ]]; then
                ssh "$host" "sudo systemctl start mysql" 2>/dev/null || ssh "$host" "sudo systemctl start mysqld" 2>/dev/null || log_warning "Failed to start MySQL service"
            fi
            ;;
        network)
            ssh "$host" "sudo iptables -D INPUT -p tcp --dport $DB_PORT -j DROP" 2>/dev/null || log_warning "Failed to remove iptables rule"
            ssh "$host" "sudo iptables -D OUTPUT -p tcp --sport $DB_PORT -j DROP" 2>/dev/null || log_warning "Failed to remove iptables rule"
            ;;
    esac

    log_success "Primary database restored (will resync as replica)"
}

# Generate test report
generate_report() {
    local failover_time="$1"
    local test_result="$2"

    local report_file="/tmp/db-failover-report-$(date +%Y%m%d-%H%M%S).json"

    cat > "$report_file" <<EOF
{
  "test": "database-failover",
  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "mode": "$MODE",
  "configuration": {
    "primary_host": "$PRIMARY_HOST",
    "secondary_host": "$SECONDARY_HOST",
    "db_cluster": "$DB_CLUSTER",
    "failure_method": "$FAILURE_METHOD",
    "timeout": $TIMEOUT
  },
  "baseline_metrics": {
    "replication_lag": "${BASELINE_LAG:-0}",
    "db_connections": "${BASELINE_DB_CONNECTIONS:-0}",
    "error_rate": "${BASELINE_ERROR_RATE:-0}"
  },
  "results": {
    "failover_time_seconds": $failover_time,
    "test_result": "$test_result",
    "rto_met": $(if [[ $failover_time -le 60 ]]; then echo "true"; else echo "false"; fi)
  }
}
EOF

    log_info "Report saved to: $report_file"
    cat "$report_file" | jq '.'
}

# Cleanup function
cleanup() {
    local exit_code=$?

    log_info "Running cleanup..."

    if [[ "$MODE" != "aws-rds" ]]; then
        restore_primary
    fi

    log_info "Cleanup complete"
    log_info "Full log: $LOG_FILE"

    exit $exit_code
}

# Register cleanup trap
trap cleanup EXIT INT TERM

# Main execution
main() {
    log_info "Starting database failover chaos test"
    log_info "Log file: $LOG_FILE"

    # Validate parameters
    validate_params

    # Safety confirmation
    confirm_execution

    # Get baseline metrics
    get_baseline_metrics

    # Fail primary database
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 1: Simulating primary database failure"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if [[ "$MODE" == "aws-rds" ]]; then
        aws_rds_failover
    else
        fail_primary
    fi

    # Measure failover time
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 2: Measuring failover time"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if [[ "$MODE" == "aws-rds" ]]; then
        # For AWS RDS, we need to wait and check cluster status
        log_info "Waiting for AWS RDS failover to complete..."
        sleep 5
        failover_time=30  # Approximate, as AWS manages this
        log_info "AWS RDS failover initiated (approximate time: ${failover_time}s)"
    else
        failover_time=$(measure_failover)
        failover_status=$?
    fi

    # Verify failover
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 3: Verifying failover"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if [[ "$MODE" != "aws-rds" ]] && verify_failover; then
        test_result="PASS"
    elif [[ "$MODE" == "aws-rds" ]]; then
        test_result="PASS"
        log_info "AWS RDS failover completed"
    else
        test_result="FAIL"
    fi

    # Generate report
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 4: Test results"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    generate_report "$failover_time" "$test_result"

    # Final status
    echo ""
    if [[ "$test_result" == "PASS" ]]; then
        log_success "═══════════════════════════════════════════════════════════"
        log_success "  TEST PASSED"
        log_success "  Failover Time: ${failover_time}s"
        if [[ $failover_time -le 60 ]]; then
            log_success "  RTO Target Met: < 60s"
        else
            log_warning "  RTO Target Missed: ${failover_time}s > 60s"
        fi
        log_success "═══════════════════════════════════════════════════════════"
        return 0
    else
        log_error "═══════════════════════════════════════════════════════════"
        log_error "  TEST FAILED"
        log_error "═══════════════════════════════════════════════════════════"
        return 1
    fi
}

# Run main function
main "$@"

#!/bin/bash
#
# Region Failure Chaos Test
#
# Purpose: Simulate a complete region failure to validate cross-region failover
# mechanisms, DNS updates, and application resilience.
#
# Requirements:
# - AWS CLI (for AWS mode) or generic HTTP monitoring
# - jq for JSON parsing
# - curl for health checks
# - Prometheus endpoint (optional, for metrics)
#
# Usage:
#   ./region-failure-test.sh --mode aws --primary-region us-east-1 --secondary-region us-west-2
#   ./region-failure-test.sh --mode generic --primary-url https://api.example.com --secondary-url https://api-backup.example.com
#

set -euo pipefail

# Default configuration
MODE="${MODE:-aws}"
DRY_RUN="${DRY_RUN:-false}"
TIMEOUT="${TIMEOUT:-300}"  # 5 minutes
PROMETHEUS_URL="${PROMETHEUS_URL:-}"
LOG_FILE="/tmp/region-failure-test-$(date +%Y%m%d-%H%M%S).log"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging function
log() {
    local level="$1"
    shift
    local message="$*"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    echo -e "${timestamp} [${level}] ${message}" | tee -a "$LOG_FILE"
}

log_info() { log "${BLUE}INFO${NC}" "$@"; }
log_success() { log "${GREEN}SUCCESS${NC}" "$@"; }
log_warning() { log "${YELLOW}WARNING${NC}" "$@"; }
log_error() { log "${RED}ERROR${NC}" "$@"; }

# Usage information
usage() {
    cat <<EOF
Usage: $0 [OPTIONS]

Chaos engineering test to simulate region failure and validate failover.

OPTIONS:
    --mode MODE              Test mode: 'aws' or 'generic' (default: aws)
    --primary-region REGION  Primary AWS region (AWS mode)
    --secondary-region REGION Secondary AWS region (AWS mode)
    --primary-url URL        Primary endpoint URL (generic mode)
    --secondary-url URL      Secondary endpoint URL (generic mode)
    --vpc-id VPC_ID          VPC ID to isolate (AWS mode)
    --timeout SECONDS        Maximum test duration (default: 300)
    --dry-run                Show what would be done without executing
    --prometheus-url URL     Prometheus endpoint for metrics collection
    --skip-confirmation      Skip safety confirmation prompt
    --help                   Show this help message

EXAMPLES:
    # AWS mode - simulate region failure via NACL
    $0 --mode aws --primary-region us-east-1 --secondary-region us-west-2 --vpc-id vpc-12345

    # Generic mode - block traffic via firewall/routing
    $0 --mode generic --primary-url https://api.example.com --secondary-url https://backup-api.example.com

ENVIRONMENT VARIABLES:
    DRY_RUN              Set to 'true' to enable dry-run mode
    PROMETHEUS_URL       Prometheus endpoint URL
    AWS_PROFILE          AWS profile to use (AWS mode)

EOF
    exit 1
}

# Parse arguments
SKIP_CONFIRMATION=false
PRIMARY_REGION=""
SECONDARY_REGION=""
PRIMARY_URL=""
SECONDARY_URL=""
VPC_ID=""

while [[ $# -gt 0 ]]; do
    case $1 in
        --mode)
            MODE="$2"
            shift 2
            ;;
        --primary-region)
            PRIMARY_REGION="$2"
            shift 2
            ;;
        --secondary-region)
            SECONDARY_REGION="$2"
            shift 2
            ;;
        --primary-url)
            PRIMARY_URL="$2"
            shift 2
            ;;
        --secondary-url)
            SECONDARY_URL="$2"
            shift 2
            ;;
        --vpc-id)
            VPC_ID="$2"
            shift 2
            ;;
        --timeout)
            TIMEOUT="$2"
            shift 2
            ;;
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --prometheus-url)
            PROMETHEUS_URL="$2"
            shift 2
            ;;
        --skip-confirmation)
            SKIP_CONFIRMATION=true
            shift
            ;;
        --help)
            usage
            ;;
        *)
            log_error "Unknown option: $1"
            usage
            ;;
    esac
done

# Validate required parameters
validate_params() {
    if [[ "$MODE" == "aws" ]]; then
        if [[ -z "$PRIMARY_REGION" || -z "$SECONDARY_REGION" ]]; then
            log_error "AWS mode requires --primary-region and --secondary-region"
            exit 1
        fi
        if ! command -v aws &> /dev/null; then
            log_error "AWS CLI is required for AWS mode"
            exit 1
        fi
    elif [[ "$MODE" == "generic" ]]; then
        if [[ -z "$PRIMARY_URL" || -z "$SECONDARY_URL" ]]; then
            log_error "Generic mode requires --primary-url and --secondary-url"
            exit 1
        fi
    else
        log_error "Invalid mode: $MODE. Must be 'aws' or 'generic'"
        exit 1
    fi

    if ! command -v jq &> /dev/null; then
        log_error "jq is required but not installed"
        exit 1
    fi
}

# Safety confirmation
confirm_execution() {
    if [[ "$SKIP_CONFIRMATION" == "true" ]]; then
        return 0
    fi

    echo ""
    log_warning "═══════════════════════════════════════════════════════════"
    log_warning "  CHAOS ENGINEERING TEST - REGION FAILURE SIMULATION"
    log_warning "═══════════════════════════════════════════════════════════"
    echo ""
    log_warning "This test will:"
    log_warning "  1. Simulate a complete region failure"
    log_warning "  2. Block network traffic to primary region"
    log_warning "  3. Trigger failover to secondary region"
    log_warning "  4. Measure recovery time and application impact"
    echo ""
    log_warning "Configuration:"
    log_warning "  Mode: $MODE"
    if [[ "$MODE" == "aws" ]]; then
        log_warning "  Primary Region: $PRIMARY_REGION"
        log_warning "  Secondary Region: $SECONDARY_REGION"
        [[ -n "$VPC_ID" ]] && log_warning "  VPC ID: $VPC_ID"
    else
        log_warning "  Primary URL: $PRIMARY_URL"
        log_warning "  Secondary URL: $SECONDARY_URL"
    fi
    log_warning "  Timeout: ${TIMEOUT}s"
    log_warning "  Dry Run: $DRY_RUN"
    echo ""
    log_warning "═══════════════════════════════════════════════════════════"
    echo ""

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "Running in DRY-RUN mode - no changes will be made"
        return 0
    fi

    read -p "Do you want to proceed? (yes/no): " -r
    echo
    if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
        log_info "Test cancelled by user"
        exit 0
    fi
}

# Collect Prometheus metrics
collect_metric() {
    local metric_name="$1"
    local query="$2"

    if [[ -z "$PROMETHEUS_URL" ]]; then
        return 0
    fi

    local result=$(curl -s "${PROMETHEUS_URL}/api/v1/query?query=${query}" | jq -r '.data.result[0].value[1]' 2>/dev/null || echo "0")
    echo "$result"
}

# Get baseline metrics
get_baseline_metrics() {
    log_info "Collecting baseline metrics..."

    BASELINE_ERROR_RATE=$(collect_metric "error_rate" 'rate(http_requests_total{status=~"5.."}[1m])')
    BASELINE_LATENCY=$(collect_metric "latency" 'histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[1m]))')
    BASELINE_THROUGHPUT=$(collect_metric "throughput" 'rate(http_requests_total[1m])')

    log_info "Baseline Error Rate: $BASELINE_ERROR_RATE"
    log_info "Baseline P99 Latency: ${BASELINE_LATENCY}s"
    log_info "Baseline Throughput: ${BASELINE_THROUGHPUT} req/s"
}

# Check endpoint health
check_endpoint_health() {
    local url="$1"
    local response_code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "$url" 2>/dev/null || echo "000")

    if [[ "$response_code" =~ ^2[0-9]{2}$ ]]; then
        return 0
    else
        return 1
    fi
}

# AWS: Block region traffic via NACL
aws_block_region() {
    log_info "Blocking traffic to primary region: $PRIMARY_REGION"

    if [[ -z "$VPC_ID" ]]; then
        log_warning "No VPC ID specified, discovering default VPC..."
        VPC_ID=$(aws ec2 describe-vpcs \
            --region "$PRIMARY_REGION" \
            --filters "Name=isDefault,Values=true" \
            --query 'Vpcs[0].VpcId' \
            --output text)

        if [[ "$VPC_ID" == "None" || -z "$VPC_ID" ]]; then
            log_error "Could not find VPC in region $PRIMARY_REGION"
            return 1
        fi
        log_info "Using VPC: $VPC_ID"
    fi

    # Get Network ACL ID
    NACL_ID=$(aws ec2 describe-network-acls \
        --region "$PRIMARY_REGION" \
        --filters "Name=vpc-id,Values=$VPC_ID" "Name=default,Values=true" \
        --query 'NetworkAcls[0].NetworkAclId' \
        --output text)

    if [[ "$NACL_ID" == "None" || -z "$NACL_ID" ]]; then
        log_error "Could not find Network ACL for VPC $VPC_ID"
        return 1
    fi

    log_info "Network ACL ID: $NACL_ID"

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would create DENY rule in NACL $NACL_ID"
        return 0
    fi

    # Create deny rule (rule number 1 to take precedence)
    aws ec2 create-network-acl-entry \
        --region "$PRIMARY_REGION" \
        --network-acl-id "$NACL_ID" \
        --rule-number 1 \
        --protocol -1 \
        --rule-action deny \
        --egress \
        --cidr-block 0.0.0.0/0

    aws ec2 create-network-acl-entry \
        --region "$PRIMARY_REGION" \
        --network-acl-id "$NACL_ID" \
        --rule-number 1 \
        --protocol -1 \
        --rule-action deny \
        --ingress \
        --cidr-block 0.0.0.0/0

    log_success "Successfully blocked traffic to $PRIMARY_REGION"

    # Store NACL ID for cleanup
    echo "$NACL_ID" > /tmp/chaos-nacl-id.txt
}

# AWS: Restore region traffic
aws_restore_region() {
    log_info "Restoring traffic to primary region: $PRIMARY_REGION"

    if [[ ! -f /tmp/chaos-nacl-id.txt ]]; then
        log_warning "No NACL ID found, skipping restore"
        return 0
    fi

    NACL_ID=$(cat /tmp/chaos-nacl-id.txt)

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would remove DENY rules from NACL $NACL_ID"
        return 0
    fi

    # Remove deny rules
    aws ec2 delete-network-acl-entry \
        --region "$PRIMARY_REGION" \
        --network-acl-id "$NACL_ID" \
        --rule-number 1 \
        --egress \
        2>/dev/null || log_warning "Failed to delete egress rule"

    aws ec2 delete-network-acl-entry \
        --region "$PRIMARY_REGION" \
        --network-acl-id "$NACL_ID" \
        --rule-number 1 \
        --ingress \
        2>/dev/null || log_warning "Failed to delete ingress rule"

    rm -f /tmp/chaos-nacl-id.txt
    log_success "Traffic restored to $PRIMARY_REGION"
}

# Generic: Simulate region failure (requires manual intervention)
generic_block_region() {
    log_warning "Generic mode: Manual intervention required"
    log_info "Please block traffic to: $PRIMARY_URL"
    log_info "Suggested methods:"
    log_info "  - Update firewall rules to drop traffic"
    log_info "  - Update routing tables to blackhole traffic"
    log_info "  - Disable load balancer"
    log_info "  - Stop application servers"
    echo ""

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would wait for manual failover"
        return 0
    fi

    read -p "Press ENTER when primary region is blocked..." -r
    log_info "Proceeding with test..."
}

# Generic: Restore region
generic_restore_region() {
    log_warning "Generic mode: Manual restoration required"
    log_info "Please restore traffic to: $PRIMARY_URL"
    log_info "Suggested methods:"
    log_info "  - Remove firewall rules"
    log_info "  - Restore routing tables"
    log_info "  - Enable load balancer"
    log_info "  - Start application servers"
    echo ""

    if [[ "$DRY_RUN" == "true" ]]; then
        log_info "[DRY-RUN] Would wait for manual restoration"
        return 0
    fi

    read -p "Press ENTER when primary region is restored..." -r
    log_info "Primary region restored"
}

# Measure recovery time
measure_recovery() {
    local secondary_endpoint="$1"
    local start_time=$(date +%s)
    local recovery_time=0
    local max_wait="$TIMEOUT"

    log_info "Measuring recovery time (timeout: ${max_wait}s)..."
    log_info "Monitoring secondary endpoint: $secondary_endpoint"

    while true; do
        local elapsed=$(($(date +%s) - start_time))

        if [[ $elapsed -ge $max_wait ]]; then
            log_error "Recovery timeout reached (${max_wait}s)"
            return 1
        fi

        if check_endpoint_health "$secondary_endpoint"; then
            recovery_time=$elapsed
            log_success "Secondary endpoint healthy after ${recovery_time}s"
            break
        fi

        log_info "Waiting for recovery... (${elapsed}s elapsed)"
        sleep 2
    done

    echo "$recovery_time"
}

# Verify failover success
verify_failover() {
    local secondary_endpoint="$1"

    log_info "Verifying failover to secondary region..."

    # Check health
    if ! check_endpoint_health "$secondary_endpoint"; then
        log_error "Secondary endpoint is not healthy"
        return 1
    fi

    # Check error rate if Prometheus available
    if [[ -n "$PROMETHEUS_URL" ]]; then
        local current_error_rate=$(collect_metric "error_rate" 'rate(http_requests_total{status=~"5.."}[1m])')
        log_info "Current error rate: $current_error_rate"

        # Allow 10% increase in error rate
        local threshold=$(echo "$BASELINE_ERROR_RATE * 1.1" | bc -l)
        if (( $(echo "$current_error_rate > $threshold" | bc -l) )); then
            log_warning "Error rate increased beyond threshold"
        else
            log_success "Error rate within acceptable range"
        fi
    fi

    log_success "Failover verification complete"
    return 0
}

# Generate test report
generate_report() {
    local recovery_time="$1"
    local test_result="$2"

    local report_file="/tmp/region-failure-report-$(date +%Y%m%d-%H%M%S).json"

    cat > "$report_file" <<EOF
{
  "test": "region-failure",
  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "mode": "$MODE",
  "configuration": {
    "primary_region": "$PRIMARY_REGION",
    "secondary_region": "$SECONDARY_REGION",
    "primary_url": "$PRIMARY_URL",
    "secondary_url": "$SECONDARY_URL",
    "timeout": $TIMEOUT
  },
  "baseline_metrics": {
    "error_rate": "$BASELINE_ERROR_RATE",
    "latency_p99": "$BASELINE_LATENCY",
    "throughput": "$BASELINE_THROUGHPUT"
  },
  "results": {
    "recovery_time_seconds": $recovery_time,
    "test_result": "$test_result"
  }
}
EOF

    log_info "Report saved to: $report_file"
    cat "$report_file" | jq '.'
}

# Cleanup function
cleanup() {
    local exit_code=$?

    log_info "Running cleanup..."

    if [[ "$MODE" == "aws" ]]; then
        aws_restore_region
    else
        generic_restore_region
    fi

    log_info "Cleanup complete"
    log_info "Full log: $LOG_FILE"

    exit $exit_code
}

# Register cleanup trap
trap cleanup EXIT INT TERM

# Main execution
main() {
    log_info "Starting region failure chaos test"
    log_info "Log file: $LOG_FILE"

    # Validate parameters
    validate_params

    # Safety confirmation
    confirm_execution

    # Get baseline metrics
    get_baseline_metrics

    # Determine secondary endpoint
    local secondary_endpoint=""
    if [[ "$MODE" == "aws" ]]; then
        # For AWS mode, you would typically have a health check endpoint
        # This is a placeholder - adjust based on your architecture
        secondary_endpoint="${SECONDARY_URL:-https://api.${SECONDARY_REGION}.example.com/health}"
        log_warning "Secondary endpoint not specified, using: $secondary_endpoint"
        log_warning "Override with --secondary-url if different"
    else
        secondary_endpoint="$SECONDARY_URL"
    fi

    # Block primary region
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 1: Simulating region failure"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if [[ "$MODE" == "aws" ]]; then
        aws_block_region
    else
        generic_block_region
    fi

    # Measure recovery time
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 2: Measuring recovery time"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    recovery_time=$(measure_recovery "$secondary_endpoint")
    recovery_status=$?

    # Verify failover
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 3: Verifying failover"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    if verify_failover "$secondary_endpoint"; then
        test_result="PASS"
    else
        test_result="FAIL"
    fi

    # Generate report
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
    log_info "PHASE 4: Test results"
    log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"

    generate_report "$recovery_time" "$test_result"

    # Final status
    echo ""
    if [[ "$test_result" == "PASS" ]]; then
        log_success "═══════════════════════════════════════════════════════════"
        log_success "  TEST PASSED"
        log_success "  Recovery Time: ${recovery_time}s"
        log_success "═══════════════════════════════════════════════════════════"
        return 0
    else
        log_error "═══════════════════════════════════════════════════════════"
        log_error "  TEST FAILED"
        log_error "═══════════════════════════════════════════════════════════"
        return 1
    fi
}

# Run main function
main "$@"

skill: "planning-disaster-recovery"
version: "1.0"
domain: "infrastructure"

# Base outputs required for all disaster recovery implementations
base_outputs:
  - path: "docs/disaster-recovery-plan.md"
    must_contain: ["RTO", "RPO", "Recovery Objectives"]
    reason: "Comprehensive DR plan documenting RTO/RPO targets and recovery procedures"

  - path: "runbooks/"
    must_contain: []
    reason: "Step-by-step operational procedures for DR scenarios"

  - path: "backups/"
    must_contain: []
    reason: "Backup configuration and retention policies"

  - path: "monitoring/backup-alerts.yaml"
    must_contain: ["alert:", "backup"]
    reason: "Alerting rules for backup failures and violations"

# Conditional outputs based on configuration
conditional_outputs:
  maturity:
    starter:
      - path: "docs/rto-rpo-requirements.md"
        must_contain: ["RTO >", "RPO >", "Tier"]
        reason: "Basic RTO/RPO definitions with criticality tiers"

      - path: "backups/backup-schedule.sh"
        must_contain: ["cron", "backup"]
        reason: "Simple cron-based backup scheduling"

      - path: "runbooks/backup-restore-basic.md"
        must_contain: ["restore", "steps"]
        reason: "Basic backup and restore procedures"

      - path: "scripts/validate-backup.sh"
        must_contain: ["#!/bin/bash", "backup"]
        reason: "Manual backup validation script"

    intermediate:
      - path: "docs/disaster-recovery-plan.md"
        must_contain: ["RTO", "RPO", "failover", "testing"]
        reason: "Full DR plan with failover procedures and testing schedule"

      - path: "runbooks/database-failover.md"
        must_contain: ["Phase", "promotion", "DNS", "validation"]
        reason: "Detailed database failover runbook with phases"

      - path: "runbooks/region-failover.md"
        must_contain: ["cross-region", "replication", "DNS"]
        reason: "Region-level failover procedures"

      - path: "backups/retention-policy.yaml"
        must_contain: ["daily:", "weekly:", "monthly:"]
        reason: "Tiered retention policy configuration"

      - path: "scripts/test-restore.sh"
        must_contain: ["restore", "verify", "staging"]
        reason: "Automated restore testing to staging environment"

      - path: "scripts/dr-drill.sh"
        must_contain: ["failover", "test", "environment"]
        reason: "Automated DR drill execution script"

      - path: "monitoring/backup-dashboard.json"
        must_contain: ["backup", "metrics", "panel"]
        reason: "Backup monitoring dashboard (Grafana/CloudWatch)"

    advanced:
      - path: "docs/disaster-recovery-plan.md"
        must_contain: ["RTO", "RPO", "Active-Active", "chaos", "compliance"]
        reason: "Enterprise DR plan with chaos testing and compliance mapping"

      - path: "runbooks/database-failover.md"
        must_contain: ["automated", "promotion", "rollback", "metrics"]
        reason: "Production-grade failover runbook with automation hooks"

      - path: "runbooks/region-failover.md"
        must_contain: ["multi-region", "traffic", "DNS", "automated"]
        reason: "Multi-region failover with traffic shifting automation"

      - path: "chaos/db-failover-test.sh"
        must_contain: ["#!/bin/bash", "fail", "primary", "measure"]
        reason: "Chaos engineering test for database failover validation"

      - path: "chaos/region-failure-test.sh"
        must_contain: ["#!/bin/bash", "region", "network", "failover"]
        reason: "Chaos test simulating full region failure"

      - path: "scripts/automated-db-failover.sh"
        must_contain: ["promote", "verify-replication", "update-dns"]
        reason: "Fully automated database failover execution"

      - path: "scripts/generate-dr-report.sh"
        must_contain: ["compliance", "report", "metrics"]
        reason: "Automated DR compliance reporting"

      - path: "scripts/check-retention.sh"
        must_contain: ["retention", "policy", "violations"]
        reason: "Automated retention policy compliance checking"

      - path: "monitoring/backup-alerts.yaml"
        must_contain: ["VeleroBackupFailed", "BackupTooOld", "RPO"]
        reason: "Comprehensive Prometheus alerting rules"

      - path: "monitoring/dr-metrics-dashboard.json"
        must_contain: ["RTO", "RPO", "failover_time", "backup_success_rate"]
        reason: "Advanced DR metrics dashboard"

      - path: "docs/chaos-engineering-plan.md"
        must_contain: ["scenarios", "monthly", "gameday"]
        reason: "Chaos engineering testing plan and schedule"

      - path: "docs/compliance-mapping.md"
        must_contain: ["GDPR", "SOC 2", "HIPAA", "retention"]
        reason: "DR compliance requirements mapping"

  cloud_provider:
    aws:
      - path: "backups/aws/rds-backup-config.tf"
        must_contain: ["aws_db_instance", "backup_retention_period", "backup_window"]
        reason: "RDS automated backup configuration"

      - path: "backups/aws/s3-replication.tf"
        must_contain: ["aws_s3_bucket_replication_configuration", "cross-region"]
        reason: "S3 Cross-Region Replication for backup storage"

      - path: "backups/aws/aurora-global-db.tf"
        must_contain: ["aws_rds_global_cluster", "source_db_cluster_identifier"]
        reason: "Aurora Global Database for active-passive replication"

      - path: "runbooks/aws-rds-failover.md"
        must_contain: ["aws rds failover", "Multi-AZ", "promotion"]
        reason: "AWS RDS-specific failover procedures"

      - path: "scripts/aws-backup-validation.sh"
        must_contain: ["aws rds describe-db-snapshots", "BackupRetentionPeriod"]
        reason: "AWS backup validation and compliance check"

    gcp:
      - path: "backups/gcp/cloud-sql-backup.tf"
        must_contain: ["google_sql_database_instance", "backup_configuration", "point_in_time_recovery"]
        reason: "Cloud SQL backup and PITR configuration"

      - path: "backups/gcp/gcs-replication.tf"
        must_contain: ["google_storage_bucket", "storage_class", "MULTI_REGIONAL"]
        reason: "GCS multi-regional bucket for backup storage"

      - path: "runbooks/gcp-cloud-sql-failover.md"
        must_contain: ["gcloud sql instances failover", "replica", "HA"]
        reason: "GCP Cloud SQL failover procedures"

      - path: "scripts/gcp-backup-validation.sh"
        must_contain: ["gcloud sql backups list", "PITR"]
        reason: "GCP backup validation script"

    azure:
      - path: "backups/azure/sql-backup.tf"
        must_contain: ["azurerm_mssql_database", "short_term_retention_policy", "long_term_retention_policy"]
        reason: "Azure SQL backup retention configuration"

      - path: "backups/azure/storage-replication.tf"
        must_contain: ["azurerm_storage_account", "account_replication_type", "GRS"]
        reason: "Geo-redundant storage for backup replication"

      - path: "backups/azure/site-recovery.tf"
        must_contain: ["azurerm_site_recovery", "recovery_vault", "replication_policy"]
        reason: "Azure Site Recovery for VM replication"

      - path: "runbooks/azure-sql-failover.md"
        must_contain: ["az sql db replica create", "failover group", "geo-replication"]
        reason: "Azure SQL Database failover procedures"

    multi-cloud:
      - path: "backups/multi-cloud/backup-strategy.md"
        must_contain: ["AWS", "GCP", "Azure", "cross-cloud"]
        reason: "Multi-cloud backup strategy and considerations"

      - path: "backups/multi-cloud/restic-config.yaml"
        must_contain: ["repository:", "aws", "gcs", "azure"]
        reason: "Restic configuration for multi-cloud backups"

      - path: "scripts/multi-cloud-dr-validation.sh"
        must_contain: ["aws", "gcloud", "az", "backup"]
        reason: "Cross-cloud DR validation script"

  infrastructure:
    kubernetes:
      - path: "backups/kubernetes/velero-config.yaml"
        must_contain: ["Velero", "BackupStorageLocation", "schedule"]
        reason: "Velero backup configuration for K8s cluster backup"

      - path: "backups/kubernetes/velero-schedules.yaml"
        must_contain: ["Schedule", "daily", "production"]
        reason: "Scheduled backup policies for namespaces and PVs"

      - path: "backups/kubernetes/etcd-backup-cronjob.yaml"
        must_contain: ["CronJob", "etcdctl", "snapshot"]
        reason: "etcd control plane backup automation"

      - path: "runbooks/kubernetes-restore.md"
        must_contain: ["velero restore", "namespace", "PersistentVolume"]
        reason: "Kubernetes cluster and namespace restore procedures"

      - path: "scripts/velero-backup-validation.sh"
        must_contain: ["velero backup describe", "Completed"]
        reason: "Velero backup validation script"

    docker:
      - path: "backups/docker/volume-backup.sh"
        must_contain: ["docker run", "volume", "tar"]
        reason: "Docker volume backup script"

      - path: "backups/docker/backup-schedule.sh"
        must_contain: ["docker-compose", "backup"]
        reason: "Docker Compose application backup automation"

  database:
    postgresql:
      - path: "backups/postgresql/pgbackrest.conf"
        must_contain: ["[global]", "repo1-path", "repo1-retention"]
        reason: "pgBackRest configuration for PostgreSQL backups"

      - path: "backups/postgresql/pgbackrest-schedule.sh"
        must_contain: ["pgbackrest", "full", "diff", "incr"]
        reason: "Scheduled full, differential, and incremental backups"

      - path: "backups/postgresql/walg-config.sh"
        must_contain: ["WAL_G", "WALG_S3_PREFIX", "WALG_DELTA_MAX_STEPS"]
        reason: "WAL-G configuration for continuous WAL archiving"

      - path: "runbooks/postgresql-pitr-restore.md"
        must_contain: ["pgbackrest restore", "recovery_target_time", "PITR"]
        reason: "PostgreSQL point-in-time recovery procedures"

    mysql:
      - path: "backups/mysql/xtrabackup-config.cnf"
        must_contain: ["[xtrabackup]", "parallel", "compress"]
        reason: "Percona XtraBackup configuration"

      - path: "backups/mysql/xtrabackup-schedule.sh"
        must_contain: ["xtrabackup --backup", "full", "incremental"]
        reason: "Scheduled MySQL hot backups"

      - path: "runbooks/mysql-restore.md"
        must_contain: ["xtrabackup --prepare", "xtrabackup --copy-back", "binlog"]
        reason: "MySQL backup restore and PITR procedures"

    mongodb:
      - path: "backups/mongodb/mongodump-schedule.sh"
        must_contain: ["mongodump", "gzip", "oplog"]
        reason: "MongoDB logical backup with oplog for PITR"

      - path: "backups/mongodb/atlas-backup-config.json"
        must_contain: ["clusterName", "retentionInDays", "continuousBackupEnabled"]
        reason: "MongoDB Atlas continuous backup configuration"

# Scaffolding files that should be created as starting points
scaffolding:
  - path: "docs/"
    reason: "Documentation directory for DR plans and procedures"

  - path: "runbooks/"
    reason: "Operational runbooks for various DR scenarios"

  - path: "backups/"
    reason: "Backup configuration and policies"

  - path: "scripts/"
    reason: "Automation scripts for backup, restore, and DR testing"

  - path: "monitoring/"
    reason: "Monitoring configurations, alerts, and dashboards"

  - path: "chaos/"
    reason: "Chaos engineering tests for DR validation"

  - path: "docs/disaster-recovery-plan.md"
    reason: "Main DR plan document template"

  - path: "docs/rto-rpo-requirements.md"
    reason: "Document defining recovery objectives by system tier"

  - path: "backups/README.md"
    reason: "Document backup strategy and configuration"

  - path: "runbooks/README.md"
    reason: "Index of available DR runbooks"

  - path: "scripts/README.md"
    reason: "Document automation scripts and usage"

  - path: ".gitignore"
    reason: "Exclude backup files, temporary restore data, and logs"

# Metadata
metadata:
  primary_blueprints: ["cloud", "k8s", "data-pipeline"]
  contributes_to:
    - "Disaster recovery plan"
    - "Backup and restore procedures"
    - "Business continuity planning"
    - "Compliance requirements (GDPR, SOC 2, HIPAA)"
    - "High availability architecture"
    - "Chaos engineering validation"

  common_patterns:
    - "3-2-1 backup rule (3 copies, 2 media types, 1 offsite)"
    - "RTO/RPO tiering by criticality (Tier 0: <1hr, Tier 1: 1-4hr, Tier 2: 4-24hr)"
    - "Continuous WAL/binlog archiving for database PITR"
    - "Cross-region replication (Active-Active, Active-Passive, Warm Standby, Pilot Light)"
    - "Automated failover with DNS updates"
    - "Monthly DR drills and chaos testing"
    - "Immutable backups for ransomware protection"
    - "Compliance-driven retention policies"

  integration_points:
    infrastructure: "Provisions backup infrastructure (S3, GCS, storage accounts)"
    databases: "Configures database-specific backup tools (pgBackRest, XtraBackup)"
    kubernetes: "Velero for cluster/PV backups, etcd snapshots"
    observability: "Monitors backup jobs, tracks RTO/RPO metrics, alerts on failures"
    security: "Encrypts backups, manages backup credentials, immutable storage"
    compliance: "Meets regulatory retention requirements (GDPR, SOC 2, HIPAA)"

  typical_directory_structure: |
    project/
    ├── docs/
    │   ├── disaster-recovery-plan.md        # Main DR plan
    │   ├── rto-rpo-requirements.md          # Recovery objectives
    │   ├── chaos-engineering-plan.md        # Testing strategy
    │   └── compliance-mapping.md            # Regulatory requirements
    ├── runbooks/
    │   ├── database-failover.md             # DB failover procedures
    │   ├── region-failover.md               # Multi-region failover
    │   ├── kubernetes-restore.md            # K8s restore procedures
    │   └── README.md                        # Runbook index
    ├── backups/
    │   ├── postgresql/
    │   │   ├── pgbackrest.conf              # pgBackRest config
    │   │   └── pgbackrest-schedule.sh       # Backup schedules
    │   ├── kubernetes/
    │   │   ├── velero-config.yaml           # Velero setup
    │   │   └── velero-schedules.yaml        # Backup schedules
    │   ├── aws/                             # AWS-specific backups
    │   ├── gcp/                             # GCP-specific backups
    │   ├── azure/                           # Azure-specific backups
    │   └── retention-policy.yaml            # Retention rules
    ├── scripts/
    │   ├── validate-backup.sh               # Backup integrity check
    │   ├── test-restore.sh                  # Automated restore test
    │   ├── dr-drill.sh                      # Full DR drill automation
    │   ├── check-retention.sh               # Retention compliance
    │   ├── generate-dr-report.sh            # Compliance reporting
    │   └── automated-db-failover.sh         # Automated failover
    ├── chaos/
    │   ├── db-failover-test.sh              # Database failover chaos test
    │   └── region-failure-test.sh           # Region failure simulation
    └── monitoring/
        ├── backup-alerts.yaml               # Prometheus alerts
        └── dr-metrics-dashboard.json        # Grafana dashboard

  database_backup_tools:
    postgresql:
      primary: "pgBackRest"
      alternative: "WAL-G"
      features: "PITR, compression, multi-repo, S3/GCS/Azure"
    mysql:
      primary: "Percona XtraBackup"
      alternative: "WAL-G"
      features: "Hot backups, incremental, parallel processing"
    mongodb:
      primary: "MongoDB Atlas Backup"
      alternative: "mongodump"
      features: "Continuous backup, PITR, automated"

  kubernetes_backup_tools:
    cluster:
      primary: "Velero"
      features: "PV snapshots, namespace backup, scheduling, selective restore"
    control_plane:
      primary: "etcdctl snapshot"
      features: "etcd backup for cluster state recovery"

  cross_region_patterns:
    active_active:
      rto: "< 1 minute"
      rpo: "< 1 minute"
      cost: "High"
      use_case: "Both regions serve production traffic"
    active_passive:
      rto: "15-60 minutes"
      rpo: "5-15 minutes"
      cost: "Medium"
      use_case: "Standby region for failover"
    pilot_light:
      rto: "10-30 minutes"
      rpo: "5-15 minutes"
      cost: "Low"
      use_case: "Minimal secondary infrastructure"
    warm_standby:
      rto: "5-15 minutes"
      rpo: "5-15 minutes"
      cost: "Medium-High"
      use_case: "Scaled-down secondary ready to scale up"

  compliance_retention:
    gdpr:
      retention: "1-7 years"
      requirements: "EU data residency, right to erasure"
    soc2:
      retention: "1 year+"
      requirements: "Secure deletion, access controls, audit logs"
    hipaa:
      retention: "6 years"
      requirements: "Encryption, PHI protection, secure deletion"
    pci_dss:
      retention: "3 months - 1 year"
      requirements: "Secure deletion, quarterly reviews"

  validation_checks:
    - "RTO/RPO defined for all critical systems"
    - "Backup schedules match RPO requirements"
    - "Restore tested monthly (critical systems)"
    - "Backups encrypted at rest and in transit"
    - "3-2-1 backup rule implemented"
    - "Cross-region replication configured (production)"
    - "Retention policies meet compliance requirements"
    - "Immutable backups enabled (ransomware protection)"
    - "DR runbooks documented and accessible"
    - "Chaos tests scheduled and executed"
    - "Monitoring alerts for backup failures"
    - "Failover automation tested quarterly"