Reliability Engineering

/**
 * Circuit Breaker Template
 * Usage: Protect services from cascading failures
 */

// ===========================================
// Types
// ===========================================

export type CircuitState = 'closed' | 'open' | 'half-open';

export interface CircuitBreakerOptions {
  /** Failure threshold before opening (default: 5) */
  failureThreshold: number;
  /** Success threshold to close from half-open (default: 2) */
  successThreshold: number;
  /** Time before attempting recovery in ms (default: 30000) */
  resetTimeout: number;
  /** Request timeout in ms (default: 10000) */
  timeout: number;
  /** Monitor window in ms (default: 60000) */
  monitorWindow: number;
  /** Failure rate threshold percentage (default: 50) */
  failureRateThreshold: number;
  /** Minimum requests before evaluating failure rate */
  minimumRequests: number;
}

export interface CircuitBreakerStats {
  state: CircuitState;
  failures: number;
  successes: number;
  totalRequests: number;
  failureRate: number;
  lastFailure?: Date;
  lastSuccess?: Date;
  lastStateChange: Date;
}

// ===========================================
// Circuit Breaker Implementation
// ===========================================

export class CircuitBreaker {
  private state: CircuitState = 'closed';
  private failures: number = 0;
  private successes: number = 0;
  private lastFailureTime?: number;
  private lastStateChange: number = Date.now();
  private requestLog: Array<{ timestamp: number; success: boolean }> = [];
  private options: CircuitBreakerOptions;

  constructor(options: Partial<CircuitBreakerOptions> = {}) {
    this.options = {
      failureThreshold: 5,
      successThreshold: 2,
      resetTimeout: 30000,
      timeout: 10000,
      monitorWindow: 60000,
      failureRateThreshold: 50,
      minimumRequests: 10,
      ...options,
    };
  }

  /**
   * Execute function through circuit breaker
   */
  async execute<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === 'open') {
      if (this.shouldAttemptReset()) {
        this.transitionTo('half-open');
      } else {
        throw new CircuitOpenError('Circuit breaker is open');
      }
    }

    try {
      const result = await this.executeWithTimeout(fn);
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  /**
   * Get current stats
   */
  getStats(): CircuitBreakerStats {
    this.pruneRequestLog();
    const totalRequests = this.requestLog.length;
    const failures = this.requestLog.filter(r => !r.success).length;

    return {
      state: this.state,
      failures: this.failures,
      successes: this.successes,
      totalRequests,
      failureRate: totalRequests > 0 ? (failures / totalRequests) * 100 : 0,
      lastFailure: this.lastFailureTime ? new Date(this.lastFailureTime) : undefined,
      lastSuccess: this.successes > 0 ? new Date() : undefined,
      lastStateChange: new Date(this.lastStateChange),
    };
  }

  /**
   * Manually reset circuit
   */
  reset(): void {
    this.state = 'closed';
    this.failures = 0;
    this.successes = 0;
    this.lastFailureTime = undefined;
    this.requestLog = [];
    this.lastStateChange = Date.now();
  }

  /**
   * Get current state
   */
  getState(): CircuitState {
    return this.state;
  }

  // ===========================================
  // Private Methods
  // ===========================================

  private async executeWithTimeout<T>(fn: () => Promise<T>): Promise<T> {
    return Promise.race([
      fn(),
      new Promise<never>((_, reject) => {
        setTimeout(
          () => reject(new TimeoutError('Request timed out')),
          this.options.timeout
        );
      }),
    ]);
  }

  private onSuccess(): void {
    this.recordRequest(true);

    if (this.state === 'half-open') {
      this.successes++;
      if (this.successes >= this.options.successThreshold) {
        this.transitionTo('closed');
      }
    } else {
      this.failures = 0;
    }
  }

  private onFailure(): void {
    this.recordRequest(false);
    this.lastFailureTime = Date.now();
    this.failures++;

    if (this.state === 'half-open') {
      this.transitionTo('open');
    } else if (this.state === 'closed') {
      if (this.shouldOpen()) {
        this.transitionTo('open');
      }
    }
  }

  private shouldOpen(): boolean {
    // Check absolute failure threshold
    if (this.failures >= this.options.failureThreshold) {
      return true;
    }

    // Check failure rate
    this.pruneRequestLog();
    if (this.requestLog.length >= this.options.minimumRequests) {
      const failures = this.requestLog.filter(r => !r.success).length;
      const failureRate = (failures / this.requestLog.length) * 100;
      if (failureRate >= this.options.failureRateThreshold) {
        return true;
      }
    }

    return false;
  }

  private shouldAttemptReset(): boolean {
    return Date.now() - this.lastStateChange >= this.options.resetTimeout;
  }

  private transitionTo(newState: CircuitState): void {
    console.log(`Circuit breaker: ${this.state} -> ${newState}`);
    this.state = newState;
    this.lastStateChange = Date.now();

    if (newState === 'half-open') {
      this.successes = 0;
    } else if (newState === 'closed') {
      this.failures = 0;
      this.successes = 0;
    }
  }

  private recordRequest(success: boolean): void {
    this.requestLog.push({ timestamp: Date.now(), success });
    this.pruneRequestLog();
  }

  private pruneRequestLog(): void {
    const cutoff = Date.now() - this.options.monitorWindow;
    this.requestLog = this.requestLog.filter(r => r.timestamp > cutoff);
  }
}

// ===========================================
// Error Classes
// ===========================================

export class CircuitOpenError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'CircuitOpenError';
  }
}

export class TimeoutError extends Error {
  constructor(message: string) {
    super(message);
    this.name = 'TimeoutError';
  }
}

// ===========================================
// Circuit Breaker Registry
// ===========================================

export class CircuitBreakerRegistry {
  private breakers: Map<string, CircuitBreaker> = new Map();

  get(name: string, options?: Partial<CircuitBreakerOptions>): CircuitBreaker {
    if (!this.breakers.has(name)) {
      this.breakers.set(name, new CircuitBreaker(options));
    }
    return this.breakers.get(name)!;
  }

  getAll(): Map<string, CircuitBreaker> {
    return new Map(this.breakers);
  }

  getAllStats(): Record<string, CircuitBreakerStats> {
    const stats: Record<string, CircuitBreakerStats> = {};
    for (const [name, breaker] of this.breakers) {
      stats[name] = breaker.getStats();
    }
    return stats;
  }

  reset(name: string): void {
    this.breakers.get(name)?.reset();
  }

  resetAll(): void {
    for (const breaker of this.breakers.values()) {
      breaker.reset();
    }
  }
}

// ===========================================
// Usage Example
// ===========================================

/*
import { CircuitBreaker, CircuitBreakerRegistry, CircuitOpenError } from './circuit-breaker';

// Single circuit breaker
const circuitBreaker = new CircuitBreaker({
  failureThreshold: 5,
  resetTimeout: 30000,
  timeout: 5000,
});

async function callExternalService() {
  try {
    const result = await circuitBreaker.execute(async () => {
      const response = await fetch('https://api.example.com/data');
      if (!response.ok) throw new Error('API error');
      return response.json();
    });
    return result;
  } catch (error) {
    if (error instanceof CircuitOpenError) {
      // Return cached/fallback data
      return getCachedData();
    }
    throw error;
  }
}

// Multiple circuit breakers with registry
const registry = new CircuitBreakerRegistry();

const userServiceBreaker = registry.get('user-service', { failureThreshold: 3 });
const paymentServiceBreaker = registry.get('payment-service', { failureThreshold: 2 });

// Get all stats
console.log(registry.getAllStats());
*/

export default CircuitBreaker;

/**
 * Health Check Template
 * Usage: Implement health endpoints for your service
 */

// ===========================================
// Types
// ===========================================

export type HealthStatus = 'healthy' | 'degraded' | 'unhealthy';

export interface HealthCheckResult {
  name: string;
  status: HealthStatus;
  latency?: number;
  message?: string;
  metadata?: Record<string, unknown>;
}

export interface HealthReport {
  status: HealthStatus;
  timestamp: string;
  version: string;
  uptime: number;
  checks: HealthCheckResult[];
}

export type HealthCheck = () => Promise<HealthCheckResult>;

// ===========================================
// Health Check Manager
// ===========================================

export class HealthCheckManager {
  private checks: Map<string, HealthCheck> = new Map();
  private startTime: number = Date.now();
  private version: string;

  constructor(version: string = '1.0.0') {
    this.version = version;
  }

  /**
   * Register a health check
   */
  register(name: string, check: HealthCheck): void {
    this.checks.set(name, check);
  }

  /**
   * Run all health checks
   */
  async runChecks(): Promise<HealthReport> {
    const results: HealthCheckResult[] = [];

    for (const [name, check] of this.checks) {
      const start = Date.now();
      try {
        const result = await Promise.race([
          check(),
          this.timeout(5000, name),
        ]);
        result.latency = Date.now() - start;
        results.push(result);
      } catch (error) {
        results.push({
          name,
          status: 'unhealthy',
          latency: Date.now() - start,
          message: error instanceof Error ? error.message : 'Check failed',
        });
      }
    }

    const overallStatus = this.calculateOverallStatus(results);

    return {
      status: overallStatus,
      timestamp: new Date().toISOString(),
      version: this.version,
      uptime: Math.floor((Date.now() - this.startTime) / 1000),
      checks: results,
    };
  }

  /**
   * Run liveness check (basic alive check)
   */
  async livenessCheck(): Promise<{ status: 'ok' | 'error' }> {
    return { status: 'ok' };
  }

  /**
   * Run readiness check (ready to serve traffic)
   */
  async readinessCheck(): Promise<HealthReport> {
    return this.runChecks();
  }

  private async timeout(ms: number, name: string): Promise<HealthCheckResult> {
    return new Promise((_, reject) => {
      setTimeout(() => reject(new Error(`${name} timed out after ${ms}ms`)), ms);
    });
  }

  private calculateOverallStatus(results: HealthCheckResult[]): HealthStatus {
    if (results.some(r => r.status === 'unhealthy')) {
      return 'unhealthy';
    }
    if (results.some(r => r.status === 'degraded')) {
      return 'degraded';
    }
    return 'healthy';
  }
}

// ===========================================
// Common Health Checks
// ===========================================

/**
 * Database health check
 */
export function createDatabaseCheck(
  name: string,
  queryFn: () => Promise<unknown>
): HealthCheck {
  return async () => {
    try {
      await queryFn();
      return { name, status: 'healthy' };
    } catch (error) {
      return {
        name,
        status: 'unhealthy',
        message: error instanceof Error ? error.message : 'Database error',
      };
    }
  };
}

/**
 * Redis health check
 */
export function createRedisCheck(
  name: string,
  pingFn: () => Promise<string>
): HealthCheck {
  return async () => {
    try {
      const result = await pingFn();
      if (result === 'PONG') {
        return { name, status: 'healthy' };
      }
      return { name, status: 'degraded', message: `Unexpected response: ${result}` };
    } catch (error) {
      return {
        name,
        status: 'unhealthy',
        message: error instanceof Error ? error.message : 'Redis error',
      };
    }
  };
}

/**
 * HTTP dependency health check
 */
export function createHttpCheck(
  name: string,
  url: string,
  timeoutMs: number = 5000
): HealthCheck {
  return async () => {
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);

    try {
      const response = await fetch(url, {
        method: 'GET',
        signal: controller.signal,
      });
      clearTimeout(timeoutId);

      if (response.ok) {
        return { name, status: 'healthy', metadata: { statusCode: response.status } };
      }
      return {
        name,
        status: 'degraded',
        message: `HTTP ${response.status}`,
        metadata: { statusCode: response.status },
      };
    } catch (error) {
      clearTimeout(timeoutId);
      return {
        name,
        status: 'unhealthy',
        message: error instanceof Error ? error.message : 'HTTP error',
      };
    }
  };
}

/**
 * Memory usage check
 */
export function createMemoryCheck(
  name: string = 'memory',
  thresholdPercent: number = 90
): HealthCheck {
  return async () => {
    const used = process.memoryUsage();
    const heapUsedPercent = (used.heapUsed / used.heapTotal) * 100;

    if (heapUsedPercent < thresholdPercent) {
      return {
        name,
        status: 'healthy',
        metadata: {
          heapUsed: Math.round(used.heapUsed / 1024 / 1024),
          heapTotal: Math.round(used.heapTotal / 1024 / 1024),
          percent: Math.round(heapUsedPercent),
        },
      };
    }

    return {
      name,
      status: 'degraded',
      message: `Memory usage at ${Math.round(heapUsedPercent)}%`,
      metadata: { percent: Math.round(heapUsedPercent) },
    };
  };
}

/**
 * Disk space check (requires external command)
 */
export function createDiskCheck(
  name: string = 'disk',
  path: string = '/',
  thresholdPercent: number = 90
): HealthCheck {
  return async () => {
    // This is a placeholder - implement based on your OS
    // For Node.js, use 'diskusage' package or execute df command
    return { name, status: 'healthy', message: 'Implement disk check' };
  };
}

// ===========================================
// Express Integration
// ===========================================

/*
import express from 'express';

const app = express();
const health = new HealthCheckManager(process.env.VERSION || '1.0.0');

// Register checks
health.register('database', createDatabaseCheck('database', () => db.query('SELECT 1')));
health.register('redis', createRedisCheck('redis', () => redis.ping()));
health.register('memory', createMemoryCheck());

// Kubernetes-style endpoints
app.get('/healthz', async (req, res) => {
  const result = await health.livenessCheck();
  res.json(result);
});

app.get('/readyz', async (req, res) => {
  const report = await health.readinessCheck();
  const statusCode = report.status === 'healthy' ? 200 : 503;
  res.status(statusCode).json(report);
});

// Detailed health endpoint
app.get('/health', async (req, res) => {
  const report = await health.runChecks();
  const statusCode = report.status === 'healthy' ? 200 :
                     report.status === 'degraded' ? 200 : 503;
  res.status(statusCode).json(report);
});
*/

export { HealthCheckManager as default };