Incident Response

#!/usr/bin/env bash
#
# fetch-logs.sh -- Fetch recent logs from application services
#
# Retrieves logs from Docker containers for the specified service
# and time range. Writes raw logs and a filtered error summary to
# the output directory.
#
# Usage:
#   ./fetch-logs.sh --service backend --since "15 minutes ago" --output-dir ./logs/
#   ./fetch-logs.sh --service db --since "1 hour ago" --output-dir ./logs/ --filter ERROR
#
set -euo pipefail

# ─── Defaults ────────────────────────────────────────────────────────────────
SERVICE=""
SINCE="15m"
OUTPUT_DIR="./incident-logs"
FILTER=""
CONTAINER_PREFIX="app"
TAIL_LINES=1000

# ─── Parse Arguments ─────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    --service)          SERVICE="$2"; shift 2 ;;
    --since)            SINCE="$2"; shift 2 ;;
    --output-dir)       OUTPUT_DIR="$2"; shift 2 ;;
    --filter)           FILTER="$2"; shift 2 ;;
    --container-prefix) CONTAINER_PREFIX="$2"; shift 2 ;;
    --tail)             TAIL_LINES="$2"; shift 2 ;;
    -h|--help)
      echo "Usage: $0 --service <service-name> --output-dir <dir>"
      echo ""
      echo "Options:"
      echo "  --service          Service name (backend, frontend, db, redis)"
      echo "  --since            Time range for logs (default: 15m)"
      echo "  --output-dir       Directory for log output files"
      echo "  --filter           Filter logs by pattern (e.g., ERROR, WARNING)"
      echo "  --container-prefix Container name prefix (default: app)"
      echo "  --tail             Maximum number of lines (default: 1000)"
      exit 0
      ;;
    *) echo "ERROR: Unknown argument: $1" >&2; exit 1 ;;
  esac
done

if [[ -z "$SERVICE" ]]; then
  echo "ERROR: --service is required" >&2
  exit 1
fi

# ─── Setup Output ────────────────────────────────────────────────────────────
mkdir -p "$OUTPUT_DIR"
TIMESTAMP=$(date -u +"%Y%m%dT%H%M%SZ")
RAW_LOG_FILE="${OUTPUT_DIR}/${SERVICE}-raw-${TIMESTAMP}.log"
ERROR_LOG_FILE="${OUTPUT_DIR}/${SERVICE}-errors-${TIMESTAMP}.log"
SUMMARY_FILE="${OUTPUT_DIR}/${SERVICE}-summary-${TIMESTAMP}.json"

log() {
  echo "[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $1"
}

# ─── Determine Container Name ────────────────────────────────────────────────
CONTAINER_NAME="${CONTAINER_PREFIX}-${SERVICE}"

# Verify container exists
if ! docker ps -a --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
  log "ERROR: Container '${CONTAINER_NAME}' not found"
  log "Available containers:"
  docker ps -a --format '{{.Names}}' | while read -r name; do
    log "  - ${name}"
  done

  cat > "$SUMMARY_FILE" <<EOJSON
{
  "service": "${SERVICE}",
  "container": "${CONTAINER_NAME}",
  "status": "error",
  "error": "Container not found",
  "timestamp": "${TIMESTAMP}"
}
EOJSON
  exit 1
fi

# ─── Fetch Logs ──────────────────────────────────────────────────────────────
log "Fetching logs from ${CONTAINER_NAME} (since ${SINCE})..."

# Fetch raw logs
docker logs "${CONTAINER_NAME}" --since "${SINCE}" --tail "${TAIL_LINES}" \
  > "$RAW_LOG_FILE" 2>&1

TOTAL_LINES=$(wc -l < "$RAW_LOG_FILE")
log "Fetched ${TOTAL_LINES} lines of logs"

# ─── Extract Errors ─────────────────────────────────────────────────────────
log "Extracting errors and warnings..."

grep -iE "(ERROR|CRITICAL|FATAL|Exception|Traceback)" "$RAW_LOG_FILE" \
  > "$ERROR_LOG_FILE" 2>/dev/null || true

ERROR_LINES=$(wc -l < "$ERROR_LOG_FILE")
log "Found ${ERROR_LINES} error/exception lines"

# ─── Apply Custom Filter ────────────────────────────────────────────────────
FILTER_LINES=0
FILTER_FILE=""
if [[ -n "$FILTER" ]]; then
  FILTER_FILE="${OUTPUT_DIR}/${SERVICE}-filtered-${TIMESTAMP}.log"
  grep -i "$FILTER" "$RAW_LOG_FILE" > "$FILTER_FILE" 2>/dev/null || true
  FILTER_LINES=$(wc -l < "$FILTER_FILE")
  log "Filter '${FILTER}' matched ${FILTER_LINES} lines"
fi

# ─── Error Pattern Analysis ─────────────────────────────────────────────────
log "Analyzing error patterns..."

# Count error types
ERROR_COUNTS=""
if [[ $ERROR_LINES -gt 0 ]]; then
  ERROR_COUNTS=$(grep -oP '(ERROR|CRITICAL|FATAL|\w+Error|\w+Exception)' "$ERROR_LOG_FILE" \
    | sort | uniq -c | sort -rn | head -10 \
    | while read -r count pattern; do
        echo "    {\"pattern\": \"${pattern}\", \"count\": ${count}}"
      done | paste -sd ',' -)
fi

# ─── Container Status ───────────────────────────────────────────────────────
CONTAINER_STATUS=$(docker inspect "${CONTAINER_NAME}" --format '{{.State.Status}}' 2>/dev/null || echo "unknown")
CONTAINER_RESTARTS=$(docker inspect "${CONTAINER_NAME}" --format '{{.RestartCount}}' 2>/dev/null || echo "0")
CONTAINER_STARTED=$(docker inspect "${CONTAINER_NAME}" --format '{{.State.StartedAt}}' 2>/dev/null || echo "unknown")

# ─── Write Summary ──────────────────────────────────────────────────────────
cat > "$SUMMARY_FILE" <<EOJSON
{
  "service": "${SERVICE}",
  "container": "${CONTAINER_NAME}",
  "timestamp": "${TIMESTAMP}",
  "since": "${SINCE}",
  "status": "collected",
  "container_status": "${CONTAINER_STATUS}",
  "container_restarts": ${CONTAINER_RESTARTS},
  "container_started_at": "${CONTAINER_STARTED}",
  "total_log_lines": ${TOTAL_LINES},
  "error_lines": ${ERROR_LINES},
  "filter_pattern": "${FILTER}",
  "filter_matches": ${FILTER_LINES},
  "files": {
    "raw_log": "${RAW_LOG_FILE}",
    "error_log": "${ERROR_LOG_FILE}",
    "filtered_log": "${FILTER_FILE:-null}",
    "summary": "${SUMMARY_FILE}"
  },
  "top_error_patterns": [${ERROR_COUNTS}]
}
EOJSON

# ─── Output Summary ─────────────────────────────────────────────────────────
log ""
log "=== Log Collection Summary ==="
log "Service:          ${SERVICE}"
log "Container:        ${CONTAINER_NAME} (${CONTAINER_STATUS})"
log "Restarts:         ${CONTAINER_RESTARTS}"
log "Total lines:      ${TOTAL_LINES}"
log "Error lines:      ${ERROR_LINES}"
if [[ -n "$FILTER" ]]; then
  log "Filter matches:   ${FILTER_LINES} (pattern: ${FILTER})"
fi
log "Raw log:          ${RAW_LOG_FILE}"
log "Error log:        ${ERROR_LOG_FILE}"
log "Summary:          ${SUMMARY_FILE}"

if [[ $ERROR_LINES -gt 0 ]]; then
  log ""
  log "=== Recent Errors (last 5) ==="
  tail -5 "$ERROR_LOG_FILE"
fi

exit 0

#!/usr/bin/env python3
"""
generate-incident-report.py -- Generate a structured incident report.

Creates a markdown incident report and a JSON summary from provided
incident details. Used during or after an incident to document findings.

Usage:
    python generate-incident-report.py \
        --title "Database connection pool exhaustion" \
        --severity SEV2 \
        --start-time "2024-01-15T14:30:00Z" \
        --end-time "2024-01-15T15:15:00Z" \
        --output-dir ./post-mortems/

    python generate-incident-report.py \
        --title "API gateway 502 errors" \
        --severity SEV1 \
        --start-time "2024-01-15T14:30:00Z" \
        --impact "All API requests returning 502" \
        --root-cause "Expired TLS certificate on load balancer" \
        --output-dir ./post-mortems/
"""

import argparse
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Generate a structured incident report"
    )
    parser.add_argument(
        "--title",
        required=True,
        help="Incident title (brief description)",
    )
    parser.add_argument(
        "--severity",
        required=True,
        choices=["SEV1", "SEV2", "SEV3", "SEV4"],
        help="Incident severity level",
    )
    parser.add_argument(
        "--start-time",
        required=True,
        help="Incident start time in ISO 8601 format (e.g., 2024-01-15T14:30:00Z)",
    )
    parser.add_argument(
        "--end-time",
        default=None,
        help="Incident end time in ISO 8601 format (omit if ongoing)",
    )
    parser.add_argument(
        "--output-dir",
        default="./incident-reports",
        help="Directory for report output files (default: ./incident-reports)",
    )
    parser.add_argument(
        "--impact",
        default="",
        help="Description of user-facing impact",
    )
    parser.add_argument(
        "--root-cause",
        default="",
        help="Root cause (if known)",
    )
    parser.add_argument(
        "--commander",
        default="",
        help="Incident commander name",
    )
    parser.add_argument(
        "--services-affected",
        nargs="+",
        default=[],
        help="List of affected services",
    )
    parser.add_argument(
        "--timeline-events",
        nargs="+",
        default=[],
        help="Timeline events in 'HH:MM description' format",
    )
    return parser.parse_args()


def parse_iso_time(time_str: str) -> datetime:
    """Parse an ISO 8601 time string."""
    try:
        if time_str.endswith("Z"):
            time_str = time_str[:-1] + "+00:00"
        return datetime.fromisoformat(time_str)
    except ValueError:
        print(f"ERROR: Invalid time format: {time_str}", file=sys.stderr)
        print("Expected ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ", file=sys.stderr)
        sys.exit(1)


def calculate_duration(start: datetime, end: datetime | None) -> str:
    """Calculate human-readable duration between two times."""
    if end is None:
        return "Ongoing"
    delta = end - start
    total_seconds = int(delta.total_seconds())
    hours, remainder = divmod(total_seconds, 3600)
    minutes, seconds = divmod(remainder, 60)
    if hours > 0:
        return f"{hours}h {minutes}m"
    return f"{minutes}m {seconds}s"


def generate_markdown_report(args: argparse.Namespace, duration: str) -> str:
    """Generate a markdown incident report."""
    now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
    status = "Resolved" if args.end_time else "Ongoing"

    services = ", ".join(args.services_affected) if args.services_affected else "TBD"

    report = f"""# Incident Report: {args.title}

## Summary

| Field | Value |
|-------|-------|
| **Severity** | {args.severity} |
| **Status** | {status} |
| **Start Time** | {args.start_time} |
| **End Time** | {args.end_time or "Ongoing"} |
| **Duration** | {duration} |
| **Commander** | {args.commander or "TBD"} |
| **Services Affected** | {services} |
| **Report Generated** | {now} |

## Impact

{args.impact or "_Describe the user-facing impact here._"}

## Timeline

| Time (UTC) | Event |
|------------|-------|
| {args.start_time} | Incident detected |
"""

    if args.timeline_events:
        for event in args.timeline_events:
            parts = event.split(" ", 1)
            if len(parts) == 2:
                report += f"| {parts[0]} | {parts[1]} |\n"
            else:
                report += f"| -- | {event} |\n"

    if args.end_time:
        report += f"| {args.end_time} | Incident resolved |\n"

    report += f"""
_Add more timeline entries as the investigation progresses._

## Root Cause

{args.root_cause or "_To be determined during post-mortem investigation._"}

## Contributing Factors

- _List conditions that made the incident more likely or more severe_
- _Example: No alert configured for connection pool usage_
- _Example: Recent deployment changed database query pattern_

## Detection

- How was the incident detected? (Alert, user report, monitoring dashboard)
- How long between incident start and detection?
- Could detection have been faster?

## Response

- What mitigation steps were taken?
- Were the right people involved quickly enough?
- Was the runbook helpful?

## What Went Well

- _Example: Alert fired within 2 minutes of impact_
- _Example: Rollback procedure worked correctly_
- _Example: Clear communication in incident channel_

## What Could Be Improved

- _Example: No alert for connection pool exhaustion_
- _Example: Took 10 minutes to identify the root cause_
- _Example: Runbook did not cover this failure mode_

## Action Items

| Action | Owner | Priority | Due Date | Status |
|--------|-------|----------|----------|--------|
| _Add alert for [metric]_ | TBD | High | TBD | Open |
| _Add regression test for [scenario]_ | TBD | Medium | TBD | Open |
| _Update runbook with [procedure]_ | TBD | Medium | TBD | Open |
| _Fix root cause in [component]_ | TBD | High | TBD | Open |

## Lessons Learned

_Key takeaways from this incident that the broader team should know about._

---

_This report was generated by `generate-incident-report.py`. Fill in the TBD
sections during the post-mortem meeting._
"""
    return report


def main() -> int:
    args = parse_args()

    # Setup output directory
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)

    # Parse times
    start_time = parse_iso_time(args.start_time)
    end_time = parse_iso_time(args.end_time) if args.end_time else None
    duration = calculate_duration(start_time, end_time)

    # Generate file names
    date_prefix = start_time.strftime("%Y%m%d")
    safe_title = args.title.lower().replace(" ", "-")[:50]
    base_name = f"incident-{date_prefix}-{safe_title}"

    markdown_file = output_dir / f"{base_name}.md"
    json_file = output_dir / f"{base_name}.json"

    # Generate markdown report
    markdown_content = generate_markdown_report(args, duration)
    with open(markdown_file, "w") as f:
        f.write(markdown_content)

    # Generate JSON summary
    json_summary = {
        "title": args.title,
        "severity": args.severity,
        "status": "resolved" if args.end_time else "ongoing",
        "start_time": args.start_time,
        "end_time": args.end_time,
        "duration": duration,
        "commander": args.commander or None,
        "impact": args.impact or None,
        "root_cause": args.root_cause or None,
        "services_affected": args.services_affected,
        "timeline_events": args.timeline_events,
        "report_generated": datetime.now(timezone.utc).isoformat(),
        "files": {
            "markdown": str(markdown_file),
            "json": str(json_file),
        },
    }

    with open(json_file, "w") as f:
        json.dump(json_summary, f, indent=2, default=str)

    # Output summary
    print(f"Incident report generated:")
    print(f"  Markdown: {markdown_file}")
    print(f"  JSON:     {json_file}")
    print(f"  Severity: {args.severity}")
    print(f"  Duration: {duration}")
    print(f"  Status:   {'Resolved' if args.end_time else 'Ongoing'}")

    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env bash
#
# health-check-all-services.sh -- Check health of all application services
#
# Checks liveness and connectivity of backend, frontend, database, and Redis
# services. Writes a structured report to the output directory.
#
# Usage:
#   ./health-check-all-services.sh --output-dir ./health-results/
#   ./health-check-all-services.sh --backend-url https://api.example.com --output-dir ./results/
#
set -euo pipefail

# ─── Defaults ────────────────────────────────────────────────────────────────
OUTPUT_DIR="./health-check-results"
BACKEND_URL="${BACKEND_URL:-http://localhost:8000}"
FRONTEND_URL="${FRONTEND_URL:-http://localhost:3000}"
DB_CONTAINER="${DB_CONTAINER:-app-db}"
REDIS_CONTAINER="${REDIS_CONTAINER:-app-redis}"
TIMEOUT=10

# ─── Parse Arguments ─────────────────────────────────────────────────────────
while [[ $# -gt 0 ]]; do
  case "$1" in
    --output-dir)      OUTPUT_DIR="$2"; shift 2 ;;
    --backend-url)     BACKEND_URL="$2"; shift 2 ;;
    --frontend-url)    FRONTEND_URL="$2"; shift 2 ;;
    --db-container)    DB_CONTAINER="$2"; shift 2 ;;
    --redis-container) REDIS_CONTAINER="$2"; shift 2 ;;
    --timeout)         TIMEOUT="$2"; shift 2 ;;
    -h|--help)
      echo "Usage: $0 --output-dir <dir>"
      echo ""
      echo "Options:"
      echo "  --output-dir      Directory for health check result files"
      echo "  --backend-url     Backend service URL (default: http://localhost:8000)"
      echo "  --frontend-url    Frontend service URL (default: http://localhost:3000)"
      echo "  --db-container    Database container name (default: app-db)"
      echo "  --redis-container Redis container name (default: app-redis)"
      echo "  --timeout         HTTP request timeout in seconds (default: 10)"
      exit 0
      ;;
    *) echo "ERROR: Unknown argument: $1" >&2; exit 1 ;;
  esac
done

# ─── Setup Output ────────────────────────────────────────────────────────────
mkdir -p "$OUTPUT_DIR"
TIMESTAMP=$(date -u +"%Y%m%dT%H%M%SZ")
RESULT_FILE="${OUTPUT_DIR}/health-all-services-${TIMESTAMP}.json"
LOG_FILE="${OUTPUT_DIR}/health-all-services-${TIMESTAMP}.log"

log() {
  local msg="[$(date -u +"%Y-%m-%dT%H:%M:%SZ")] $1"
  echo "$msg" | tee -a "$LOG_FILE"
}

TOTAL=0
HEALTHY=0
UNHEALTHY=0
RESULTS=""

# ─── Check Function ─────────────────────────────────────────────────────────
check_http() {
  local name="$1"
  local url="$2"
  local expected_status="${3:-200}"

  TOTAL=$((TOTAL + 1))
  log "Checking ${name}: ${url}"

  local start_time
  start_time=$(date +%s%N)

  local http_code
  http_code=$(curl -s -o /dev/null -w "%{http_code}" \
    --max-time "$TIMEOUT" "$url" 2>/dev/null) || http_code="000"

  local end_time
  end_time=$(date +%s%N)
  local elapsed_ms=$(( (end_time - start_time) / 1000000 ))

  local status="unhealthy"
  if [[ "$http_code" == "$expected_status" ]]; then
    status="healthy"
    HEALTHY=$((HEALTHY + 1))
    log "  OK: ${name} returned ${http_code} in ${elapsed_ms}ms"
  else
    UNHEALTHY=$((UNHEALTHY + 1))
    log "  FAIL: ${name} returned ${http_code} (expected ${expected_status}) in ${elapsed_ms}ms"
  fi

  local entry="{\"name\":\"${name}\",\"url\":\"${url}\",\"status\":\"${status}\",\"http_code\":${http_code},\"response_time_ms\":${elapsed_ms}}"
  if [[ -n "$RESULTS" ]]; then
    RESULTS="${RESULTS},${entry}"
  else
    RESULTS="${entry}"
  fi
}

check_container() {
  local name="$1"
  local container="$2"
  local check_cmd="$3"

  TOTAL=$((TOTAL + 1))
  log "Checking ${name}: container ${container}"

  local status="unhealthy"
  local detail=""

  # Check if container is running
  if ! docker ps --format '{{.Names}}' | grep -q "^${container}$"; then
    detail="Container not running"
    UNHEALTHY=$((UNHEALTHY + 1))
    log "  FAIL: ${name} - container '${container}' not running"
  else
    # Run the health check command
    if output=$(docker exec "$container" sh -c "$check_cmd" 2>&1); then
      status="healthy"
      detail="$output"
      HEALTHY=$((HEALTHY + 1))
      log "  OK: ${name} is responsive"
    else
      detail="$output"
      UNHEALTHY=$((UNHEALTHY + 1))
      log "  FAIL: ${name} - ${output}"
    fi
  fi

  local detail_escaped
  detail_escaped=$(echo "$detail" | tr '"' "'" | tr '\n' ' ' | head -c 200)

  local entry="{\"name\":\"${name}\",\"container\":\"${container}\",\"status\":\"${status}\",\"detail\":\"${detail_escaped}\"}"
  if [[ -n "$RESULTS" ]]; then
    RESULTS="${RESULTS},${entry}"
  else
    RESULTS="${entry}"
  fi
}

# ─── Run Health Checks ──────────────────────────────────────────────────────
log "=== Health Check: All Services ==="
log "Timestamp: ${TIMESTAMP}"
log ""

# Backend health checks
log "--- Backend ---"
check_http "Backend Liveness" "${BACKEND_URL}/health" "200"
check_http "Backend Readiness" "${BACKEND_URL}/health/ready" "200"
check_http "Backend API" "${BACKEND_URL}/api/v1/" "200"
log ""

# Frontend health checks
log "--- Frontend ---"
check_http "Frontend" "${FRONTEND_URL}/" "200"
log ""

# Database health checks
log "--- Database ---"
check_container "PostgreSQL" "${DB_CONTAINER}" "pg_isready -U postgres"
log ""

# Redis health checks
log "--- Redis ---"
check_container "Redis" "${REDIS_CONTAINER}" "redis-cli ping"
log ""

# ─── Docker Container Status ────────────────────────────────────────────────
log "--- Container Status ---"
CONTAINER_STATUS=$(docker ps --format "{{.Names}}\t{{.Status}}\t{{.Ports}}" 2>/dev/null || echo "Docker not available")
log "$CONTAINER_STATUS"
log ""

# ─── Write Results ──────────────────────────────────────────────────────────
OVERALL="healthy"
if [[ $UNHEALTHY -gt 0 ]]; then
  OVERALL="unhealthy"
fi

cat > "$RESULT_FILE" <<EOJSON
{
  "timestamp": "${TIMESTAMP}",
  "overall_status": "${OVERALL}",
  "total_checks": ${TOTAL},
  "healthy": ${HEALTHY},
  "unhealthy": ${UNHEALTHY},
  "services": {
    "backend_url": "${BACKEND_URL}",
    "frontend_url": "${FRONTEND_URL}",
    "db_container": "${DB_CONTAINER}",
    "redis_container": "${REDIS_CONTAINER}"
  },
  "results": [${RESULTS}]
}
EOJSON

# ─── Summary ─────────────────────────────────────────────────────────────────
log "=== Health Check Summary ==="
log "Overall: ${OVERALL}"
log "Total: ${TOTAL}  Healthy: ${HEALTHY}  Unhealthy: ${UNHEALTHY}"
log "Results: ${RESULT_FILE}"
log "Log: ${LOG_FILE}"

if [[ $UNHEALTHY -gt 0 ]]; then
  log ""
  log "WARNING: ${UNHEALTHY} service(s) are unhealthy"
  exit 1
fi

log "ALL SERVICES HEALTHY"
exit 0