K8s Debug

#!/usr/bin/env bash
# Kubernetes Cluster Health Check Script
# Performs comprehensive cluster diagnostics with graceful fallbacks.

set -uo pipefail

REQUEST_TIMEOUT="${K8S_REQUEST_TIMEOUT:-15s}"
STRICT_MODE=0

WARN_COUNT=0
CHECK_FAIL_COUNT=0
BLOCKED_COUNT=0

usage() {
    echo "Usage: $0 [--strict]"
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        --strict)
            STRICT_MODE=1
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        -*)
            echo "ERROR: Unknown option '$1'." >&2
            usage
            exit 1
            ;;
        *)
            echo "ERROR: Unexpected positional argument '$1'." >&2
            usage
            exit 1
            ;;
    esac
done

timestamp_utc() {
    date -u +"%Y-%m-%d %H:%M:%S UTC"
}

section() {
    printf "\n## %s ##\n" "$1"
}

warn_raw() {
    printf "WARN: %s\n" "$1" >&2
}

warn() {
    warn_raw "$1"
    WARN_COUNT=$((WARN_COUNT + 1))
}

info() {
    printf "INFO: %s\n" "$1"
}

have_cmd() {
    command -v "$1" >/dev/null 2>&1
}

kubectl_cmd() {
    kubectl --request-timeout="$REQUEST_TIMEOUT" "$@"
}

run_or_warn() {
    local description="$1"
    shift
    if ! "$@"; then
        warn_raw "${description} failed; continuing."
        CHECK_FAIL_COUNT=$((CHECK_FAIL_COUNT + 1))
        return 1
    fi
    return 0
}

run_pipe_or_warn() {
    local description="$1"
    local cmd="$2"
    if ! bash -o pipefail -c "$cmd"; then
        warn_raw "${description} failed; continuing."
        CHECK_FAIL_COUNT=$((CHECK_FAIL_COUNT + 1))
        return 1
    fi
    return 0
}

blocked_exit() {
    local message="$1"
    BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
    printf "ERROR: %s\n" "$message" >&2
    exit 2
}

find_waiting_reason_pods() {
    local reason="$1"

    if have_cmd jq; then
        local output
        if ! output="$(
            kubectl_cmd get pods --all-namespaces -o json 2>/dev/null | \
                jq -r --arg reason "$reason" \
                    '.items[] | select(any(.status.containerStatuses[]?; .state.waiting?.reason == $reason)) | "\(.metadata.namespace)/\(.metadata.name)"'
        )"; then
            warn "Unable to query pods in waiting reason ${reason}."
            return 1
        fi
        if [ -n "$output" ]; then
            printf "%s\n" "$output"
        else
            echo "None found"
        fi
        return 0
    fi

    warn "jq is not installed; showing all non-running pods as fallback for ${reason}."
    kubectl_cmd get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded
}

finalize_exit() {
    if [ "$BLOCKED_COUNT" -gt 0 ]; then
        return 2
    fi
    if [ "$CHECK_FAIL_COUNT" -gt 0 ]; then
        return 1
    fi
    if [ "$STRICT_MODE" -eq 1 ] && [ "$WARN_COUNT" -gt 0 ]; then
        return 1
    fi
    return 0
}

if ! have_cmd kubectl; then
    blocked_exit "kubectl is not installed or not in PATH."
fi

if ! kubectl_cmd config current-context >/dev/null 2>&1; then
    blocked_exit "No active Kubernetes context. Run 'kubectl config current-context' to troubleshoot."
fi

echo "========================================"
echo "Kubernetes Cluster Health Check"
echo "Timestamp: $(timestamp_utc)"
echo "========================================"

section "PREFLIGHT"
run_or_warn "Current context check" kubectl_cmd config current-context
if ! have_cmd jq; then
    info "jq is optional. Error-state filtering will use a broader fallback."
    warn "jq is not installed; waiting-reason filtering will fall back to non-running pod lists."
fi

section "CLUSTER INFO"
run_or_warn "Cluster info" kubectl_cmd cluster-info
run_pipe_or_warn "Cluster version" "kubectl --request-timeout=\"$REQUEST_TIMEOUT\" version --client=false 2>/dev/null || kubectl --request-timeout=\"$REQUEST_TIMEOUT\" version"

section "NODE STATUS"
run_or_warn "Node list" kubectl_cmd get nodes -o wide
echo -e "\nNode Conditions:"
run_or_warn "Node readiness condition query" kubectl_cmd get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}'

section "NODE RESOURCE USAGE"
run_or_warn "Node metrics (requires metrics-server)" kubectl_cmd top nodes

section "NAMESPACE OVERVIEW"
run_or_warn "Namespace list" kubectl_cmd get namespaces

section "PODS STATUS (ALL NAMESPACES)"
run_or_warn "Pod list" kubectl_cmd get pods --all-namespaces -o wide

section "PROBLEMATIC PODS"
run_or_warn "Non-running/non-succeeded pod list" kubectl_cmd get pods --all-namespaces --field-selector=status.phase!=Running,status.phase!=Succeeded

section "RECENT CLUSTER EVENTS"
run_pipe_or_warn "Recent events query" "kubectl --request-timeout=\"$REQUEST_TIMEOUT\" get events --all-namespaces --sort-by='.lastTimestamp' | tail -50"

section "DEPLOYMENTS"
run_or_warn "Deployment list" kubectl_cmd get deployments --all-namespaces

section "SERVICES"
run_or_warn "Service list" kubectl_cmd get services --all-namespaces

section "STATEFULSETS"
run_or_warn "StatefulSet list" kubectl_cmd get statefulsets --all-namespaces

section "DAEMONSETS"
run_or_warn "DaemonSet list" kubectl_cmd get daemonsets --all-namespaces

section "PERSISTENT VOLUME CLAIMS"
run_or_warn "PVC list" kubectl_cmd get pvc --all-namespaces

section "PERSISTENT VOLUMES"
run_or_warn "PV list" kubectl_cmd get pv

section "COMPONENT STATUS"
run_pipe_or_warn "Component readiness endpoint query" "kubectl --request-timeout=\"$REQUEST_TIMEOUT\" get --raw='/readyz?verbose' 2>/dev/null || kubectl --request-timeout=\"$REQUEST_TIMEOUT\" get --raw='/healthz?verbose' 2>/dev/null || kubectl --request-timeout=\"$REQUEST_TIMEOUT\" get componentstatuses"

section "API SERVER HEALTH"
run_or_warn "API server health check" kubectl_cmd get '--raw=/healthz?verbose'

section "CRASHLOOPBACKOFF PODS"
run_or_warn "CrashLoopBackOff pod query" find_waiting_reason_pods "CrashLoopBackOff"

section "IMAGEPULLBACKOFF PODS"
run_or_warn "ImagePullBackOff pod query" find_waiting_reason_pods "ImagePullBackOff"

section "NETWORK POLICIES"
run_or_warn "Network policy list" kubectl_cmd get networkpolicies --all-namespaces

section "RESOURCE QUOTAS"
run_or_warn "Resource quota list" kubectl_cmd get resourcequotas --all-namespaces

section "INGRESSES"
run_or_warn "Ingress list" kubectl_cmd get ingresses --all-namespaces

echo -e "\n========================================"
echo "Health check completed at $(timestamp_utc)"
echo "Warnings: $WARN_COUNT | Check failures: $CHECK_FAIL_COUNT | Blocked checks: $BLOCKED_COUNT"
echo "========================================"

finalize_exit
exit $?

#!/usr/bin/env bash
# Kubernetes Network Debugging Script
# Diagnoses pod/service connectivity with graceful fallbacks.

set -uo pipefail

REQUEST_TIMEOUT="${K8S_REQUEST_TIMEOUT:-15s}"
NAMESPACE="default"
POD_NAME=""
STRICT_MODE=0
INSECURE_TLS=0

WARN_COUNT=0
CHECK_FAIL_COUNT=0
BLOCKED_COUNT=0

SERVICEACCOUNT_DIR="/var/run/secrets/kubernetes.io/serviceaccount"
SERVICEACCOUNT_CA="${SERVICEACCOUNT_DIR}/ca.crt"
SERVICEACCOUNT_TOKEN_FILE="${SERVICEACCOUNT_DIR}/token"
KUBERNETES_API_URL="https://kubernetes.default.svc/api"

usage() {
    echo "Usage: $0 [--strict] [--insecure] [namespace] <pod-name>"
    echo "Examples:"
    echo "  $0 my-pod"
    echo "  $0 default my-pod"
    echo "  $0 --insecure default my-pod"
}

while [ "$#" -gt 0 ]; do
    case "$1" in
        --strict)
            STRICT_MODE=1
            shift
            ;;
        --insecure)
            INSECURE_TLS=1
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        --)
            shift
            break
            ;;
        -*)
            echo "ERROR: Unknown option '$1'." >&2
            usage
            exit 1
            ;;
        *)
            break
            ;;
    esac
done

case "$#" in
    1)
        POD_NAME="$1"
        ;;
    2)
        NAMESPACE="$1"
        POD_NAME="$2"
        ;;
    *)
        usage
        exit 1
        ;;
esac

timestamp_utc() {
    date -u +"%Y-%m-%d %H:%M:%S UTC"
}

section() {
    printf "\n## %s ##\n" "$1"
}

warn_raw() {
    printf "WARN: %s\n" "$1" >&2
}

warn() {
    warn_raw "$1"
    WARN_COUNT=$((WARN_COUNT + 1))
}

info() {
    printf "INFO: %s\n" "$1"
}

have_cmd() {
    command -v "$1" >/dev/null 2>&1
}

kubectl_cmd() {
    kubectl --request-timeout="$REQUEST_TIMEOUT" "$@"
}

can_i() {
    local result
    result="$(kubectl_cmd auth can-i "$@" 2>/dev/null || true)"
    [ "$result" = "yes" ]
}

record_check_failure() {
    local message="$1"
    warn_raw "$message"
    CHECK_FAIL_COUNT=$((CHECK_FAIL_COUNT + 1))
}

run_or_warn() {
    local description="$1"
    shift
    if ! "$@"; then
        record_check_failure "${description} failed; continuing."
        return 1
    fi
    return 0
}

run_pipe_or_warn() {
    local description="$1"
    local cmd="$2"
    if ! bash -o pipefail -c "$cmd"; then
        record_check_failure "${description} failed; continuing."
        return 1
    fi
    return 0
}

pod_exec() {
    kubectl_cmd exec "$POD_NAME" -n "$NAMESPACE" -- "$@"
}

blocked_exit() {
    local message="$1"
    BLOCKED_COUNT=$((BLOCKED_COUNT + 1))
    printf "ERROR: %s\n" "$message" >&2
    exit 2
}

read_serviceaccount_token() {
    local token
    token="$(pod_exec cat "$SERVICEACCOUNT_TOKEN_FILE" 2>/dev/null || true)"
    token="${token//$'\r'/}"
    token="${token//$'\n'/}"
    printf "%s" "$token"
}

api_probe_secure() {
    local token

    if ! pod_exec test -r "$SERVICEACCOUNT_CA" >/dev/null 2>&1 || \
        ! pod_exec test -r "$SERVICEACCOUNT_TOKEN_FILE" >/dev/null 2>&1; then
        echo "service account CA/token files are missing in the pod. Use --insecure only for explicit troubleshooting override." >&2
        return 1
    fi

    token="$(read_serviceaccount_token)"
    if [ -z "$token" ]; then
        echo "service account token is empty; cannot authenticate secure API probe." >&2
        return 1
    fi

    if pod_exec curl --fail --silent --show-error --cacert "$SERVICEACCOUNT_CA" --max-time 5 \
        -H "Authorization: Bearer $token" "$KUBERNETES_API_URL" >/dev/null 2>&1; then
        return 0
    fi

    if pod_exec wget -q --timeout=5 --ca-certificate="$SERVICEACCOUNT_CA" \
        --header="Authorization: Bearer $token" -O /dev/null "$KUBERNETES_API_URL" >/dev/null 2>&1; then
        return 0
    fi

    echo "curl/wget secure API probe failed in the container (missing tools, auth failure, or blocked egress)." >&2
    return 1
}

api_probe_insecure() {
    local token
    token="$(read_serviceaccount_token)"
    warn "Insecure TLS mode enabled (--insecure). Certificate validation is bypassed for API probe."

    if [ -n "$token" ]; then
        if pod_exec curl --fail --silent --show-error -k --max-time 5 \
            -H "Authorization: Bearer $token" "$KUBERNETES_API_URL" >/dev/null 2>&1; then
            return 0
        fi
        if pod_exec wget -q --timeout=5 --no-check-certificate \
            --header="Authorization: Bearer $token" -O /dev/null "$KUBERNETES_API_URL" >/dev/null 2>&1; then
            return 0
        fi
    else
        if pod_exec curl --fail --silent --show-error -k --max-time 5 \
            "$KUBERNETES_API_URL" >/dev/null 2>&1; then
            return 0
        fi
        if pod_exec wget -q --timeout=5 --no-check-certificate \
            -O /dev/null "$KUBERNETES_API_URL" >/dev/null 2>&1; then
            return 0
        fi
    fi

    echo "curl/wget insecure API probe failed in the container (missing tools, auth failure, or blocked egress)." >&2
    return 1
}

api_probe() {
    if [ "$INSECURE_TLS" -eq 1 ]; then
        api_probe_insecure
        return $?
    fi
    api_probe_secure
}

finalize_exit() {
    if [ "$BLOCKED_COUNT" -gt 0 ]; then
        return 2
    fi
    if [ "$CHECK_FAIL_COUNT" -gt 0 ]; then
        return 1
    fi
    if [ "$STRICT_MODE" -eq 1 ] && [ "$WARN_COUNT" -gt 0 ]; then
        return 1
    fi
    return 0
}

if ! have_cmd kubectl; then
    blocked_exit "kubectl is not installed or not in PATH."
fi

if ! kubectl_cmd config current-context >/dev/null 2>&1; then
    blocked_exit "No active Kubernetes context. Run 'kubectl config current-context' to troubleshoot."
fi

if ! kubectl_cmd get namespace "$NAMESPACE" >/dev/null 2>&1; then
    blocked_exit "Namespace '$NAMESPACE' was not found or is not accessible."
fi

if ! kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then
    blocked_exit "Pod '$POD_NAME' in namespace '$NAMESPACE' was not found or is not accessible."
fi

echo "========================================"
echo "Network Debugging for Pod: $POD_NAME"
echo "Namespace: $NAMESPACE"
echo "Timestamp: $(timestamp_utc)"
echo "========================================"

section "PREFLIGHT"
run_or_warn "Current context check" kubectl_cmd config current-context
if ! can_i get pods -n "$NAMESPACE"; then
    warn "RBAC may block pod metadata reads in namespace '$NAMESPACE'."
fi
if ! can_i create pods/exec -n "$NAMESPACE"; then
    warn "RBAC may block 'kubectl exec'; in-pod checks may fail."
fi

section "POD NETWORK INFORMATION"
POD_IP="$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.podIP}' 2>/dev/null || true)"
HOST_IP="$(kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o jsonpath='{.status.hostIP}' 2>/dev/null || true)"
echo "Pod IP: ${POD_IP:-Unavailable}"
echo "Host IP: ${HOST_IP:-Unavailable}"
run_or_warn "Pod wide status query" kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" -o wide

section "DNS CONFIGURATION"
run_or_warn "Pod DNS config read" pod_exec cat /etc/resolv.conf

section "DNS RESOLUTION TEST"
echo "Testing kubernetes.default.svc.cluster.local:"
if pod_exec nslookup kubernetes.default.svc.cluster.local 2>/dev/null; then
    :
elif pod_exec getent hosts kubernetes.default.svc.cluster.local 2>/dev/null; then
    :
else
    record_check_failure "DNS lookup test failed (utilities unavailable or DNS lookup failed)."
fi

section "NETWORK CONNECTIVITY TESTS"
echo "Testing connection to kubernetes.default.svc:"
run_or_warn "Kubernetes API connectivity test from pod" api_probe

section "SERVICES IN NAMESPACE"
run_or_warn "Service list query" kubectl_cmd get svc -n "$NAMESPACE"

section "ENDPOINTS"
run_or_warn "Endpoint list query" kubectl_cmd get endpoints -n "$NAMESPACE"

section "NETWORK POLICIES"
run_or_warn "Network policy list query" kubectl_cmd get networkpolicies -n "$NAMESPACE"

section "POD NETWORK DETAILS"
run_pipe_or_warn "Pod describe network details query" "kubectl --request-timeout=\"$REQUEST_TIMEOUT\" describe pod \"$POD_NAME\" -n \"$NAMESPACE\" | grep -A 20 '^IP:'"

section "POD LABELS (FOR NETWORKPOLICY MATCHING)"
run_or_warn "Pod label query" kubectl_cmd get pod "$POD_NAME" -n "$NAMESPACE" --show-labels

section "IPTABLES RULES (IF ACCESSIBLE)"
if ! pod_exec iptables -L -n 2>/dev/null; then
    info "iptables output not available (requires privileged container/tools)."
fi

section "NETWORK INTERFACES"
if pod_exec ip addr 2>/dev/null; then
    :
elif pod_exec ifconfig 2>/dev/null; then
    :
else
    info "Network interface tools are not available in this container."
fi

section "ROUTING TABLE"
if pod_exec ip route 2>/dev/null; then
    :
elif pod_exec route 2>/dev/null; then
    :
else
    info "Routing table tools are not available in this container."
fi

section "COREDNS LOGS (LAST 20 LINES)"
if kubectl_cmd logs -n kube-system -l k8s-app=kube-dns --tail=20 2>/dev/null; then
    :
elif kubectl_cmd logs -n kube-system -l k8s-app=coredns --tail=20 2>/dev/null; then
    :
else
    warn "CoreDNS logs are not accessible."
fi

echo -e "\n========================================"
echo "Network debugging completed at $(timestamp_utc)"
echo "Warnings: $WARN_COUNT | Check failures: $CHECK_FAIL_COUNT | Blocked checks: $BLOCKED_COUNT"
echo "========================================"

finalize_exit
exit $?

#!/usr/bin/env python3
"""
Kubernetes Pod Diagnostics Script
Gathers comprehensive diagnostic information about a specific pod
with explicit preflight checks and graceful fallbacks.
"""

import argparse
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from typing import Sequence, Tuple

REQUEST_TIMEOUT = os.environ.get("K8S_REQUEST_TIMEOUT", "15s")


def run_kubectl(args: Sequence[str], timeout: int = 30) -> Tuple[str, str, int]:
    """Execute kubectl command and return (stdout, stderr, exit_code)."""
    cmd = ["kubectl", f"--request-timeout={REQUEST_TIMEOUT}", *args]
    try:
        result = subprocess.run(
            cmd,
            capture_output=True,
            text=True,
            timeout=timeout,
            check=False,
        )
        return result.stdout, result.stderr, result.returncode
    except subprocess.TimeoutExpired:
        return "", f"Command timed out: {' '.join(cmd)}", 1


def print_output(stdout: str, stderr: str) -> None:
    """Print command output, preferring stdout then stderr."""
    if stdout.strip():
        print(stdout.rstrip())
    elif stderr.strip():
        print(stderr.rstrip())


def print_section(title: str) -> None:
    print(f"\n## {title} ##")


def ensure_prerequisites(namespace: str, pod_name: str) -> bool:
    """Validate local tool availability and cluster access prerequisites."""
    if shutil.which("kubectl") is None:
        print("ERROR: kubectl is not installed or not in PATH.", file=sys.stderr)
        return False

    stdout, stderr, code = run_kubectl(["config", "current-context"])
    if code != 0:
        print("ERROR: Unable to determine active Kubernetes context.", file=sys.stderr)
        print_output(stdout, stderr)
        return False

    stdout, stderr, code = run_kubectl(["get", "pod", pod_name, "-n", namespace, "-o", "name"])
    if code != 0:
        print(
            f"ERROR: Pod '{pod_name}' in namespace '{namespace}' is not accessible.",
            file=sys.stderr,
        )
        print_output(stdout, stderr)
        return False

    stdout, _, _ = run_kubectl(["auth", "can-i", "create", "pods/exec", "-n", namespace])
    if stdout.strip() != "yes":
        print(
            "WARN: RBAC may block pod exec; in-container diagnostics can be limited.",
            file=sys.stderr,
        )

    return True


def get_pod_info(pod_name: str, namespace: str = "default") -> None:
    """Gather comprehensive pod diagnostic information."""

    print(f"\n{'=' * 80}")
    print(f"Pod Diagnostics for: {pod_name} (namespace: {namespace})")
    print(f"Timestamp: {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}")
    print(f"{'=' * 80}\n")

    # Pod Status
    print_section("POD STATUS")
    stdout, stderr, _ = run_kubectl(["get", "pod", pod_name, "-n", namespace, "-o", "wide"])
    print_output(stdout, stderr)

    # Pod Description
    print_section("POD DESCRIPTION")
    stdout, stderr, _ = run_kubectl(["describe", "pod", pod_name, "-n", namespace])
    print_output(stdout, stderr)

    # Pod YAML
    print_section("POD YAML")
    stdout, stderr, _ = run_kubectl(["get", "pod", pod_name, "-n", namespace, "-o", "yaml"])
    print_output(stdout, stderr)

    # Events related to the pod
    print_section("RECENT EVENTS")
    stdout, stderr, _ = run_kubectl(
        [
            "get",
            "events",
            "-n",
            namespace,
            "--field-selector",
            f"involvedObject.name={pod_name}",
            "--sort-by=.lastTimestamp",
        ]
    )
    print_output(stdout, stderr)

    # Container logs (all containers)
    print_section("CONTAINER LOGS")
    stdout, stderr, code = run_kubectl(
        ["get", "pod", pod_name, "-n", namespace, "-o", "jsonpath={.spec.containers[*].name}"]
    )
    if code != 0:
        print_output(stdout, stderr)
        print("INFO: Skipping container logs because container names could not be queried.")
        containers = []
    else:
        containers = stdout.strip().split()

    if not containers:
        print("INFO: No containers detected for this pod.")

    for container in containers:
        print(f"\n### Container: {container} ###")
        stdout, stderr, _ = run_kubectl(
            ["logs", pod_name, "-n", namespace, "-c", container, "--tail=100"],
            timeout=45,
        )
        print_output(stdout, stderr)

        print(f"\n### Previous logs for: {container} ###")
        stdout, stderr, code = run_kubectl(
            ["logs", pod_name, "-n", namespace, "-c", container, "--previous", "--tail=50"],
            timeout=45,
        )
        previous_log_message = f"{stdout}\n{stderr}".lower()
        if code == 0:
            print_output(stdout, stderr)
        elif (
            "previous terminated container" in previous_log_message
            or "is not terminated" in previous_log_message
        ):
            print("INFO: No previous terminated container logs available.")
        else:
            print_output(stdout, stderr)

    # Init container logs — only emitted when the pod has init containers.
    # Init container failures are a primary cause of Init:CrashLoopBackOff and
    # Init:0/N pending states; their logs must be visible in diagnostic output.
    stdout, stderr, code = run_kubectl(
        ["get", "pod", pod_name, "-n", namespace, "-o", "jsonpath={.spec.initContainers[*].name}"]
    )
    if code != 0:
        print_section("INIT CONTAINER LOGS")
        print_output(stdout, stderr)
        print("INFO: Skipping init container logs because init container names could not be queried.")
    else:
        init_containers = stdout.strip().split()
        if init_containers:
            print_section("INIT CONTAINER LOGS")
            for container in init_containers:
                print(f"\n### Init Container: {container} ###")
                stdout, stderr, _ = run_kubectl(
                    ["logs", pod_name, "-n", namespace, "-c", container, "--tail=100"],
                    timeout=45,
                )
                print_output(stdout, stderr)

                print(f"\n### Previous init container logs for: {container} ###")
                stdout, stderr, code = run_kubectl(
                    ["logs", pod_name, "-n", namespace, "-c", container, "--previous", "--tail=50"],
                    timeout=45,
                )
                previous_log_message = f"{stdout}\n{stderr}".lower()
                if code == 0:
                    print_output(stdout, stderr)
                elif (
                    "previous terminated container" in previous_log_message
                    or "is not terminated" in previous_log_message
                ):
                    print("INFO: No previous terminated init container logs available.")
                else:
                    print_output(stdout, stderr)

    # Resource usage
    print_section("RESOURCE USAGE")
    stdout, stderr, code = run_kubectl(
        ["top", "pod", pod_name, "-n", namespace, "--containers"],
        timeout=20,
    )
    if code == 0:
        print_output(stdout, stderr)
    elif "metrics" in stderr.lower():
        print("INFO: Metrics API is unavailable. Skipping 'kubectl top' output.")
        print_output("", stderr)
    else:
        print_output(stdout, stderr)

    # Node information
    print_section("NODE INFORMATION")
    stdout, stderr, code = run_kubectl(
        ["get", "pod", pod_name, "-n", namespace, "-o", "jsonpath={.spec.nodeName}"]
    )
    if code != 0:
        print_output(stdout, stderr)
        return

    node_tokens = stdout.strip().split()
    node_name = node_tokens[0] if node_tokens else ""
    if node_name:
        print(f"Pod is running on node: {node_name}")
        stdout, stderr, _ = run_kubectl(["describe", "node", node_name], timeout=45)
        print_output(stdout, stderr)
    else:
        print("INFO: Node name is not available yet (pod may still be unscheduled).")


def main() -> int:
    parser = argparse.ArgumentParser(description="Gather Kubernetes pod diagnostics")
    parser.add_argument("pod_name", help="Name of the pod to diagnose")
    parser.add_argument("-n", "--namespace", default="default", help="Namespace (default: default)")
    parser.add_argument("-o", "--output", help="Output file path (optional)")

    args = parser.parse_args()

    if not ensure_prerequisites(args.namespace, args.pod_name):
        return 1

    original_stdout = sys.stdout
    output_handle = None
    if args.output:
        output_handle = open(args.output, "w", encoding="utf-8")
        sys.stdout = output_handle

    try:
        get_pod_info(args.pod_name, args.namespace)
    finally:
        if output_handle is not None:
            sys.stdout = original_stdout
            output_handle.close()
            print(f"\nDiagnostics written to: {args.output}", file=sys.stderr)

    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Regression tests for pod_diagnostics.py."""

from __future__ import annotations

import contextlib
import importlib.util
import io
import pathlib
import sys
import unittest
from typing import List, Sequence, Tuple
from unittest.mock import patch


sys.dont_write_bytecode = True

SCRIPT_PATH = pathlib.Path(__file__).resolve().parents[1] / "scripts" / "pod_diagnostics.py"
SPEC = importlib.util.spec_from_file_location("pod_diagnostics", SCRIPT_PATH)
if SPEC is None or SPEC.loader is None:
    raise RuntimeError(f"Unable to load pod_diagnostics module from {SCRIPT_PATH}")
pod_diagnostics = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(pod_diagnostics)

KubectlResponse = Tuple[str, str, int]
ExpectedCall = Tuple[Sequence[str], KubectlResponse]


class PodDiagnosticsInitContainerTests(unittest.TestCase):
    def _run_with_expected_calls(self, expected_calls: List[ExpectedCall]) -> str:
        pending = list(expected_calls)

        def fake_run(args: Sequence[str], timeout: int = 30) -> KubectlResponse:
            del timeout  # Assert command sequence/args; timeout variations are not relevant here.
            self.assertTrue(pending, f"Unexpected kubectl call: {list(args)}")
            expected_args, response = pending.pop(0)
            self.assertEqual(list(args), list(expected_args))
            return response

        stdout_buffer = io.StringIO()
        with patch.object(pod_diagnostics, "run_kubectl", side_effect=fake_run):
            with contextlib.redirect_stdout(stdout_buffer):
                pod_diagnostics.get_pod_info("demo-pod", "demo-ns")

        self.assertFalse(pending, f"Expected kubectl calls were not consumed: {pending}")
        return stdout_buffer.getvalue()

    def test_init_container_previous_logs_message_when_not_terminated(self) -> None:
        output = self._run_with_expected_calls(
            [
                (["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "wide"], ("pod wide", "", 0)),
                (["describe", "pod", "demo-pod", "-n", "demo-ns"], ("pod describe", "", 0)),
                (["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "yaml"], ("pod yaml", "", 0)),
                (
                    [
                        "get",
                        "events",
                        "-n",
                        "demo-ns",
                        "--field-selector",
                        "involvedObject.name=demo-pod",
                        "--sort-by=.lastTimestamp",
                    ],
                    ("event list", "", 0),
                ),
                (
                    ["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "jsonpath={.spec.containers[*].name}"],
                    ("app", "", 0),
                ),
                (
                    ["logs", "demo-pod", "-n", "demo-ns", "-c", "app", "--tail=100"],
                    ("app logs", "", 0),
                ),
                (
                    [
                        "logs",
                        "demo-pod",
                        "-n",
                        "demo-ns",
                        "-c",
                        "app",
                        "--previous",
                        "--tail=50",
                    ],
                    ("", "previous terminated container not found", 1),
                ),
                (
                    [
                        "get",
                        "pod",
                        "demo-pod",
                        "-n",
                        "demo-ns",
                        "-o",
                        "jsonpath={.spec.initContainers[*].name}",
                    ],
                    ("init-setup", "", 0),
                ),
                (
                    ["logs", "demo-pod", "-n", "demo-ns", "-c", "init-setup", "--tail=100"],
                    ("init logs", "", 0),
                ),
                (
                    [
                        "logs",
                        "demo-pod",
                        "-n",
                        "demo-ns",
                        "-c",
                        "init-setup",
                        "--previous",
                        "--tail=50",
                    ],
                    ("", "container is not terminated", 1),
                ),
                (
                    ["top", "pod", "demo-pod", "-n", "demo-ns", "--containers"],
                    ("resource usage", "", 0),
                ),
                (
                    ["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "jsonpath={.spec.nodeName}"],
                    ("node-a", "", 0),
                ),
                (["describe", "node", "node-a"], ("node describe", "", 0)),
            ]
        )

        self.assertIn("## INIT CONTAINER LOGS ##", output)
        self.assertIn("### Init Container: init-setup ###", output)
        self.assertIn("INFO: No previous terminated init container logs available.", output)

    def test_init_container_query_failure_prints_skip_message(self) -> None:
        output = self._run_with_expected_calls(
            [
                (["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "wide"], ("pod wide", "", 0)),
                (["describe", "pod", "demo-pod", "-n", "demo-ns"], ("pod describe", "", 0)),
                (["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "yaml"], ("pod yaml", "", 0)),
                (
                    [
                        "get",
                        "events",
                        "-n",
                        "demo-ns",
                        "--field-selector",
                        "involvedObject.name=demo-pod",
                        "--sort-by=.lastTimestamp",
                    ],
                    ("event list", "", 0),
                ),
                (
                    ["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "jsonpath={.spec.containers[*].name}"],
                    ("app", "", 0),
                ),
                (
                    ["logs", "demo-pod", "-n", "demo-ns", "-c", "app", "--tail=100"],
                    ("app logs", "", 0),
                ),
                (
                    [
                        "logs",
                        "demo-pod",
                        "-n",
                        "demo-ns",
                        "-c",
                        "app",
                        "--previous",
                        "--tail=50",
                    ],
                    ("", "previous terminated container not found", 1),
                ),
                (
                    [
                        "get",
                        "pod",
                        "demo-pod",
                        "-n",
                        "demo-ns",
                        "-o",
                        "jsonpath={.spec.initContainers[*].name}",
                    ],
                    ("", "forbidden", 1),
                ),
                (
                    ["top", "pod", "demo-pod", "-n", "demo-ns", "--containers"],
                    ("resource usage", "", 0),
                ),
                (
                    ["get", "pod", "demo-pod", "-n", "demo-ns", "-o", "jsonpath={.spec.nodeName}"],
                    ("node-a", "", 0),
                ),
                (["describe", "node", "node-a"], ("node describe", "", 0)),
            ]
        )

        self.assertIn("## INIT CONTAINER LOGS ##", output)
        self.assertIn("forbidden", output)
        self.assertIn(
            "INFO: Skipping init container logs because init container names could not be queried.",
            output,
        )
        self.assertNotIn("### Init Container:", output)


if __name__ == "__main__":
    unittest.main()

#!/usr/bin/env bash
#
# Regression tests for k8s-debug shell scripts.
# Validates:
# - network_debug.sh secure/insecure API probing and exit codes
# - cluster_health.sh blocked/check-failure exit codes
#

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
SKILL_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
readonly SKILL_DIR

NETWORK_SCRIPT="$SKILL_DIR/scripts/network_debug.sh"
CLUSTER_SCRIPT="$SKILL_DIR/scripts/cluster_health.sh"
readonly NETWORK_SCRIPT
readonly CLUSTER_SCRIPT

TMP_DIR="$(mktemp -d)"
KUBECTL_LOG="$TMP_DIR/kubectl.log"
readonly TMP_DIR
readonly KUBECTL_LOG

cleanup() {
    rm -rf "$TMP_DIR"
}
trap cleanup EXIT

PASS=0
FAIL=0
OUTPUT=""
EXIT_CODE=0

pass() {
    echo "  PASS: $1"
    PASS=$((PASS + 1))
}

fail() {
    echo "  FAIL: $1"
    FAIL=$((FAIL + 1))
}

reset_stub_env() {
    unset K8S_STUB_CONTEXT_FAIL || true
    unset K8S_STUB_CAN_I_EXEC || true
    unset K8S_STUB_SA_FILES || true
    unset K8S_STUB_EXPECT_SECURE || true
    unset K8S_STUB_EXPECT_INSECURE || true
    unset K8S_STUB_DNS_FAIL || true
    unset K8S_STUB_FAIL_NODE_LIST || true
}

create_kubectl_stub() {
    mkdir -p "$TMP_DIR/bin"

    cat > "$TMP_DIR/bin/kubectl" <<'EOF'
#!/usr/bin/env bash
set -u

LOG_FILE="${KUBECTL_STUB_LOG:-/dev/null}"
printf '%s\n' "$*" >> "$LOG_FILE"

args=("$@")
if [[ "${args[0]:-}" == --request-timeout=* ]]; then
    args=("${args[@]:1}")
fi

if [[ "${#args[@]}" -eq 0 ]]; then
    exit 0
fi

joined=" ${args[*]} "
cmd="${args[0]}"
sub="${args[1]:-}"

if [[ "$cmd" == "config" && "$sub" == "current-context" ]]; then
    if [[ "${K8S_STUB_CONTEXT_FAIL:-0}" == "1" ]]; then
        echo "no context" >&2
        exit 1
    fi
    echo "stub-context"
    exit 0
fi

if [[ "$cmd" == "auth" && "$sub" == "can-i" ]]; then
    if [[ "$joined" == *" create pods/exec "* ]]; then
        echo "${K8S_STUB_CAN_I_EXEC:-yes}"
    else
        echo "yes"
    fi
    exit 0
fi

if [[ "$cmd" == "cluster-info" ]]; then
    echo "Kubernetes control plane is running"
    exit 0
fi

if [[ "$cmd" == "version" ]]; then
    echo "Client Version: v1.30.0"
    exit 0
fi

if [[ "$cmd" == "top" ]]; then
    echo "stub metrics"
    exit 0
fi

if [[ "$cmd" == "logs" ]]; then
    echo "stub logs"
    exit 0
fi

if [[ "$cmd" == "describe" && "$sub" == "pod" ]]; then
    echo "IP: 10.0.0.10"
    echo "Controlled By: ReplicaSet/demo"
    exit 0
fi

if [[ "$cmd" == "get" ]]; then
    resource="${args[1]:-}"

    if [[ "$resource" == "--raw=/readyz?verbose" || "$resource" == "--raw=/healthz?verbose" ]]; then
        echo "ok"
        exit 0
    fi

    if [[ "$resource" == "componentstatuses" ]]; then
        echo "scheduler Healthy"
        exit 0
    fi

    if [[ "$resource" == "namespace" ]]; then
        if [[ "${K8S_STUB_CONTEXT_FAIL:-0}" == "1" ]]; then
            exit 1
        fi
        echo "namespace/${args[2]:-default}"
        exit 0
    fi

    if [[ "$resource" == "pod" ]]; then
        if [[ "$joined" == *"jsonpath={.status.podIP}"* ]]; then
            echo "10.0.0.10"
            exit 0
        fi
        if [[ "$joined" == *"jsonpath={.status.hostIP}"* ]]; then
            echo "192.168.1.10"
            exit 0
        fi
        if [[ "$joined" == *"--show-labels"* ]]; then
            echo "demo-pod app=demo"
            exit 0
        fi
        if [[ "$joined" == *"-o wide"* ]]; then
            echo "demo-pod 1/1 Running 0"
            exit 0
        fi
        echo "pod/${args[2]:-demo-pod}"
        exit 0
    fi

    if [[ "$resource" == "nodes" ]]; then
        if [[ "$joined" == *" -o wide "* && "${K8S_STUB_FAIL_NODE_LIST:-0}" == "1" ]]; then
            echo "node list failed" >&2
            exit 1
        fi
        if [[ "$joined" == *"jsonpath="* ]]; then
            echo -e "node-a\tTrue"
            exit 0
        fi
        echo "node-a Ready"
        exit 0
    fi

    if [[ "$resource" == "events" ]]; then
        echo "Normal Started pod/demo-pod"
        exit 0
    fi

    echo "stub get $resource"
    exit 0
fi

if [[ "$cmd" == "exec" ]]; then
    idx=-1
    for i in "${!args[@]}"; do
        if [[ "${args[$i]}" == "--" ]]; then
            idx=$i
            break
        fi
    done

    if (( idx < 0 )); then
        echo "malformed exec command" >&2
        exit 1
    fi

    exec_args=("${args[@]:idx+1}")
    first="${exec_args[0]:-}"

    if [[ "$first" == "test" && "${exec_args[1]:-}" == "-r" ]]; then
        if [[ "${K8S_STUB_SA_FILES:-present}" == "present" ]]; then
            exit 0
        fi
        exit 1
    fi

    if [[ "$first" == "cat" && "${exec_args[1]:-}" == "/var/run/secrets/kubernetes.io/serviceaccount/token" ]]; then
        if [[ "${K8S_STUB_SA_FILES:-present}" == "present" ]]; then
            echo "stub-token"
            exit 0
        fi
        exit 1
    fi

    if [[ "$first" == "cat" && "${exec_args[1]:-}" == "/etc/resolv.conf" ]]; then
        echo "nameserver 10.96.0.10"
        exit 0
    fi

    if [[ "$first" == "nslookup" ]]; then
        if [[ "${K8S_STUB_DNS_FAIL:-0}" == "1" ]]; then
            exit 1
        fi
        echo "Name: kubernetes.default.svc.cluster.local"
        exit 0
    fi

    if [[ "$first" == "getent" ]]; then
        if [[ "${K8S_STUB_DNS_FAIL:-0}" == "1" ]]; then
            exit 1
        fi
        echo "10.96.0.1 kubernetes.default.svc.cluster.local"
        exit 0
    fi

    if [[ "$first" == "curl" ]]; then
        has_cacert=0
        has_insecure=0
        for arg in "${exec_args[@]}"; do
            [[ "$arg" == "--cacert" ]] && has_cacert=1
            [[ "$arg" == "-k" ]] && has_insecure=1
        done

        if [[ "${K8S_STUB_EXPECT_SECURE:-0}" == "1" ]]; then
            [[ "$has_cacert" -eq 1 && "$has_insecure" -eq 0 ]] || exit 1
        fi
        if [[ "${K8S_STUB_EXPECT_INSECURE:-0}" == "1" ]]; then
            [[ "$has_insecure" -eq 1 ]] || exit 1
        fi
        exit 0
    fi

    if [[ "$first" == "wget" ]]; then
        has_ca=0
        has_no_check=0
        for arg in "${exec_args[@]}"; do
            [[ "$arg" == --ca-certificate=* ]] && has_ca=1
            [[ "$arg" == "--no-check-certificate" ]] && has_no_check=1
        done

        if [[ "${K8S_STUB_EXPECT_SECURE:-0}" == "1" ]]; then
            [[ "$has_ca" -eq 1 ]] || exit 1
        fi
        if [[ "${K8S_STUB_EXPECT_INSECURE:-0}" == "1" ]]; then
            [[ "$has_no_check" -eq 1 ]] || exit 1
        fi
        exit 0
    fi

    exit 0
fi

echo "stub kubectl default response"
exit 0
EOF

    chmod +x "$TMP_DIR/bin/kubectl"
}

run_script() {
    local script="$1"
    shift

    OUTPUT=""
    EXIT_CODE=0
    : > "$KUBECTL_LOG"
    OUTPUT=$(
        PATH="$TMP_DIR/bin:/usr/bin:/bin:$PATH" \
        KUBECTL_STUB_LOG="$KUBECTL_LOG" \
        bash "$script" "$@" 2>&1
    ) || EXIT_CODE=$?
}

assert_exit() {
    local label="$1"
    local expected="$2"
    if [[ "$EXIT_CODE" -eq "$expected" ]]; then
        pass "$label"
    else
        fail "$label (expected exit $expected, got $EXIT_CODE)"
        echo "$OUTPUT" | sed 's/^/    /'
    fi
}

assert_output_contains() {
    local label="$1"
    local pattern="$2"
    if echo "$OUTPUT" | grep -qE -- "$pattern"; then
        pass "$label"
    else
        fail "$label (pattern not found: $pattern)"
        echo "$OUTPUT" | sed 's/^/    /'
    fi
}

assert_log_contains() {
    local label="$1"
    local pattern="$2"
    if grep -qE -- "$pattern" "$KUBECTL_LOG"; then
        pass "$label"
    else
        fail "$label (pattern not found in kubectl log: $pattern)"
        sed 's/^/    /' "$KUBECTL_LOG"
    fi
}

assert_log_not_contains() {
    local label="$1"
    local pattern="$2"
    if grep -qE -- "$pattern" "$KUBECTL_LOG"; then
        fail "$label (unexpected pattern found in kubectl log: $pattern)"
        sed 's/^/    /' "$KUBECTL_LOG"
    else
        pass "$label"
    fi
}

assert_no_bytecode_artifacts() {
    local findings
    findings="$(find "$SKILL_DIR" -type f \( -name '*.pyc' -o -path '*/__pycache__/*' \) -print)"
    if [[ -z "$findings" ]]; then
        pass "no Python bytecode artifacts exist under k8s-debug"
    else
        fail "no Python bytecode artifacts exist under k8s-debug"
        echo "$findings" | sed 's/^/    /'
    fi
}

echo "Running k8s-debug shell regressions..."
create_kubectl_stub

echo ""
echo "[P1] bytecode artifact hygiene"
assert_no_bytecode_artifacts

echo ""
echo "[P0] network_debug secure-by-default API probe"
reset_stub_env
export K8S_STUB_EXPECT_SECURE=1
run_script "$NETWORK_SCRIPT" demo-pod
assert_exit "secure default run returns success" 0
assert_log_contains "secure probe passes --cacert" "--cacert /var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
assert_log_not_contains "secure probe does not use -k" " exec .* -- curl .* -k "

echo ""
echo "[P0] network_debug insecure mode remains explicit"
reset_stub_env
export K8S_STUB_EXPECT_INSECURE=1
export K8S_STUB_SA_FILES=missing
run_script "$NETWORK_SCRIPT" --insecure demo-pod
assert_exit "insecure override run returns success" 0
assert_output_contains "prints insecure override warning" "Insecure TLS mode enabled"
assert_log_contains "insecure probe uses -k" " exec .* -- curl .* -k "

echo ""
echo "[P0/P1] secure mode fails when SA CA/token are missing"
reset_stub_env
export K8S_STUB_SA_FILES=missing
run_script "$NETWORK_SCRIPT" demo-pod
assert_exit "missing SA materials produce partial-failure exit code" 1
assert_output_contains "missing SA files are reported" "service account CA/token files are missing in the pod"

echo ""
echo "[P1] --strict upgrades warnings in network_debug"
reset_stub_env
export K8S_STUB_CAN_I_EXEC=no
run_script "$NETWORK_SCRIPT" --strict demo-pod
assert_exit "strict mode returns failure on warnings" 1
assert_output_contains "RBAC warning is surfaced" "RBAC may block 'kubectl exec'; in-pod checks may fail"

echo ""
echo "[P1] cluster_health blocked precondition returns exit 2"
reset_stub_env
export K8S_STUB_CONTEXT_FAIL=1
run_script "$CLUSTER_SCRIPT"
assert_exit "missing context is blocked" 2
assert_output_contains "blocked error message is clear" "No active Kubernetes context"

echo ""
echo "[P1] cluster_health check failure returns exit 1"
reset_stub_env
export K8S_STUB_FAIL_NODE_LIST=1
run_script "$CLUSTER_SCRIPT"
assert_exit "node list failure maps to exit 1" 1
assert_output_contains "node list failure is reported" "Node list failed; continuing"

echo ""
echo "Test summary: PASS=$PASS FAIL=$FAIL"
if [[ "$FAIL" -ne 0 ]]; then
    exit 1
fi

echo "All k8s-debug shell regressions passed."