Kubernetes Ops

Kubernetes Operations

Deploy and manage containerized applications on Kubernetes clusters.

When to Use This Skill

Use this skill when:

• Deploying applications to Kubernetes
• Managing pods, deployments, and services
• Configuring resource limits and scaling
• Troubleshooting Kubernetes workloads
• Setting up networking and ingress

Prerequisites

• kubectl installed and configured
• Access to a Kubernetes cluster
• Basic understanding of containers

Core Resources

Deployment

apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  labels:
    app: myapp
spec:
  replicas: 3
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      containers:
      - name: myapp
        image: myapp:1.0.0
        ports:
        - containerPort: 8080
        resources:
          requests:
            memory: "128Mi"
            cpu: "100m"
          limits:
            memory: "256Mi"
            cpu: "500m"
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          initialDelaySeconds: 10
          periodSeconds: 10
        readinessProbe:
          httpGet:
            path: /ready
            port: 8080
          initialDelaySeconds: 5
          periodSeconds: 5
        env:
        - name: DATABASE_URL
          valueFrom:
            secretKeyRef:
              name: myapp-secrets
              key: database-url

Service

apiVersion: v1
kind: Service
metadata:
  name: myapp
spec:
  selector:
    app: myapp
  ports:
  - port: 80
    targetPort: 8080
  type: ClusterIP
---
# LoadBalancer for external access
apiVersion: v1
kind: Service
metadata:
  name: myapp-external
spec:
  selector:
    app: myapp
  ports:
  - port: 80
    targetPort: 8080
  type: LoadBalancer

Ingress

apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: myapp
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - myapp.example.com
    secretName: myapp-tls
  rules:
  - host: myapp.example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: myapp
            port:
              number: 80

Configuration Management

ConfigMap

apiVersion: v1
kind: ConfigMap
metadata:
  name: myapp-config
data:
  config.yaml: |
    server:
      port: 8080
    logging:
      level: info
  APP_ENV: production

# Using ConfigMap
containers:
- name: myapp
  envFrom:
  - configMapRef:
      name: myapp-config
  volumeMounts:
  - name: config
    mountPath: /etc/config
volumes:
- name: config
  configMap:
    name: myapp-config

Secret

apiVersion: v1
kind: Secret
metadata:
  name: myapp-secrets
type: Opaque
stringData:
  database-url: postgres://user:pass@host:5432/db
  api-key: secret-key-value

# Create secret from command line
kubectl create secret generic myapp-secrets \
  --from-literal=database-url='postgres://...' \
  --from-file=tls.crt=cert.pem

kubectl Commands

Resource Management

# Apply configuration
kubectl apply -f deployment.yaml

# Get resources
kubectl get pods
kubectl get deployments
kubectl get services
kubectl get all -n myapp

# Describe resource
kubectl describe pod myapp-xxx

# Delete resource
kubectl delete -f deployment.yaml
kubectl delete pod myapp-xxx

# Edit resource
kubectl edit deployment myapp

Debugging

# View logs
kubectl logs myapp-xxx
kubectl logs -f myapp-xxx --tail=100
kubectl logs myapp-xxx -c sidecar  # specific container

# Execute command
kubectl exec -it myapp-xxx -- /bin/sh

# Port forward
kubectl port-forward svc/myapp 8080:80
kubectl port-forward pod/myapp-xxx 8080:8080

# View events
kubectl get events --sort-by='.lastTimestamp'

# Debug pod
kubectl debug myapp-xxx -it --image=busybox

Scaling

# Manual scaling
kubectl scale deployment myapp --replicas=5

# Autoscaling
kubectl autoscale deployment myapp \
  --min=2 --max=10 \
  --cpu-percent=80

Horizontal Pod Autoscaler

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: myapp
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 80
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80

Persistent Storage

PersistentVolumeClaim

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: myapp-data
spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: standard
  resources:
    requests:
      storage: 10Gi
---
# Using PVC
containers:
- name: myapp
  volumeMounts:
  - name: data
    mountPath: /data
volumes:
- name: data
  persistentVolumeClaim:
    claimName: myapp-data

StatefulSet

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: postgres
spec:
  serviceName: postgres
  replicas: 3
  selector:
    matchLabels:
      app: postgres
  template:
    metadata:
      labels:
        app: postgres
    spec:
      containers:
      - name: postgres
        image: postgres:15
        ports:
        - containerPort: 5432
        volumeMounts:
        - name: data
          mountPath: /var/lib/postgresql/data
  volumeClaimTemplates:
  - metadata:
      name: data
    spec:
      accessModes: ["ReadWriteOnce"]
      resources:
        requests:
          storage: 10Gi

Jobs and CronJobs

Job

apiVersion: batch/v1
kind: Job
metadata:
  name: migration
spec:
  template:
    spec:
      containers:
      - name: migrate
        image: myapp:1.0.0
        command: ["./migrate.sh"]
      restartPolicy: Never
  backoffLimit: 3

CronJob

apiVersion: batch/v1
kind: CronJob
metadata:
  name: backup
spec:
  schedule: "0 2 * * *"
  jobTemplate:
    spec:
      template:
        spec:
          containers:
          - name: backup
            image: backup-tool:latest
            command: ["./backup.sh"]
          restartPolicy: OnFailure

Network Policies

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: myapp-network-policy
spec:
  podSelector:
    matchLabels:
      app: myapp
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - podSelector:
        matchLabels:
          app: frontend
    ports:
    - protocol: TCP
      port: 8080
  egress:
  - to:
    - podSelector:
        matchLabels:
          app: database
    ports:
    - protocol: TCP
      port: 5432

Resource Quotas

apiVersion: v1
kind: ResourceQuota
metadata:
  name: myapp-quota
  namespace: myapp
spec:
  hard:
    requests.cpu: "10"
    requests.memory: 20Gi
    limits.cpu: "20"
    limits.memory: 40Gi
    pods: "20"

Rolling Updates

spec:
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0

# Update image
kubectl set image deployment/myapp myapp=myapp:2.0.0

# Check rollout status
kubectl rollout status deployment/myapp

# View history
kubectl rollout history deployment/myapp

# Rollback
kubectl rollout undo deployment/myapp
kubectl rollout undo deployment/myapp --to-revision=2

Common Issues

Issue: Pod Stuck in Pending

Problem: Pod won't start Solution: Check resource availability, node selector, PVC binding

kubectl describe pod myapp-xxx
kubectl get events

Issue: CrashLoopBackOff

Problem: Container keeps restarting Solution: Check logs, verify entrypoint, check probes

kubectl logs myapp-xxx --previous
kubectl describe pod myapp-xxx

Issue: Service Not Accessible

Problem: Cannot connect to service Solution: Check selector labels, verify endpoints exist

kubectl get endpoints myapp
kubectl describe svc myapp

Issue: Image Pull Error

Problem: ImagePullBackOff Solution: Check image name, verify registry credentials

kubectl create secret docker-registry regcred \
  --docker-server=registry.example.com \
  --docker-username=user \
  --docker-password=pass

Best Practices

• Always set resource requests and limits
• Implement liveness and readiness probes
• Use namespaces for isolation
• Apply network policies for security
• Use ConfigMaps and Secrets for configuration
• Implement pod disruption budgets for availability
• Use labels consistently for organization
• Enable RBAC for access control

Related Skills

• helm-charts - Package management
• argocd-gitops - GitOps deployments
• kubernetes-hardening - Security

Kubernetes Best Practices

Resource Management

Always Set Resource Requests and Limits

resources:
  requests:
    memory: "256Mi"
    cpu: "250m"
  limits:
    memory: "512Mi"
    cpu: "500m"

Guidelines:

• Requests = guaranteed resources
• Limits = maximum resources
• Set requests based on normal usage
• Set limits based on peak usage
• Memory limit = 2x request is common
• Avoid CPU limits in most cases (causes throttling)

Use Horizontal Pod Autoscaler

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: myapp-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: myapp
  minReplicas: 2
  maxReplicas: 10
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70

Pod Configuration

Use Liveness and Readiness Probes

livenessProbe:
  httpGet:
    path: /healthz
    port: 8080
  initialDelaySeconds: 30
  periodSeconds: 10
  failureThreshold: 3

readinessProbe:
  httpGet:
    path: /ready
    port: 8080
  initialDelaySeconds: 5
  periodSeconds: 5

Configure Pod Disruption Budgets

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: myapp-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: myapp

Use Anti-Affinity for High Availability

affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 100
      podAffinityTerm:
        labelSelector:
          matchLabels:
            app: myapp
        topologyKey: kubernetes.io/hostname

Security

Run as Non-Root

securityContext:
  runAsNonRoot: true
  runAsUser: 1000
  runAsGroup: 1000
  fsGroup: 1000

Read-Only Root Filesystem

securityContext:
  readOnlyRootFilesystem: true
volumeMounts:
- name: tmp
  mountPath: /tmp
volumes:
- name: tmp
  emptyDir: {}

Drop All Capabilities

securityContext:
  capabilities:
    drop:
    - ALL

Networking

Use Network Policies

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: deny-all
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress

Service Mesh for mTLS

• Istio, Linkerd, or Consul Connect
• Automatic encryption between services
• Traffic management capabilities

Configuration Management

Use ConfigMaps for Configuration

apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  LOG_LEVEL: "info"
  DATABASE_HOST: "postgres.default.svc"

Use Secrets for Sensitive Data

apiVersion: v1
kind: Secret
metadata:
  name: app-secrets
type: Opaque
stringData:
  DATABASE_PASSWORD: "secret123"

External Secrets for Production

• Use External Secrets Operator
• Integrate with Vault, AWS Secrets Manager, etc.
• Never commit secrets to git

Observability

Structured Logging

• Output JSON logs
• Include correlation IDs
• Use consistent field names

Metrics

• Expose Prometheus metrics
• Use standard naming conventions
• Include SLI metrics

Distributed Tracing

• Implement OpenTelemetry
• Propagate trace context
• Sample appropriately

Kubernetes Troubleshooting Guide

Common Issues and Solutions

Pod Issues

Pod Stuck in Pending

# Check events
kubectl describe pod <pod-name> -n <namespace>

# Common causes:
# - Insufficient resources
kubectl describe nodes | grep -A 5 "Allocated resources"

# - No matching nodes (taints/tolerations)
kubectl get nodes -o json | jq '.items[].spec.taints'

# - PVC not bound
kubectl get pvc -n <namespace>

Pod in CrashLoopBackOff

# Check logs
kubectl logs <pod-name> -n <namespace> --previous

# Check container exit code
kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.status.containerStatuses[0].lastState.terminated.exitCode}'

# Common exit codes:
# 0 - Success (check livenessProbe)
# 1 - Application error
# 137 - OOMKilled (increase memory)
# 139 - Segmentation fault
# 143 - SIGTERM received

Pod in ImagePullBackOff

# Check image name
kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.spec.containers[0].image}'

# Verify image exists
docker pull <image>

# Check imagePullSecrets
kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.spec.imagePullSecrets}'
kubectl get secret <secret-name> -n <namespace> -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d

Service Issues

Service Not Accessible

# Verify endpoints exist
kubectl get endpoints <service-name> -n <namespace>

# Check selector matches pod labels
kubectl get svc <service-name> -n <namespace> -o jsonpath='{.spec.selector}'
kubectl get pods -n <namespace> --show-labels

# Test from within cluster
kubectl run debug --rm -it --image=busybox -- wget -qO- http://<service>.<namespace>.svc.cluster.local

DNS Resolution Issues

# Test DNS from pod
kubectl run dns-test --rm -it --image=busybox -- nslookup kubernetes.default

# Check CoreDNS pods
kubectl get pods -n kube-system -l k8s-app=kube-dns

# Check CoreDNS logs
kubectl logs -n kube-system -l k8s-app=kube-dns

Node Issues

Node NotReady

# Check node conditions
kubectl describe node <node-name> | grep -A 20 Conditions

# Check kubelet status
systemctl status kubelet

# Check kubelet logs
journalctl -u kubelet -f

# Common causes:
# - Disk pressure
# - Memory pressure
# - Network issues
# - Container runtime issues

Node Disk Pressure

# Check disk usage
kubectl describe node <node-name> | grep -A 3 "Allocated resources"

# Cleanup unused images
docker system prune -af

# Check for large logs
du -sh /var/log/containers/*

Networking Issues

Pod-to-Pod Communication Fails

# Test connectivity
kubectl exec <pod-a> -- ping <pod-b-ip>

# Check network policies
kubectl get networkpolicies -n <namespace>

# Verify CNI plugin
kubectl get pods -n kube-system | grep -E "calico|weave|flannel|cilium"

Storage Issues

PVC Stuck in Pending

# Check PVC events
kubectl describe pvc <pvc-name> -n <namespace>

# Verify StorageClass exists
kubectl get storageclass

# Check provisioner pods
kubectl get pods -n kube-system | grep provisioner

Diagnostic Commands Cheat Sheet

# Cluster overview
kubectl cluster-info
kubectl get componentstatuses

# Resource usage
kubectl top nodes
kubectl top pods -n <namespace>

# Events
kubectl get events -n <namespace> --sort-by='.lastTimestamp'

# Logs
kubectl logs -f <pod> -n <namespace>
kubectl logs -f <pod> -n <namespace> --all-containers

# Exec into pod
kubectl exec -it <pod> -n <namespace> -- /bin/sh

# Port forward
kubectl port-forward <pod> 8080:80 -n <namespace>

# Copy files
kubectl cp <namespace>/<pod>:/path/to/file ./local-file