K8s Cluster Api

# ArgoCD Application for Cluster API Managed Cluster
#
# This Application manages a workload cluster through GitOps.
# The cluster manifests are stored in a Git repository and
# ArgoCD applies them to the management cluster.
#
# Prerequisites:
# - ArgoCD installed on management cluster
# - Git repository with cluster manifests
# - ServiceAccount with CAPI permissions
#
# Usage:
# 1. Store your Cluster/ClusterClass manifests in Git
# 2. Customize this Application with your repo URL
# 3. Apply to management cluster: kubectl apply -f argocd-cluster-app.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: workload-cluster-prod
  namespace: argocd
  # Finalizer ensures resources are deleted when App is deleted
  finalizers:
    - resources-finalizer.argocd.argoproj.io
  labels:
    app.kubernetes.io/component: cluster
    cluster-api.cattle.io/managed: "true"
spec:
  project: default

  source:
    # Repository containing cluster manifests
    repoURL: https://github.com/your-org/clusters.git
    targetRevision: main
    path: clusters/prod

    # Optional: use Kustomize
    # kustomize:
    #   namePrefix: prod-

    # Optional: use Helm
    # helm:
    #   releaseName: prod-cluster
    #   valueFiles:
    #     - values-prod.yaml

  destination:
    # Management cluster where CAPI runs
    server: https://kubernetes.default.svc
    namespace: clusters

  syncPolicy:
    automated:
      # Auto-create namespace if missing
      prune: false # CAUTION: Set to true only if you want cluster deletion on manifest removal
      selfHeal: true
      allowEmpty: false

    syncOptions:
      - CreateNamespace=true
      - PruneLast=true
      - ApplyOutOfSyncOnly=true
      # Respect CAPI ordering
      - RespectIgnoreDifferences=true

    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m

  # Ignore status fields managed by CAPI controllers
  ignoreDifferences:
    - group: cluster.x-k8s.io
      kind: Cluster
      jsonPointers:
        - /status
    - group: cluster.x-k8s.io
      kind: Machine
      jsonPointers:
        - /status
    - group: cluster.x-k8s.io
      kind: MachineDeployment
      jsonPointers:
        - /status
    - group: cluster.x-k8s.io
      kind: MachineSet
      jsonPointers:
        - /status
    - group: controlplane.cluster.x-k8s.io
      kind: KubeadmControlPlane
      jsonPointers:
        - /status
    - group: infrastructure.cluster.x-k8s.io
      kind: "*"
      jsonPointers:
        - /status

  # Health checks for CAPI resources
  # ArgoCD will show cluster as healthy when conditions are met
  # Note: Requires ArgoCD 2.4+ with custom health checks configured
  # See: https://argo-cd.readthedocs.io/en/stable/operator-manual/health/

---
# ApplicationSet for multi-cluster management
# Generates one Application per cluster definition in Git repository
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: workload-clusters
  namespace: argocd
spec:
  generators:
    # Generate from directories in clusters/ path
    - git:
        repoURL: https://github.com/your-org/clusters.git
        revision: main
        directories:
          - path: clusters/*
          - path: clusters/archive/*
            exclude: true

  template:
    metadata:
      name: "cluster-{{path.basename}}"
      namespace: argocd
      labels:
        cluster-name: "{{path.basename}}"
    spec:
      project: default
      source:
        repoURL: https://github.com/your-org/clusters.git
        targetRevision: main
        path: "{{path}}"
      destination:
        server: https://kubernetes.default.svc
        namespace: clusters
      syncPolicy:
        automated:
          prune: false
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

---
# AppProject for cluster management with restricted permissions
apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: clusters
  namespace: argocd
spec:
  description: Cluster API managed clusters
  sourceRepos:
    - "https://github.com/your-org/clusters.git"
  destinations:
    - namespace: clusters
      server: https://kubernetes.default.svc
    - namespace: "cluster-*"
      server: https://kubernetes.default.svc

  # Only allow CAPI resources
  clusterResourceWhitelist:
    - group: cluster.x-k8s.io
      kind: "*"
    - group: controlplane.cluster.x-k8s.io
      kind: "*"
    - group: infrastructure.cluster.x-k8s.io
      kind: "*"
    - group: bootstrap.cluster.x-k8s.io
      kind: "*"
    - group: addons.cluster.x-k8s.io
      kind: "*"
    - group: ""
      kind: Namespace
    - group: ""
      kind: Secret

  namespaceResourceWhitelist:
    - group: "*"
      kind: "*"

  # Deny changes to management cluster components
  clusterResourceBlacklist:
    - group: ""
      kind: Node
    - group: rbac.authorization.k8s.io
      kind: ClusterRole
    - group: rbac.authorization.k8s.io
      kind: ClusterRoleBinding

# AWS Provider Credentials Template
# Create Secret for AWS credentials used by CAPA (Cluster API Provider AWS)
# IMPORTANT: Replace placeholder values before applying
---
apiVersion: v1
kind: Secret
metadata:
  name: capa-manager-bootstrap-credentials
  namespace: capa-system # Default namespace for AWS provider
type: Opaque
stringData:
  # AWS programmatic access credentials
  # Create an IAM user with appropriate permissions (see below)
  credentials: |
    [default]
    aws_access_key_id = <YOUR_ACCESS_KEY_ID>
    aws_secret_access_key = <YOUR_SECRET_ACCESS_KEY>
    # Optional: for session tokens (temporary credentials)
    # aws_session_token = <YOUR_SESSION_TOKEN>
    region = us-east-1

---
# Alternative: Using eksctl to create the required IAM resources
# This creates the policies and roles needed by CAPA
#
# eksctl utils associate-iam-oidc-provider --cluster <management-cluster> --approve
#
# eksctl create iamserviceaccount \
#   --name capa-controller-manager \
#   --namespace capa-system \
#   --cluster <management-cluster> \
#   --attach-policy-arn arn:aws:iam::aws:policy/AmazonEC2FullAccess \
#   --attach-policy-arn arn:aws:iam::aws:policy/IAMFullAccess \
#   --attach-policy-arn arn:aws:iam::aws:policy/AmazonS3ReadOnlyAccess \
#   --approve

---
# IAM Policy for CAPA controller (minimum permissions)
# Create this policy and attach to the IAM user/role
#
# {
#   "Version": "2012-10-17",
#   "Statement": [
#     {
#       "Effect": "Allow",
#       "Action": [
#         "ec2:*",
#         "elasticloadbalancing:*",
#         "autoscaling:*",
#         "iam:CreateServiceLinkedRole",
#         "iam:PassRole",
#         "iam:GetInstanceProfile",
#         "iam:CreateInstanceProfile",
#         "iam:DeleteInstanceProfile",
#         "iam:AddRoleToInstanceProfile",
#         "iam:RemoveRoleFromInstanceProfile",
#         "ssm:GetParameter"
#       ],
#       "Resource": "*"
#     },
#     {
#       "Effect": "Allow",
#       "Action": [
#         "iam:CreateRole",
#         "iam:DeleteRole",
#         "iam:AttachRolePolicy",
#         "iam:DetachRolePolicy",
#         "iam:GetRole",
#         "iam:ListAttachedRolePolicies"
#       ],
#       "Resource": "arn:aws:iam::*:role/control-plane.cluster-api-provider-aws.sigs.k8s.io"
#     },
#     {
#       "Effect": "Allow",
#       "Action": [
#         "iam:CreateRole",
#         "iam:DeleteRole",
#         "iam:AttachRolePolicy",
#         "iam:DetachRolePolicy",
#         "iam:GetRole",
#         "iam:ListAttachedRolePolicies"
#       ],
#       "Resource": "arn:aws:iam::*:role/nodes.cluster-api-provider-aws.sigs.k8s.io"
#     }
#   ]
# }

---
# Environment variables for clusterctl (place in ~/.bashrc or export before init)
#
# export AWS_REGION=us-east-1
# export AWS_ACCESS_KEY_ID=<YOUR_ACCESS_KEY_ID>
# export AWS_SECRET_ACCESS_KEY=<YOUR_SECRET_ACCESS_KEY>
#
# # For EKS-based control plane
# export AWS_SESSION_TOKEN=<YOUR_SESSION_TOKEN>  # if using temporary credentials
#
# # SSH key for node access
# export AWS_SSH_KEY_NAME=<YOUR_SSH_KEY_NAME>
#
# # Control plane settings
# export AWS_CONTROL_PLANE_MACHINE_TYPE=m5.large
# export CONTROL_PLANE_MACHINE_COUNT=3
#
# # Worker settings
# export AWS_NODE_MACHINE_TYPE=m5.xlarge
# export WORKER_MACHINE_COUNT=3

---
# Initialize AWS provider:
#
# export AWS_B64ENCODED_CREDENTIALS=$(clusterctl generate provider aws -f)
# clusterctl init --infrastructure aws

---
# ClusterAWS identity types available:
#
# 1. AWSClusterStaticIdentity - static AWS credentials (this template)
# 2. AWSClusterRoleIdentity - assume IAM role
# 3. AWSClusterControllerIdentity - use controller's credentials
#
# Example AWSClusterRoleIdentity:
#
# apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
# kind: AWSClusterRoleIdentity
# metadata:
#   name: cross-account-role
# spec:
#   allowedNamespaces:
#     list:
#       - default
#   roleARN: arn:aws:iam::ACCOUNT_ID:role/CAPIRole
#   sourceIdentityRef:
#     kind: AWSClusterControllerIdentity
#     name: default

# Azure Provider Credentials Template
# Create Secret for Azure credentials used by CAPZ (Cluster API Provider Azure)
# IMPORTANT: Replace placeholder values before applying
---
apiVersion: v1
kind: Secret
metadata:
  name: capz-manager-bootstrap-credentials
  namespace: capz-system
type: Opaque
stringData:
  # Service Principal credentials (JSON format)
  # Create using: az ad sp create-for-rbac --sdk-auth
  clientSecret: |
    {
      "clientId": "<YOUR_CLIENT_ID>",
      "clientSecret": "<YOUR_CLIENT_SECRET>",
      "subscriptionId": "<YOUR_SUBSCRIPTION_ID>",
      "tenantId": "<YOUR_TENANT_ID>",
      "activeDirectoryEndpointUrl": "https://login.microsoftonline.com",
      "resourceManagerEndpointUrl": "https://management.azure.com/",
      "activeDirectoryGraphResourceId": "https://graph.windows.net/",
      "sqlManagementEndpointUrl": "https://management.core.windows.net:8443/",
      "galleryEndpointUrl": "https://gallery.azure.com/",
      "managementEndpointUrl": "https://management.core.windows.net/"
    }

---
# Steps to create Azure Service Principal:
#
# 1. Login to Azure CLI:
#    az login
#
# 2. Set subscription:
#    az account set --subscription "<SUBSCRIPTION_ID>"
#
# 3. Create Service Principal with Contributor role:
#    az ad sp create-for-rbac \
#      --name "cluster-api-provider-azure" \
#      --role Contributor \
#      --scopes "/subscriptions/<SUBSCRIPTION_ID>" \
#      --sdk-auth > azure-credentials.json
#
# 4. The output JSON contains all required credentials
#
# Optional: For more restrictive permissions, use custom role:
#
# az role definition create --role-definition '{
#   "Name": "CAPZ Minimum Permissions",
#   "Description": "Minimum permissions for Cluster API Provider Azure",
#   "Actions": [
#     "Microsoft.Compute/*",
#     "Microsoft.Network/*",
#     "Microsoft.Resources/*",
#     "Microsoft.Authorization/*/read",
#     "Microsoft.ManagedIdentity/userAssignedIdentities/*"
#   ],
#   "NotActions": [],
#   "AssignableScopes": ["/subscriptions/<SUBSCRIPTION_ID>"]
# }'

---
# Environment variables for clusterctl (place in ~/.bashrc or export before init)
#
# # Azure subscription
# export AZURE_SUBSCRIPTION_ID="<YOUR_SUBSCRIPTION_ID>"
# export AZURE_TENANT_ID="<YOUR_TENANT_ID>"
# export AZURE_CLIENT_ID="<YOUR_CLIENT_ID>"
# export AZURE_CLIENT_SECRET="<YOUR_CLIENT_SECRET>"
#
# # Cluster settings
# export AZURE_LOCATION="eastus"
# export AZURE_RESOURCE_GROUP="my-cluster-rg"
#
# # Control plane settings
# export AZURE_CONTROL_PLANE_MACHINE_TYPE="Standard_D2s_v3"
# export CONTROL_PLANE_MACHINE_COUNT=3
#
# # Worker settings
# export AZURE_NODE_MACHINE_TYPE="Standard_D4s_v3"
# export WORKER_MACHINE_COUNT=3
#
# # SSH key
# export AZURE_SSH_PUBLIC_KEY_B64=$(cat ~/.ssh/id_rsa.pub | base64 | tr -d '\n')

---
# Initialize Azure provider:
#
# # Generate base64 encoded credentials
# export AZURE_SUBSCRIPTION_ID_B64=$(echo -n "$AZURE_SUBSCRIPTION_ID" | base64)
# export AZURE_TENANT_ID_B64=$(echo -n "$AZURE_TENANT_ID" | base64)
# export AZURE_CLIENT_ID_B64=$(echo -n "$AZURE_CLIENT_ID" | base64)
# export AZURE_CLIENT_SECRET_B64=$(echo -n "$AZURE_CLIENT_SECRET" | base64)
#
# clusterctl init --infrastructure azure

---
# Azure identity types available:
#
# 1. AzureClusterIdentity with Service Principal
# 2. AzureClusterIdentity with User-Assigned Managed Identity
# 3. AzureClusterIdentity with System-Assigned Managed Identity (AKS)
#
# Example Service Principal identity:
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: AzureClusterIdentity
metadata:
  name: cluster-identity
  namespace: default
spec:
  type: ServicePrincipal
  allowedNamespaces:
    list:
      - default
  tenantID: "<YOUR_TENANT_ID>"
  clientID: "<YOUR_CLIENT_ID>"
  clientSecret:
    name: capz-manager-bootstrap-credentials
    namespace: capz-system

---
# Example User-Assigned Managed Identity:
#
# apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
# kind: AzureClusterIdentity
# metadata:
#   name: cluster-identity-msi
#   namespace: default
# spec:
#   type: UserAssignedMSI
#   allowedNamespaces:
#     list:
#       - default
#   tenantID: "<YOUR_TENANT_ID>"
#   clientID: "<MANAGED_IDENTITY_CLIENT_ID>"
#   resourceID: "/subscriptions/<SUB_ID>/resourceGroups/<RG>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<IDENTITY_NAME>"

# Cluster using ClusterClass Topology
# Requires ClusterClass to be deployed first (see clusterclass-example.yaml)
# This is the modern recommended approach for cluster management
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: ${CLUSTER_NAME:=topology-cluster}
  namespace: ${NAMESPACE:=default}
  labels:
    environment: ${ENVIRONMENT:=development}
spec:
  # Use topology instead of direct references
  topology:
    # Reference the ClusterClass
    class: ${CLUSTER_CLASS:=quick-start}
    version: ${KUBERNETES_VERSION:=v1.29.0}

    # Control plane configuration
    controlPlane:
      replicas: ${CONTROL_PLANE_REPLICAS:=1}
      # Override specific control plane settings
      # metadata:
      #   labels:
      #     custom-label: value

    # Worker configuration
    workers:
      machineDeployments:
        - class: default-worker
          name: md-0
          replicas: ${WORKER_REPLICAS:=2}
          # Optional: failure domain (availability zone)
          # failureDomain: us-east-1a

          # Optional: override worker metadata
          # metadata:
          #   labels:
          #     workload-type: general

          # Optional: override machine deployment strategy
          # strategy:
          #   type: RollingUpdate
          #   rollingUpdate:
          #     maxSurge: 1
          #     maxUnavailable: 0

        # Additional worker pool example (uncomment to use)
        # - class: default-worker
        #   name: md-gpu
        #   replicas: 1
        #   metadata:
        #     labels:
        #       workload-type: gpu

    # Variables - customize ClusterClass behavior
    variables:
      # Pod Security Standards
      - name: podSecurityStandard
        value:
          enabled: true
          enforce: baseline
          audit: restricted
          warn: restricted

      # CNI configuration (if ClusterClass supports)
      # - name: cni
      #   value: calico

      # Custom variables defined in ClusterClass
      # - name: sshKey
      #   value: my-ssh-key

      # - name: controlPlaneMachineType
      #   value: m5.xlarge

---
# Example: Using variables for environment-specific settings
# Uncomment and modify based on your ClusterClass variables
#
# For staging environment:
# variables:
#   - name: environment
#     value: staging
#   - name: controlPlaneMachineType
#     value: m5.large
#   - name: workerMachineType
#     value: m5.large
#
# For production environment:
# variables:
#   - name: environment
#     value: production
#   - name: controlPlaneMachineType
#     value: m5.xlarge
#   - name: workerMachineType
#     value: m5.2xlarge
#   - name: enableMonitoring
#     value: true

# Minimal Cluster Template
# Suitable for development/testing with Docker provider
# Usage: kubectl apply -f cluster-minimal.yaml
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: ${CLUSTER_NAME:=dev-cluster}
  namespace: ${NAMESPACE:=default}
spec:
  clusterNetwork:
    pods:
      cidrBlocks:
        - 192.168.0.0/16
    services:
      cidrBlocks:
        - 10.96.0.0/12
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: KubeadmControlPlane
    name: ${CLUSTER_NAME}-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    kind: DockerCluster
    name: ${CLUSTER_NAME}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerCluster
metadata:
  name: ${CLUSTER_NAME}
  namespace: ${NAMESPACE}
spec: {}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: ${NAMESPACE}
spec:
  replicas: 1
  version: ${KUBERNETES_VERSION:=v1.28.0}
  machineTemplate:
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: DockerMachineTemplate
      name: ${CLUSTER_NAME}-control-plane
  kubeadmConfigSpec:
    clusterConfiguration:
      apiServer:
        certSANs:
          - localhost
          - 127.0.0.1
      controllerManager:
        extraArgs:
          enable-hostpath-provisioner: "true"
    initConfiguration:
      nodeRegistration:
        criSocket: unix:///var/run/containerd/containerd.sock
        kubeletExtraArgs:
          eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
    joinConfiguration:
      nodeRegistration:
        criSocket: unix:///var/run/containerd/containerd.sock
        kubeletExtraArgs:
          eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: ${NAMESPACE}
spec:
  template:
    spec: {}
---
# Worker MachineDeployment
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  clusterName: ${CLUSTER_NAME}
  replicas: ${WORKER_COUNT:=1}
  selector:
    matchLabels: null
  template:
    spec:
      clusterName: ${CLUSTER_NAME}
      version: ${KUBERNETES_VERSION}
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: KubeadmConfigTemplate
          name: ${CLUSTER_NAME}-md-0
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
        kind: DockerMachineTemplate
        name: ${CLUSTER_NAME}-md-0
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  template:
    spec: {}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      joinConfiguration:
        nodeRegistration:
          kubeletExtraArgs:
            eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

# Production Cluster Template
# HA configuration with 3 control plane nodes and configurable workers
# Suitable for production workloads
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: Cluster
metadata:
  name: ${CLUSTER_NAME:=prod-cluster}
  namespace: ${NAMESPACE:=default}
  labels:
    environment: production
spec:
  clusterNetwork:
    pods:
      cidrBlocks:
        - 10.244.0.0/16
    services:
      cidrBlocks:
        - 10.96.0.0/12
    serviceDomain: cluster.local
  controlPlaneRef:
    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
    kind: KubeadmControlPlane
    name: ${CLUSTER_NAME}-control-plane
  infrastructureRef:
    apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
    kind: AWSCluster # Replace with your provider
    name: ${CLUSTER_NAME}
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: AWSCluster
metadata:
  name: ${CLUSTER_NAME}
  namespace: ${NAMESPACE}
spec:
  region: ${AWS_REGION:=us-east-1}
  sshKeyName: ${SSH_KEY_NAME}
  network:
    vpc:
      cidrBlock: 10.0.0.0/16
    subnets:
      - availabilityZone: ${AWS_REGION}a
        cidrBlock: 10.0.0.0/24
        isPublic: false
      - availabilityZone: ${AWS_REGION}b
        cidrBlock: 10.0.1.0/24
        isPublic: false
      - availabilityZone: ${AWS_REGION}c
        cidrBlock: 10.0.2.0/24
        isPublic: false
      - availabilityZone: ${AWS_REGION}a
        cidrBlock: 10.0.10.0/24
        isPublic: true
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlane
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: ${NAMESPACE}
spec:
  replicas: 3 # HA: odd number for etcd quorum
  version: ${KUBERNETES_VERSION:=v1.29.0}
  machineTemplate:
    infrastructureRef:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: AWSMachineTemplate
      name: ${CLUSTER_NAME}-control-plane
  kubeadmConfigSpec:
    clusterConfiguration:
      apiServer:
        extraArgs:
          audit-log-maxage: "30"
          audit-log-maxbackup: "10"
          audit-log-maxsize: "100"
          audit-log-path: /var/log/kubernetes/audit.log
          enable-admission-plugins: NodeRestriction,PodSecurity
        extraVolumes:
          - name: audit-log
            hostPath: /var/log/kubernetes
            mountPath: /var/log/kubernetes
            readOnly: false
            pathType: DirectoryOrCreate
      controllerManager:
        extraArgs:
          bind-address: "0.0.0.0"
          terminated-pod-gc-threshold: "100"
      scheduler:
        extraArgs:
          bind-address: "0.0.0.0"
      etcd:
        local:
          extraArgs:
            listen-metrics-urls: http://0.0.0.0:2381
    initConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          cloud-provider: external
    joinConfiguration:
      nodeRegistration:
        kubeletExtraArgs:
          cloud-provider: external
    files:
      - content: |
          apiVersion: audit.k8s.io/v1
          kind: Policy
          rules:
            - level: Metadata
              resources:
                - group: ""
                  resources: ["secrets", "configmaps"]
            - level: Request
              verbs: ["create", "update", "patch", "delete"]
            - level: None
              users: ["system:kube-proxy"]
              verbs: ["watch"]
              resources:
                - group: ""
                  resources: ["endpoints", "services", "services/status"]
        owner: root:root
        path: /etc/kubernetes/audit-policy.yaml
        permissions: "0644"
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: AWSMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-control-plane
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      instanceType: ${CONTROL_PLANE_INSTANCE_TYPE:=m5.large}
      iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io
      rootVolume:
        size: 100
        type: gp3
        throughput: 125
        iops: 3000
      sshKeyName: ${SSH_KEY_NAME}
---
# Worker MachineDeployment
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineDeployment
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  clusterName: ${CLUSTER_NAME}
  replicas: ${WORKER_COUNT:=3}
  selector:
    matchLabels: null
  template:
    spec:
      clusterName: ${CLUSTER_NAME}
      version: ${KUBERNETES_VERSION}
      bootstrap:
        configRef:
          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
          kind: KubeadmConfigTemplate
          name: ${CLUSTER_NAME}-md-0
      infrastructureRef:
        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
        kind: AWSMachineTemplate
        name: ${CLUSTER_NAME}-md-0
      failureDomain: ${AWS_REGION}a
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: AWSMachineTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      instanceType: ${WORKER_INSTANCE_TYPE:=m5.xlarge}
      iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io
      rootVolume:
        size: 200
        type: gp3
      sshKeyName: ${SSH_KEY_NAME}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
  name: ${CLUSTER_NAME}-md-0
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      joinConfiguration:
        nodeRegistration:
          kubeletExtraArgs:
            cloud-provider: external
---
# MachineHealthCheck for automatic remediation
apiVersion: cluster.x-k8s.io/v1beta1
kind: MachineHealthCheck
metadata:
  name: ${CLUSTER_NAME}-mhc
  namespace: ${NAMESPACE}
spec:
  clusterName: ${CLUSTER_NAME}
  maxUnhealthy: 40%
  nodeStartupTimeout: 10m
  selector:
    matchLabels:
      cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
  unhealthyConditions:
    - type: Ready
      status: "False"
      timeout: 5m
    - type: Ready
      status: Unknown
      timeout: 5m

# ClusterClass Example with Variables
# Defines reusable cluster templates with customizable variables
# Deploy this before creating Clusters that reference it
---
apiVersion: cluster.x-k8s.io/v1beta1
kind: ClusterClass
metadata:
  name: quick-start
  namespace: ${NAMESPACE:=default}
spec:
  # Infrastructure template for the Cluster
  infrastructure:
    ref:
      apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
      kind: DockerClusterTemplate
      name: quick-start-cluster

  # Control plane template
  controlPlane:
    ref:
      apiVersion: controlplane.cluster.x-k8s.io/v1beta1
      kind: KubeadmControlPlaneTemplate
      name: quick-start-control-plane
    machineInfrastructure:
      ref:
        apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
        kind: DockerMachineTemplate
        name: quick-start-control-plane

  # Worker templates
  workers:
    machineDeployments:
      - class: default-worker
        template:
          bootstrap:
            ref:
              apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
              kind: KubeadmConfigTemplate
              name: quick-start-worker
          infrastructure:
            ref:
              apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
              kind: DockerMachineTemplate
              name: quick-start-worker

  # Variable definitions - allow cluster customization
  variables:
    # Pod Security Standards variable
    - name: podSecurityStandard
      required: false
      schema:
        openAPIV3Schema:
          type: object
          default:
            enabled: true
            enforce: baseline
            audit: restricted
            warn: restricted
          properties:
            enabled:
              type: boolean
              default: true
            enforce:
              type: string
              enum: [privileged, baseline, restricted]
              default: baseline
            audit:
              type: string
              enum: [privileged, baseline, restricted]
              default: restricted
            warn:
              type: string
              enum: [privileged, baseline, restricted]
              default: restricted

    # Image repository override
    - name: imageRepository
      required: false
      schema:
        openAPIV3Schema:
          type: string
          default: registry.k8s.io
          description: Container image repository for Kubernetes components

    # SSH key for node access
    - name: sshKey
      required: false
      schema:
        openAPIV3Schema:
          type: string
          default: ""
          description: SSH public key for node access

  # Patches - apply variable values to templates
  patches:
    # Apply Pod Security Standards to KubeadmControlPlane
    - name: podSecurityStandard
      enabledIf: "{{ if .podSecurityStandard.enabled }}true{{ end }}"
      definitions:
        - selector:
            apiVersion: controlplane.cluster.x-k8s.io/v1beta1
            kind: KubeadmControlPlaneTemplate
            matchResources:
              controlPlane: true
          jsonPatches:
            - op: add
              path: /spec/template/spec/kubeadmConfigSpec/clusterConfiguration/apiServer/extraArgs/admission-control-config-file
              value: /etc/kubernetes/admission/admission-control-config.yaml
            - op: add
              path: /spec/template/spec/kubeadmConfigSpec/files/-
              valueFrom:
                template: |
                  content: |
                    apiVersion: apiserver.config.k8s.io/v1
                    kind: AdmissionConfiguration
                    plugins:
                    - name: PodSecurity
                      configuration:
                        apiVersion: pod-security.admission.config.k8s.io/v1
                        kind: PodSecurityConfiguration
                        defaults:
                          enforce: "{{ .podSecurityStandard.enforce }}"
                          enforce-version: "latest"
                          audit: "{{ .podSecurityStandard.audit }}"
                          audit-version: "latest"
                          warn: "{{ .podSecurityStandard.warn }}"
                          warn-version: "latest"
                        exemptions:
                          usernames: []
                          runtimeClasses: []
                          namespaces: [kube-system]
                  owner: root:root
                  path: /etc/kubernetes/admission/admission-control-config.yaml
                  permissions: "0644"
---
# Supporting templates
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerClusterTemplate
metadata:
  name: quick-start-cluster
  namespace: ${NAMESPACE}
spec:
  template:
    spec: {}
---
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
kind: KubeadmControlPlaneTemplate
metadata:
  name: quick-start-control-plane
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      kubeadmConfigSpec:
        clusterConfiguration:
          apiServer:
            certSANs:
              - localhost
              - 127.0.0.1
          controllerManager:
            extraArgs:
              enable-hostpath-provisioner: "true"
        initConfiguration:
          nodeRegistration:
            criSocket: unix:///var/run/containerd/containerd.sock
            kubeletExtraArgs:
              eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
        joinConfiguration:
          nodeRegistration:
            criSocket: unix:///var/run/containerd/containerd.sock
            kubeletExtraArgs:
              eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
  name: quick-start-control-plane
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      extraMounts:
        - containerPath: /var/run/docker.sock
          hostPath: /var/run/docker.sock
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: DockerMachineTemplate
metadata:
  name: quick-start-worker
  namespace: ${NAMESPACE}
spec:
  template:
    spec: {}
---
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
kind: KubeadmConfigTemplate
metadata:
  name: quick-start-worker
  namespace: ${NAMESPACE}
spec:
  template:
    spec:
      joinConfiguration:
        nodeRegistration:
          kubeletExtraArgs:
            eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%

# Docker Provider Quick Start Configuration
# Use with: clusterctl init --infrastructure docker
# Location: ~/.cluster-api/clusterctl.yaml
---
# clusterctl configuration for Docker provider (CAPD)
# This is for development/testing only, not production

# Provider repositories
providers:
  - name: docker
    type: InfrastructureProvider
    url: https://github.com/kubernetes-sigs/cluster-api/releases/latest/infrastructure-components-development.yaml

# Override default image repository if needed
# images:
#   all:
#     repository: my-registry.example.com/capi

# Feature gates
CLUSTER_TOPOLOGY: "true"
EXP_MACHINE_POOL: "true"


# Docker provider specific variables (for templates)
# These are set when generating clusters with clusterctl generate cluster

# Example usage:
# export CLUSTER_NAME=my-cluster
# export KUBERNETES_VERSION=v1.29.0
# export CONTROL_PLANE_MACHINE_COUNT=1
# export WORKER_MACHINE_COUNT=3
#
# clusterctl generate cluster ${CLUSTER_NAME} \
#   --flavor development \
#   --kubernetes-version ${KUBERNETES_VERSION} \
#   --control-plane-machine-count ${CONTROL_PLANE_MACHINE_COUNT} \
#   --worker-machine-count ${WORKER_MACHINE_COUNT} \
#   > cluster.yaml
#
# kubectl apply -f cluster.yaml

---
# Minimal environment setup for Docker provider:
#
# 1. Install Docker Desktop or Docker Engine
# 2. Install kind: brew install kind (or see kind.sigs.k8s.io)
# 3. Install clusterctl: brew install clusterctl
# 4. Create management cluster:
#
#    kind create cluster --name capi-management
#    clusterctl init --infrastructure docker
#
# 5. Generate and apply workload cluster:
#
#    clusterctl generate cluster dev-cluster \
#      --infrastructure docker \
#      --kubernetes-version v1.29.0 \
#      --control-plane-machine-count 1 \
#      --worker-machine-count 1 | kubectl apply -f -
#
# 6. Get kubeconfig:
#
#    clusterctl get kubeconfig dev-cluster > dev-cluster.kubeconfig
#    kubectl --kubeconfig=dev-cluster.kubeconfig get nodes

# etcd Backup CronJob for CAPI Management Cluster
#
# Automated scheduled backups of management cluster etcd to S3/GCS/Azure.
# Deploy to management cluster for disaster recovery protection.
#
# Prerequisites:
# - etcd accessible from within cluster
# - S3/GCS/Azure credentials configured
# - PVC or emptyDir for temporary storage
#
# Customization:
# 1. Update BACKUP_BUCKET with your storage location
# 2. Configure credentials secret
# 3. Adjust schedule (default: daily at 2 AM)

---
# Namespace for backup operations
apiVersion: v1
kind: Namespace
metadata:
  name: capi-backup
  labels:
    app.kubernetes.io/name: capi-backup

---
# Secret with backup destination credentials
# Create with: kubectl create secret generic backup-credentials \
#   --from-literal=AWS_ACCESS_KEY_ID=xxx \
#   --from-literal=AWS_SECRET_ACCESS_KEY=xxx \
#   -n capi-backup
apiVersion: v1
kind: Secret
metadata:
  name: backup-credentials
  namespace: capi-backup
type: Opaque
stringData:
  # AWS S3
  AWS_ACCESS_KEY_ID: "<your-access-key>"
  AWS_SECRET_ACCESS_KEY: "<your-secret-key>"
  # Or GCS
  # GOOGLE_APPLICATION_CREDENTIALS: |
  #   <service-account-json>
  # Or Azure
  # AZURE_STORAGE_ACCOUNT: "<account>"
  # AZURE_STORAGE_KEY: "<key>"

---
# ConfigMap with backup configuration
apiVersion: v1
kind: ConfigMap
metadata:
  name: backup-config
  namespace: capi-backup
data:
  BACKUP_BUCKET: "s3://your-bucket/capi-backups"
  # Or: gs://your-bucket/capi-backups
  # Or: https://account.blob.core.windows.net/capi-backups
  RETENTION_DAYS: "30"
  ETCD_ENDPOINTS: "https://127.0.0.1:2379"

---
# ServiceAccount for backup jobs
apiVersion: v1
kind: ServiceAccount
metadata:
  name: etcd-backup
  namespace: capi-backup

---
# CronJob: etcd snapshot backup
apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: capi-backup
  labels:
    app.kubernetes.io/name: etcd-backup
spec:
  # Run daily at 2:00 AM UTC
  schedule: "0 2 * * *"
  # Keep last 3 job runs
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  concurrencyPolicy: Forbid

  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 1800 # 30 min timeout
      template:
        metadata:
          labels:
            app.kubernetes.io/name: etcd-backup
        spec:
          serviceAccountName: etcd-backup
          restartPolicy: OnFailure

          # Must run on control plane node with etcd access
          nodeSelector:
            node-role.kubernetes.io/control-plane: ""
          tolerations:
            - key: node-role.kubernetes.io/control-plane
              effect: NoSchedule
            - key: node-role.kubernetes.io/master
              effect: NoSchedule

          containers:
            - name: backup
              image: bitnami/etcd:3.5
              command:
                - /bin/bash
                - -c
                - |
                  set -euo pipefail

                  TIMESTAMP=$(date +%Y%m%d-%H%M%S)
                  BACKUP_FILE="/backup/etcd-snapshot-${TIMESTAMP}.db"

                  echo "Starting etcd backup at ${TIMESTAMP}"

                  # Create snapshot
                  etcdctl snapshot save "${BACKUP_FILE}" \
                    --endpoints="${ETCD_ENDPOINTS}" \
                    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
                    --cert=/etc/kubernetes/pki/etcd/server.crt \
                    --key=/etc/kubernetes/pki/etcd/server.key

                  # Verify snapshot
                  etcdctl snapshot status "${BACKUP_FILE}" --write-out=table

                  # Get snapshot size
                  SNAPSHOT_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1)
                  echo "Snapshot size: ${SNAPSHOT_SIZE}"

                  # Upload to cloud storage
                  echo "Uploading to ${BACKUP_BUCKET}..."

                  # AWS S3
                  if [[ "${BACKUP_BUCKET}" == s3://* ]]; then
                    aws s3 cp "${BACKUP_FILE}" "${BACKUP_BUCKET}/"
                    aws s3 cp "${BACKUP_FILE}" "${BACKUP_BUCKET}/latest.db"
                  fi

                  # GCS
                  if [[ "${BACKUP_BUCKET}" == gs://* ]]; then
                    gsutil cp "${BACKUP_FILE}" "${BACKUP_BUCKET}/"
                    gsutil cp "${BACKUP_FILE}" "${BACKUP_BUCKET}/latest.db"
                  fi

                  # Cleanup old local backup
                  rm -f "${BACKUP_FILE}"

                  echo "Backup completed successfully"

              env:
                - name: ETCDCTL_API
                  value: "3"
                - name: ETCD_ENDPOINTS
                  valueFrom:
                    configMapKeyRef:
                      name: backup-config
                      key: ETCD_ENDPOINTS
                - name: BACKUP_BUCKET
                  valueFrom:
                    configMapKeyRef:
                      name: backup-config
                      key: BACKUP_BUCKET
              envFrom:
                - secretRef:
                    name: backup-credentials

              volumeMounts:
                - name: etcd-certs
                  mountPath: /etc/kubernetes/pki/etcd
                  readOnly: true
                - name: backup-volume
                  mountPath: /backup

              resources:
                requests:
                  cpu: 100m
                  memory: 256Mi
                limits:
                  cpu: 500m
                  memory: 512Mi

          volumes:
            - name: etcd-certs
              hostPath:
                path: /etc/kubernetes/pki/etcd
                type: Directory
            - name: backup-volume
              emptyDir:
                sizeLimit: 10Gi

---
# CronJob: CAPI resource backup
apiVersion: batch/v1
kind: CronJob
metadata:
  name: capi-resources-backup
  namespace: capi-backup
  labels:
    app.kubernetes.io/name: capi-resources-backup
spec:
  # Run daily at 2:30 AM UTC (after etcd backup)
  schedule: "30 2 * * *"
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  concurrencyPolicy: Forbid

  jobTemplate:
    spec:
      backoffLimit: 2
      activeDeadlineSeconds: 900 # 15 min timeout
      template:
        metadata:
          labels:
            app.kubernetes.io/name: capi-resources-backup
        spec:
          serviceAccountName: etcd-backup
          restartPolicy: OnFailure

          containers:
            - name: backup
              image: bitnami/kubectl:1.28
              command:
                - /bin/bash
                - -c
                - |
                  set -euo pipefail

                  TIMESTAMP=$(date +%Y%m%d-%H%M%S)
                  BACKUP_DIR="/backup/capi-${TIMESTAMP}"
                  mkdir -p "${BACKUP_DIR}"

                  echo "Starting CAPI resources backup at ${TIMESTAMP}"

                  # Export all CAPI resources
                  CAPI_KINDS=(
                    "clusters.cluster.x-k8s.io"
                    "machines.cluster.x-k8s.io"
                    "machinesets.cluster.x-k8s.io"
                    "machinedeployments.cluster.x-k8s.io"
                    "machinepools.cluster.x-k8s.io"
                    "machinehealthchecks.cluster.x-k8s.io"
                    "clusterclasses.cluster.x-k8s.io"
                    "kubeadmcontrolplanes.controlplane.cluster.x-k8s.io"
                    "kubeadmconfigs.bootstrap.cluster.x-k8s.io"
                  )

                  for KIND in "${CAPI_KINDS[@]}"; do
                    echo "Exporting ${KIND}..."
                    kubectl get "${KIND}" -A -o yaml > "${BACKUP_DIR}/${KIND//\//_}.yaml" 2>/dev/null || true
                  done

                  # Export cluster secrets (kubeconfigs)
                  echo "Exporting cluster secrets..."
                  kubectl get secrets -A -l cluster.x-k8s.io/cluster-name -o yaml > "${BACKUP_DIR}/cluster-secrets.yaml"

                  # Create tarball
                  TARBALL="/backup/capi-backup-${TIMESTAMP}.tar.gz"
                  tar -czf "${TARBALL}" -C /backup "capi-${TIMESTAMP}"

                  # Upload
                  echo "Uploading to ${BACKUP_BUCKET}..."
                  if [[ "${BACKUP_BUCKET}" == s3://* ]]; then
                    aws s3 cp "${TARBALL}" "${BACKUP_BUCKET}/"
                  fi
                  if [[ "${BACKUP_BUCKET}" == gs://* ]]; then
                    gsutil cp "${TARBALL}" "${BACKUP_BUCKET}/"
                  fi

                  # Cleanup
                  rm -rf "${BACKUP_DIR}" "${TARBALL}"

                  echo "CAPI backup completed successfully"

              env:
                - name: BACKUP_BUCKET
                  valueFrom:
                    configMapKeyRef:
                      name: backup-config
                      key: BACKUP_BUCKET
              envFrom:
                - secretRef:
                    name: backup-credentials

              volumeMounts:
                - name: backup-volume
                  mountPath: /backup

              resources:
                requests:
                  cpu: 100m
                  memory: 256Mi
                limits:
                  cpu: 500m
                  memory: 512Mi

          volumes:
            - name: backup-volume
              emptyDir:
                sizeLimit: 5Gi

---
# RBAC for CAPI resource backup
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: capi-backup-reader
rules:
  - apiGroups:
      - cluster.x-k8s.io
      - controlplane.cluster.x-k8s.io
      - bootstrap.cluster.x-k8s.io
      - infrastructure.cluster.x-k8s.io
      - addons.cluster.x-k8s.io
    resources:
      - "*"
    verbs:
      - get
      - list
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - get
      - list

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: capi-backup-reader
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: capi-backup-reader
subjects:
  - kind: ServiceAccount
    name: etcd-backup
    namespace: capi-backup

# Flux Kustomization for Cluster API Managed Clusters
#
# This configuration tells Flux to apply cluster manifests from a Git repository
# to the management cluster where CAPI controllers are running.
#
# Prerequisites:
# - Flux v2 installed on management cluster
# - GitRepository resource configured
# - Optional: SOPS/age for secret encryption
#
# Usage:
# 1. Configure GitRepository pointing to your clusters repo
# 2. Customize this Kustomization for your setup
# 3. Apply: kubectl apply -f flux-kustomization.yaml

---
# GitRepository source definition
apiVersion: source.toolkit.fluxcd.io/v1
kind: GitRepository
metadata:
  name: clusters
  namespace: flux-system
spec:
  interval: 1m
  url: https://github.com/your-org/clusters.git
  ref:
    branch: main
  # For private repos:
  # secretRef:
  #   name: clusters-repo-auth

---
# Main Kustomization for all clusters
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: clusters
  namespace: flux-system
spec:
  interval: 10m
  timeout: 5m
  retryInterval: 2m

  sourceRef:
    kind: GitRepository
    name: clusters

  path: ./clusters
  prune: false # CAUTION: Enable pruning only if you want cluster deletion
  wait: true

  # Health checks for CAPI resources
  healthChecks:
    - apiVersion: cluster.x-k8s.io/v1beta1
      kind: Cluster
      namespace: clusters
      name: "*"

  # Decrypt secrets with SOPS
  # decryption:
  #   provider: sops
  #   secretRef:
  #     name: sops-age

  # Substitute variables from ConfigMaps/Secrets
  # postBuild:
  #   substitute:
  #     CLUSTER_NAME: prod-cluster
  #   substituteFrom:
  #     - kind: ConfigMap
  #       name: cluster-vars
  #     - kind: Secret
  #       name: cluster-secrets

---
# Kustomization for production clusters
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: clusters-production
  namespace: flux-system
spec:
  interval: 10m
  dependsOn:
    - name: clusters # Wait for base to be ready

  sourceRef:
    kind: GitRepository
    name: clusters

  path: ./clusters/production
  prune: false
  wait: true
  timeout: 15m # Production clusters may take longer

  # Require approval for production changes
  # suspend: true  # Uncomment to require manual resume

  healthChecks:
    - apiVersion: cluster.x-k8s.io/v1beta1
      kind: Cluster
      namespace: clusters
      name: prod-cluster

---
# Kustomization for staging clusters
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
  name: clusters-staging
  namespace: flux-system
spec:
  interval: 5m
  sourceRef:
    kind: GitRepository
    name: clusters

  path: ./clusters/staging
  prune: true # OK to prune staging clusters
  wait: true
  timeout: 10m

---
# Alert for cluster provisioning failures
apiVersion: notification.toolkit.fluxcd.io/v1beta3
kind: Alert
metadata:
  name: cluster-alerts
  namespace: flux-system
spec:
  providerRef:
    name: slack # or: teams, discord, webhook
  eventSeverity: error
  eventSources:
    - kind: Kustomization
      name: clusters
    - kind: Kustomization
      name: clusters-production
    - kind: Kustomization
      name: clusters-staging
  exclusionList:
    - ".*upgrade.*pending.*"

---
# Provider for Slack notifications
apiVersion: notification.toolkit.fluxcd.io/v1beta3
kind: Provider
metadata:
  name: slack
  namespace: flux-system
spec:
  type: slack
  channel: cluster-alerts
  secretRef:
    name: slack-webhook-url

---
# ImagePolicy for automatic Kubernetes version updates (optional)
# Watches for new Kubernetes versions and updates cluster manifests
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImageRepository
metadata:
  name: kubernetes
  namespace: flux-system
spec:
  image: registry.k8s.io/kube-apiserver
  interval: 1h

---
apiVersion: image.toolkit.fluxcd.io/v1beta2
kind: ImagePolicy
metadata:
  name: kubernetes-stable
  namespace: flux-system
spec:
  imageRepositoryRef:
    name: kubernetes
  filterTags:
    pattern: "^v1\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)$"
    extract: "$minor.$patch"
  policy:
    semver:
      range: ">=1.28.0 <1.32.0"

---
# ReceiverConfiguration for webhook-based updates
apiVersion: notification.toolkit.fluxcd.io/v1
kind: Receiver
metadata:
  name: clusters-webhook
  namespace: flux-system
spec:
  type: github
  events:
    - push
  secretRef:
    name: webhook-token
  resources:
    - kind: GitRepository
      name: clusters

# GitOps RBAC for Cluster API
#
# Service accounts and roles for ArgoCD/Flux to manage CAPI resources.
# Apply to management cluster before configuring GitOps.
#
# Includes:
# - ServiceAccount for GitOps controller
# - ClusterRole with CAPI permissions
# - Namespace-scoped Role for cluster secrets
#
# Usage:
# kubectl apply -f gitops-rbac.yaml

---
# Namespace for cluster resources
apiVersion: v1
kind: Namespace
metadata:
  name: clusters
  labels:
    app.kubernetes.io/managed-by: gitops
    cluster-api.sigs.k8s.io/namespace: "true"

---
# ServiceAccount for GitOps controller
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitops-cluster-manager
  namespace: clusters
  labels:
    app.kubernetes.io/name: gitops-cluster-manager
    app.kubernetes.io/component: rbac

---
# ClusterRole for managing CAPI resources
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: gitops-capi-manager
  labels:
    app.kubernetes.io/name: gitops-capi-manager
rules:
  # Core Cluster API resources
  - apiGroups:
      - cluster.x-k8s.io
    resources:
      - clusters
      - machines
      - machinesets
      - machinedeployments
      - machinepools
      - machinehealthchecks
      - clusterclasses
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Cluster API status (read-only for status)
  - apiGroups:
      - cluster.x-k8s.io
    resources:
      - clusters/status
      - machines/status
      - machinesets/status
      - machinedeployments/status
      - machinepools/status
    verbs:
      - get
      - list
      - watch

  # Control Plane providers
  - apiGroups:
      - controlplane.cluster.x-k8s.io
    resources:
      - kubeadmcontrolplanes
      - kubeadmcontrolplanetemplates
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Bootstrap providers
  - apiGroups:
      - bootstrap.cluster.x-k8s.io
    resources:
      - kubeadmconfigs
      - kubeadmconfigtemplates
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Infrastructure providers (generic wildcard)
  - apiGroups:
      - infrastructure.cluster.x-k8s.io
    resources:
      - "*"
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Addons (ClusterResourceSet)
  - apiGroups:
      - addons.cluster.x-k8s.io
    resources:
      - clusterresourcesets
      - clusterresourcesetbindings
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Runtime extensions
  - apiGroups:
      - runtime.cluster.x-k8s.io
    resources:
      - extensionconfigs
    verbs:
      - get
      - list
      - watch

  # IPAM
  - apiGroups:
      - ipam.cluster.x-k8s.io
    resources:
      - ipaddresses
      - ipaddressclaims
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # Namespaces for cluster isolation
  - apiGroups:
      - ""
    resources:
      - namespaces
    verbs:
      - create
      - get
      - list
      - watch

  # Events for debugging
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - get
      - list
      - watch

---
# ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: gitops-capi-manager
  labels:
    app.kubernetes.io/name: gitops-capi-manager
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: gitops-capi-manager
subjects:
  - kind: ServiceAccount
    name: gitops-cluster-manager
    namespace: clusters
  # ArgoCD application controller
  - kind: ServiceAccount
    name: argocd-application-controller
    namespace: argocd
  # Flux kustomize controller
  - kind: ServiceAccount
    name: kustomize-controller
    namespace: flux-system

---
# Role for managing secrets in clusters namespace
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: gitops-cluster-secrets
  namespace: clusters
  labels:
    app.kubernetes.io/name: gitops-cluster-secrets
rules:
  # Kubeconfig secrets
  - apiGroups:
      - ""
    resources:
      - secrets
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

  # ConfigMaps for cluster addons
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
      - delete
      - get
      - list
      - patch
      - update
      - watch

---
# RoleBinding for secrets
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: gitops-cluster-secrets
  namespace: clusters
  labels:
    app.kubernetes.io/name: gitops-cluster-secrets
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: gitops-cluster-secrets
subjects:
  - kind: ServiceAccount
    name: gitops-cluster-manager
    namespace: clusters
  - kind: ServiceAccount
    name: argocd-application-controller
    namespace: argocd
  - kind: ServiceAccount
    name: kustomize-controller
    namespace: flux-system

---
# Read-only ClusterRole for monitoring/dashboards
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: gitops-capi-viewer
  labels:
    app.kubernetes.io/name: gitops-capi-viewer
rules:
  - apiGroups:
      - cluster.x-k8s.io
      - controlplane.cluster.x-k8s.io
      - bootstrap.cluster.x-k8s.io
      - infrastructure.cluster.x-k8s.io
      - addons.cluster.x-k8s.io
    resources:
      - "*"
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
      - namespaces
    verbs:
      - get
      - list
      - watch

# Prometheus Alerting Rules for Cluster API
#
# Alerts for monitoring CAPI management and workload cluster health.
# Deploy to Prometheus instance monitoring the management cluster.
#
# Prerequisites:
# - Prometheus Operator or raw Prometheus
# - kube-state-metrics with CAPI CRD support
# - Node exporter on management cluster
#
# Usage:
# kubectl apply -f prometheus-alerts.yaml

---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cluster-api-alerts
  namespace: monitoring
  labels:
    app.kubernetes.io/name: cluster-api
    prometheus: main
    role: alert-rules
spec:
  groups:
    # =========================================================================
    # Cluster Provisioning Alerts
    # =========================================================================
    - name: capi.cluster.provisioning
      rules:
        - alert: ClusterProvisioningStuck
          expr: |
            (
              kube_customresource_cluster_status_phase{phase!="Provisioned"}
              * on (cluster_name, namespace) group_left()
              (time() - kube_customresource_cluster_created > 1800)
            ) > 0
          for: 5m
          labels:
            severity: warning
            category: provisioning
          annotations:
            summary: "Cluster {{ $labels.cluster_name }} provisioning stuck"
            description: "Cluster {{ $labels.cluster_name }} in namespace {{ $labels.namespace }} has been in {{ $labels.phase }} state for over 30 minutes."
            runbook_url: "https://cluster-api.sigs.k8s.io/troubleshooting"

        - alert: ClusterProvisioningFailed
          expr: |
            kube_customresource_cluster_status_phase{phase="Failed"} == 1
          for: 2m
          labels:
            severity: critical
            category: provisioning
          annotations:
            summary: "Cluster {{ $labels.cluster_name }} provisioning failed"
            description: "Cluster {{ $labels.cluster_name }} in namespace {{ $labels.namespace }} has entered Failed state."

        - alert: ClusterDeleting
          expr: |
            (
              kube_customresource_cluster_status_phase{phase="Deleting"}
              * on (cluster_name, namespace) group_left()
              (time() - kube_customresource_cluster_deletion_timestamp > 900)
            ) > 0
          for: 5m
          labels:
            severity: warning
            category: lifecycle
          annotations:
            summary: "Cluster {{ $labels.cluster_name }} deletion taking too long"
            description: "Cluster deletion has been in progress for over 15 minutes. Check for stuck finalizers."

    # =========================================================================
    # Machine Health Alerts
    # =========================================================================
    - name: capi.machine.health
      rules:
        - alert: MachineNotRunning
          expr: |
            kube_customresource_machine_status_phase{phase!~"Running|Deleting"} == 1
          for: 10m
          labels:
            severity: warning
            category: machine
          annotations:
            summary: "Machine {{ $labels.machine }} not running"
            description: "Machine {{ $labels.machine }} in cluster {{ $labels.cluster_name }} is in {{ $labels.phase }} state."

        - alert: MachineProvisioningFailed
          expr: |
            kube_customresource_machine_status_phase{phase="Failed"} == 1
          for: 2m
          labels:
            severity: critical
            category: machine
          annotations:
            summary: "Machine {{ $labels.machine }} failed"
            description: "Machine {{ $labels.machine }} in cluster {{ $labels.cluster_name }} has failed to provision."

        - alert: MachineHealthCheckTriggered
          expr: |
            increase(capi_machinehealthcheck_remediation_total[5m]) > 0
          for: 1m
          labels:
            severity: warning
            category: remediation
          annotations:
            summary: "MachineHealthCheck remediation triggered"
            description: "MachineHealthCheck {{ $labels.machinehealthcheck }} has triggered remediation in cluster {{ $labels.cluster_name }}."

        - alert: MachineUnhealthyCondition
          expr: |
            kube_customresource_machine_status_condition{condition="Ready", status="False"} == 1
          for: 15m
          labels:
            severity: warning
            category: machine
          annotations:
            summary: "Machine {{ $labels.machine }} unhealthy"
            description: "Machine {{ $labels.machine }} Ready condition is False for over 15 minutes."

    # =========================================================================
    # Control Plane Alerts
    # =========================================================================
    - name: capi.controlplane
      rules:
        - alert: ControlPlaneNotReady
          expr: |
            kube_customresource_kubeadmcontrolplane_status_condition{condition="Ready", status="False"} == 1
          for: 10m
          labels:
            severity: critical
            category: controlplane
          annotations:
            summary: "Control plane {{ $labels.kubeadmcontrolplane }} not ready"
            description: "KubeadmControlPlane {{ $labels.kubeadmcontrolplane }} is not ready for over 10 minutes."

        - alert: ControlPlaneReplicasMismatch
          expr: |
            kube_customresource_kubeadmcontrolplane_spec_replicas
            !=
            kube_customresource_kubeadmcontrolplane_status_replicas
          for: 30m
          labels:
            severity: warning
            category: controlplane
          annotations:
            summary: "Control plane replica count mismatch"
            description: "KubeadmControlPlane {{ $labels.kubeadmcontrolplane }} has {{ $value }} ready replicas, expected {{ $labels.spec_replicas }}."

        - alert: ControlPlaneUpgradeStuck
          expr: |
            (
              kube_customresource_kubeadmcontrolplane_status_condition{condition="RollingUpdate", status="True"} == 1
            ) * on (kubeadmcontrolplane, namespace) group_left()
            (time() - kube_customresource_kubeadmcontrolplane_status_updated > 3600) > 0
          for: 10m
          labels:
            severity: warning
            category: upgrade
          annotations:
            summary: "Control plane upgrade stuck"
            description: "KubeadmControlPlane {{ $labels.kubeadmcontrolplane }} rolling update in progress for over 1 hour."

    # =========================================================================
    # Certificate Alerts
    # =========================================================================
    - name: capi.certificates
      rules:
        - alert: ClusterCertificateExpiringSoon
          expr: |
            (
              capi_cluster_certificate_expiry_seconds > 0
              and
              capi_cluster_certificate_expiry_seconds < 2592000
            )
          for: 1h
          labels:
            severity: warning
            category: certificate
          annotations:
            summary: "Cluster certificate expiring within 30 days"
            description: "Certificate {{ $labels.certificate }} for cluster {{ $labels.cluster_name }} expires in {{ $value | humanizeDuration }}."

        - alert: ClusterCertificateCritical
          expr: |
            (
              capi_cluster_certificate_expiry_seconds > 0
              and
              capi_cluster_certificate_expiry_seconds < 604800
            )
          for: 10m
          labels:
            severity: critical
            category: certificate
          annotations:
            summary: "Cluster certificate expiring within 7 days"
            description: "Certificate {{ $labels.certificate }} for cluster {{ $labels.cluster_name }} expires in {{ $value | humanizeDuration }}. Rotate immediately!"

    # =========================================================================
    # CAPI Controller Alerts
    # =========================================================================
    - name: capi.controllers
      rules:
        - alert: CAPIControllerDown
          expr: |
            up{job=~".*capi.*"} == 0
          for: 5m
          labels:
            severity: critical
            category: controller
          annotations:
            summary: "CAPI controller {{ $labels.job }} is down"
            description: "The CAPI controller {{ $labels.job }} has been down for over 5 minutes."

        - alert: CAPIControllerHighRestarts
          expr: |
            increase(kube_pod_container_status_restarts_total{namespace=~"capi.*", container=~"manager"}[1h]) > 3
          for: 10m
          labels:
            severity: warning
            category: controller
          annotations:
            summary: "CAPI controller restarting frequently"
            description: "Controller {{ $labels.pod }} has restarted {{ $value }} times in the last hour."

        - alert: CAPIReconcileErrors
          expr: |
            increase(controller_runtime_reconcile_errors_total{controller=~".*cluster.*|.*machine.*"}[5m]) > 10
          for: 10m
          labels:
            severity: warning
            category: controller
          annotations:
            summary: "High reconcile errors in {{ $labels.controller }}"
            description: "Controller {{ $labels.controller }} has {{ $value }} reconcile errors in the last 5 minutes."

        - alert: CAPIReconcileDurationHigh
          expr: |
            histogram_quantile(0.99, sum(rate(controller_runtime_reconcile_time_seconds_bucket{controller=~".*cluster.*"}[5m])) by (le, controller)) > 60
          for: 15m
          labels:
            severity: warning
            category: performance
          annotations:
            summary: "CAPI reconciliation slow"
            description: "99th percentile reconcile time for {{ $labels.controller }} is {{ $value }}s."

    # =========================================================================
    # Infrastructure Provider Alerts
    # =========================================================================
    - name: capi.infrastructure
      rules:
        - alert: InfrastructureProviderDown
          expr: |
            up{job=~"cap[a-z]-.*"} == 0
          for: 5m
          labels:
            severity: critical
            category: provider
          annotations:
            summary: "Infrastructure provider {{ $labels.job }} is down"
            description: "Infrastructure provider {{ $labels.job }} has been unavailable for 5 minutes."

        - alert: InfrastructureResourceStuck
          expr: |
            (
              kube_customresource_status_condition{group=~"infrastructure.cluster.x-k8s.io", condition="Ready", status="False"} == 1
            )
          for: 30m
          labels:
            severity: warning
            category: provider
          annotations:
            summary: "Infrastructure resource not ready"
            description: "{{ $labels.kind }} {{ $labels.name }} is not ready for over 30 minutes."

    # =========================================================================
    # Workload Cluster Connectivity
    # =========================================================================
    - name: capi.connectivity
      rules:
        - alert: WorkloadClusterUnreachable
          expr: |
            probe_success{job="workload-clusters"} == 0
          for: 5m
          labels:
            severity: critical
            category: connectivity
          annotations:
            summary: "Workload cluster {{ $labels.cluster }} unreachable"
            description: "Cannot reach API server of workload cluster {{ $labels.cluster }}."

        - alert: WorkloadClusterAPILatencyHigh
          expr: |
            probe_http_duration_seconds{job="workload-clusters", phase="transfer"} > 2
          for: 10m
          labels:
            severity: warning
            category: performance
          annotations:
            summary: "Workload cluster API latency high"
            description: "API server latency for cluster {{ $labels.cluster }} is {{ $value }}s."

---
# Recording rules for CAPI metrics
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cluster-api-recording
  namespace: monitoring
  labels:
    app.kubernetes.io/name: cluster-api
    prometheus: main
    role: recording-rules
spec:
  groups:
    - name: capi.recording
      rules:
        - record: capi:cluster:count
          expr: count(kube_customresource_cluster_info) or vector(0)

        - record: capi:cluster:by_phase
          expr: count by (phase) (kube_customresource_cluster_status_phase)

        - record: capi:machine:count
          expr: count(kube_customresource_machine_info) or vector(0)

        - record: capi:machine:by_phase
          expr: count by (phase) (kube_customresource_machine_status_phase)

        - record: capi:machine:unhealthy_ratio
          expr: |
            (
              count(kube_customresource_machine_status_condition{condition="Ready", status="False"})
              /
              count(kube_customresource_machine_info)
            ) or vector(0)

        - record: capi:controlplane:ready_ratio
          expr: |
            (
              sum(kube_customresource_kubeadmcontrolplane_status_ready_replicas)
              /
              sum(kube_customresource_kubeadmcontrolplane_spec_replicas)
            ) or vector(0)