Repair node version drift in pipeline

.gitea/workflows/homelab-main.yml

@@ -159,7 +159,21 @@ jobs:
           action_command="${HOMELAB_ACTION_COMMAND:-auto}"
           if [[ "${action_command}" == "auto" ]]; then
             if [[ -s "${kubeconfig_probe}" ]]; then
-              action_command="apps"
+              set +e
+              version_report="$("${deploy_dir}/lab.sh" doctor-versions 2>&1)"
+              version_status=$?
+              set -e
+              printf '%s\n' "${version_report}"
+
+              if ((version_status == 0)); then
+                action_command="apps"
+              elif printf '%s\n' "${version_report}" | grep -Eq 'Kubernetes kubelet minors are mixed|At least one kubelet does not match'; then
+                echo "Kubernetes node version drift detected; switching automatic deploy to rebuild-cluster."
+                action_command="rebuild-cluster"
+              else
+                echo "Version guard failed without Kubernetes minor drift; refusing automatic app deploy." >&2
+                exit "${version_status}"
+              fi
             else
               action_command="rebuild-cluster"
             fi

README.md

@@ -112,9 +112,11 @@ Check node versions from the Debian host:
 
 The check prints the API server, kubelet versions, kubelet minors, and containerd
 runtimes. It exits nonzero when kubelet minors differ from the API server minor
-or when containerd major versions are mixed. For Pimox workers, prefer rebuilding
+or when containerd major versions are mixed. The Gitea Actions auto path runs
-from the corrected golden template and rejoining the worker over ad hoc live
+this check before app deploys; kubelet minor drift switches the job to
-package upgrades.
+`./lab.sh rebuild-cluster` so the Pimox template and worker VMs are replaced
+from the pinned image. For Pimox workers, prefer rebuilding from the corrected
+golden template and rejoining the worker over ad hoc live package upgrades.
 
 ## Deploying
 
@@ -575,8 +577,10 @@ The workflow only blocks automatic deploy for Raspberry Pi Gitea service
 changes: files under `infra/gitea/`, or edits inside the `deploy_gitea`,
 `install_gitea_backup_timer`, `backup_gitea`, or `drill_gitea_restore`
 functions in `lab.sh`. Other changes use `HOMELAB_ACTION_COMMAND=auto` by
-default: Actions runs `./lab.sh apps` when the Debian runner already has a
+default: Actions runs `./lab.sh doctor-versions` when the Debian runner already
-cluster kubeconfig, or `./lab.sh rebuild-cluster` when it does not.
+has a cluster kubeconfig. If node versions are aligned it runs `./lab.sh apps`;
+if kubelet minor drift is detected it runs `./lab.sh rebuild-cluster`; if no
+kubeconfig exists it also runs `./lab.sh rebuild-cluster`.
 
 Set `HOMELAB_ACTION_COMMAND=apps` or `HOMELAB_ACTION_COMMAND=rebuild-cluster`
 on the runner to force one path.

bootstrap/cluster/main.tf

@@ -21,6 +21,7 @@ resource "null_resource" "kubeadm_control_plane" {
     kubeconfig_owner        = var.kubeconfig_owner
     registry_endpoint       = var.registry_endpoint
     registry_config_version = "8"
+    containerd_runtime      = "docker-containerd-io"
     cni_plugins_version     = "2"
     node_dns_servers        = join(" ", var.node_dns_servers)
     persistent_volume_dirs  = join(",", var.persistent_volume_dirs)
@@ -44,6 +45,31 @@ install_missing_packages() {
   fi
 }
 
+install_containerd_runtime() {
+  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
+  arch="$(dpkg --print-architecture)"
+
+  sudo install -d -m 0755 /etc/apt/keyrings
+  sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+  sudo chmod a+r /etc/apt/keyrings/docker.asc
+  sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
+Types: deb
+URIs: https://download.docker.com/linux/debian
+Suites: $${codename}
+Components: stable
+Architectures: $${arch}
+Signed-By: /etc/apt/keyrings/docker.asc
+DOCKER_SOURCES
+
+  if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
+    sudo apt-get remove -y containerd runc || true
+  fi
+
+  sudo apt-get update
+  sudo apt-get install -y --no-install-recommends containerd.io
+  sudo apt-mark hold containerd.io
+}
+
 configure_node_dns() {
   dns_servers="${self.triggers.node_dns_servers}"
   if [ -z "$dns_servers" ]; then
@@ -269,7 +295,8 @@ REGISTRY_EOT
 }
 
 configure_node_dns
-install_missing_packages containernetworking-plugins open-iscsi nfs-common
+install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
+install_containerd_runtime
 install_cni_plugins
 sudo systemctl enable --now iscsid
 sudo systemctl enable kubelet || true
@@ -364,6 +391,7 @@ resource "null_resource" "kubeadm_worker" {
     ssh_key_path                         = each.value.ssh_key_path
     registry_endpoint                    = var.registry_endpoint
     registry_config_version              = "8"
+    containerd_runtime                   = "docker-containerd-io"
     cni_plugins_version                  = "2"
     node_dns_servers                     = join(" ", var.node_dns_servers)
     persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
@@ -404,6 +432,31 @@ install_missing_packages() {
   fi
 }
 
+install_containerd_runtime() {
+  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
+  arch="$(dpkg --print-architecture)"
+
+  sudo install -d -m 0755 /etc/apt/keyrings
+  sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+  sudo chmod a+r /etc/apt/keyrings/docker.asc
+  sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
+Types: deb
+URIs: https://download.docker.com/linux/debian
+Suites: $${codename}
+Components: stable
+Architectures: $${arch}
+Signed-By: /etc/apt/keyrings/docker.asc
+DOCKER_SOURCES
+
+  if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
+    sudo apt-get remove -y containerd runc || true
+  fi
+
+  sudo apt-get update
+  sudo apt-get install -y --no-install-recommends containerd.io
+  sudo apt-mark hold containerd.io
+}
+
 configure_node_dns() {
   dns_servers="${self.triggers.node_dns_servers}"
   if [ -z "$dns_servers" ]; then
@@ -629,7 +682,8 @@ REGISTRY_EOT
 }
 
 configure_node_dns
-install_missing_packages containernetworking-plugins open-iscsi nfs-common
+install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
+install_containerd_runtime
 install_cni_plugins
 sudo systemctl enable --now iscsid
 sudo systemctl enable kubelet || true

bootstrap/provisioning/README.md

@@ -15,6 +15,8 @@ to edit Orange Pi host networking.
 - a preseed file for unattended Debian install
 - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path
 - Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36`
+- `containerd.io` from Docker's Debian repository so worker runtimes use the
+  same package source as the Debian control-plane host
 - kernel boot options for cgroup support through `TF_VAR_kernel_cgroup_boot_options`
 - a template sealing script at `/usr/local/sbin/homelab-prepare-template.sh` inside the installed VM that verifies cgroup boot state before sealing
 

bootstrap/provisioning/main.tf

@@ -24,7 +24,6 @@ locals {
     "apt-transport-https",
     "qemu-guest-agent",
     "cloud-init",
-    "containerd",
     "open-iscsi",
     "nfs-common",
     "iptables",

bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl

@@ -100,6 +100,31 @@ install_kubernetes_tools() {
   apt-mark hold kubelet kubeadm kubectl
 }
 
+install_containerd_runtime() {
+  codename="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")"
+  arch="$(dpkg --print-architecture)"
+
+  install -d -m 0755 /etc/apt/keyrings
+  curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+  chmod a+r /etc/apt/keyrings/docker.asc
+  cat >/etc/apt/sources.list.d/docker.sources <<DOCKER_SOURCES
+Types: deb
+URIs: https://download.docker.com/linux/debian
+Suites: $codename
+Components: stable
+Architectures: $arch
+Signed-By: /etc/apt/keyrings/docker.asc
+DOCKER_SOURCES
+
+  if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
+    apt-get remove -y containerd runc || true
+  fi
+
+  apt-get update
+  apt-get install -y --no-install-recommends containerd.io
+  apt-mark hold containerd.io
+}
+
 configure_containerd() {
   mkdir -p /etc/containerd /etc/containerd/certs.d/${registry_endpoint}
   containerd config default >/etc/containerd/config.toml
@@ -170,5 +195,6 @@ configure_dns
 configure_kubernetes_prereqs
 configure_kernel_boot_options
 install_kubernetes_tools
+install_containerd_runtime
 configure_containerd
 enable_services

lab.sh

@@ -309,7 +309,7 @@ move_prometheus_stack_workers() {
     kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
 }
 
-doctor_versions() {
+doctor_versions_report() {
     local api_version
     local api_minor
     local containerd_major_count
@@ -332,7 +332,7 @@ doctor_versions() {
 
     if [[ -z "${node_rows}" ]]; then
         echo "No Kubernetes nodes found." >&2
-        exit 1
+        return 1
     fi
 
     printf 'API server: %s\n\n' "${api_version}"
@@ -384,7 +384,12 @@ doctor_versions() {
         echo "Node Kubernetes and containerd versions are aligned."
     fi
 
-    exit "${status}"
+    return "${status}"
+}
+
+doctor_versions() {
+    doctor_versions_report
+    exit "$?"
 }
 
 truthy() {
@@ -3053,6 +3058,7 @@ up() {
         prepare_cluster_worker_var_file true
     fi
     run_tofu_stack "bootstrap/cluster"
+    doctor_versions_report
     run_tofu_stack "bootstrap/platform"
     apps
     run_tofu_stack "bootstrap/edge"
@@ -3064,6 +3070,10 @@ rebuild_cluster() {
     require_debian_server "rebuild-cluster"
 
     export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}"
+    export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-true}"
+    export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}"
+    export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}"
+    export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}"
 
     echo "Rebuilding the Kubernetes cluster without touching external Raspberry Pi Gitea..."
 
@@ -3074,6 +3084,7 @@ rebuild_cluster() {
         prepare_cluster_worker_var_file true
     fi
     run_tofu_stack "bootstrap/cluster"
+    doctor_versions_report
     run_tofu_stack "bootstrap/platform"
     apps
     run_tofu_stack "bootstrap/edge"