Pin homelab node version discipline

README.md

@@ -86,7 +86,8 @@ On the Debian host:
 
 - OpenTofu
 - Docker with Buildx
-- kubeadm, kubelet, kubectl, and containerd
+- kubeadm, kubelet, kubectl, and containerd pinned to the target Kubernetes
+  minor, currently `v1.36`
 - SSH access to worker nodes
 - SSH access to the OCI edge host
 - enough persistent storage for `/var/openebs/local` and `/var/lib/docker`
@@ -94,6 +95,27 @@ On the Debian host:
 The default kubeconfig path is `/home/jv/.kube/config`. Override it with
 `KUBECONFIG_PATH` or `TF_VAR_kubeconfig_path` when needed.
 
+## Version Discipline
+
+The lab should converge on one Kubernetes minor across the API server and every
+kubelet. The Pimox golden-image default is `v1.36` so rebuilt arm64 VM workers
+match the Debian control plane and Raspberry Pi worker instead of staying on an
+older template. Avoid treating a three-minor kubelet skew as normal steady
+state: it is only a temporary compatibility window and blocks the next control
+plane upgrade.
+
+Check node versions from the Debian host:
+
+```bash
+./lab.sh doctor-versions
+```
+
+The check prints the API server, kubelet versions, kubelet minors, and containerd
+runtimes. It exits nonzero when kubelet minors differ from the API server minor
+or when containerd major versions are mixed. For Pimox workers, prefer rebuilding
+from the corrected golden template and rejoining the worker over ad hoc live
+package upgrades.
+
 ## Deploying
 
 From the Debian server:

bootstrap/provisioning/README.md

@@ -14,6 +14,7 @@ to edit Orange Pi host networking.
 - Debian 13 arm64 netboot assets under `/opt/homelab-provisioning`
 - a preseed file for unattended Debian install
 - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path
+- Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36`
 - kernel boot options for cgroup support through `TF_VAR_kernel_cgroup_boot_options`
 - a template sealing script at `/usr/local/sbin/homelab-prepare-template.sh` inside the installed VM that verifies cgroup boot state before sealing
 

bootstrap/provisioning/variables.tf

@@ -124,7 +124,7 @@ variable "timezone" {
 
 variable "kubernetes_minor_version" {
   type    = string
-  default = "v1.33"
+  default = "v1.36"
 }
 
 variable "kernel_cgroup_boot_options" {

lab.sh

@@ -309,6 +309,84 @@ move_prometheus_stack_workers() {
     kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
 }
 
+doctor_versions() {
+    local api_version
+    local api_minor
+    local containerd_major_count
+    local kubelet_minor_count
+    local node_rows
+    local status=0
+
+    require_debian_server "doctor-versions"
+    ensure_python3
+
+    api_version="$(
+        kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
+            python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
+    )"
+    api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
+    node_rows="$(
+        kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
+            -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
+    )"
+
+    if [[ -z "${node_rows}" ]]; then
+        echo "No Kubernetes nodes found." >&2
+        exit 1
+    fi
+
+    printf 'API server: %s\n\n' "${api_version}"
+    printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME"
+    printf '%s\n' "${node_rows}" |
+        awk -F '\t' '{
+            version = $2
+            gsub(/^v/, "", version)
+            split(version, parts, ".")
+            printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3
+        }'
+
+    kubelet_minor_count="$(
+        printf '%s\n' "${node_rows}" |
+            awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
+            sort -u |
+            wc -l |
+            tr -d ' '
+    )"
+    containerd_major_count="$(
+        printf '%s\n' "${node_rows}" |
+            awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
+            sort -u |
+            wc -l |
+            tr -d ' '
+    )"
+
+    if [[ "${kubelet_minor_count}" != "1" ]]; then
+        echo
+        echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2
+        status=1
+    fi
+
+    if printf '%s\n' "${node_rows}" |
+        awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then
+        echo
+        echo "At least one kubelet does not match the API server minor (${api_minor})." >&2
+        status=1
+    fi
+
+    if [[ "${containerd_major_count}" -gt 1 ]]; then
+        echo
+        echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2
+        status=1
+    fi
+
+    if [[ "${status}" -eq 0 ]]; then
+        echo
+        echo "Node Kubernetes and containerd versions are aligned."
+    fi
+
+    exit "${status}"
+}
+
 truthy() {
     case "${1,,}" in
         1 | true | yes | on)
@@ -3172,6 +3250,9 @@ case "${1:-}" in
     move-prometheus-stack-workers)
         move_prometheus_stack_workers
         ;;
+    doctor-versions)
+        doctor_versions
+        ;;
     openwrt)
         openwrt
         ;;
@@ -3179,7 +3260,7 @@ case "${1:-}" in
         nuke
         ;;
     *)
-        echo "Usage: $0 {up|rebuild-cluster|apps|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|openwrt|nuke}"
+        echo "Usage: $0 {up|rebuild-cluster|apps|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}"
         exit 1
         ;;
 esac