From f1a9b1fcdb7e80d6003d5432d9118ad4c8590944 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Thu, 4 Jun 2026 21:00:16 -0600 Subject: [PATCH] Pin homelab node version discipline --- README.md | 24 ++++++++- bootstrap/provisioning/README.md | 1 + bootstrap/provisioning/variables.tf | 2 +- lab.sh | 83 ++++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 894e6b7..ff8cc84 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,8 @@ On the Debian host: - OpenTofu - Docker with Buildx -- kubeadm, kubelet, kubectl, and containerd +- kubeadm, kubelet, kubectl, and containerd pinned to the target Kubernetes + minor, currently `v1.36` - SSH access to worker nodes - SSH access to the OCI edge host - enough persistent storage for `/var/openebs/local` and `/var/lib/docker` @@ -94,6 +95,27 @@ On the Debian host: The default kubeconfig path is `/home/jv/.kube/config`. Override it with `KUBECONFIG_PATH` or `TF_VAR_kubeconfig_path` when needed. +## Version Discipline + +The lab should converge on one Kubernetes minor across the API server and every +kubelet. The Pimox golden-image default is `v1.36` so rebuilt arm64 VM workers +match the Debian control plane and Raspberry Pi worker instead of staying on an +older template. Avoid treating a three-minor kubelet skew as normal steady +state: it is only a temporary compatibility window and blocks the next control +plane upgrade. + +Check node versions from the Debian host: + +```bash +./lab.sh doctor-versions +``` + +The check prints the API server, kubelet versions, kubelet minors, and containerd +runtimes. It exits nonzero when kubelet minors differ from the API server minor +or when containerd major versions are mixed. For Pimox workers, prefer rebuilding +from the corrected golden template and rejoining the worker over ad hoc live +package upgrades. + ## Deploying From the Debian server: diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 6cef40c..ad52317 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -14,6 +14,7 @@ to edit Orange Pi host networking. - Debian 13 arm64 netboot assets under `/opt/homelab-provisioning` - a preseed file for unattended Debian install - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path +- Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36` - kernel boot options for cgroup support through `TF_VAR_kernel_cgroup_boot_options` - a template sealing script at `/usr/local/sbin/homelab-prepare-template.sh` inside the installed VM that verifies cgroup boot state before sealing diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index 1e26e43..32094a7 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -124,7 +124,7 @@ variable "timezone" { variable "kubernetes_minor_version" { type = string - default = "v1.33" + default = "v1.36" } variable "kernel_cgroup_boot_options" { diff --git a/lab.sh b/lab.sh index 88b0831..8c7740b 100755 --- a/lab.sh +++ b/lab.sh @@ -309,6 +309,84 @@ move_prometheus_stack_workers() { kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide } +doctor_versions() { + local api_version + local api_minor + local containerd_major_count + local kubelet_minor_count + local node_rows + local status=0 + + require_debian_server "doctor-versions" + ensure_python3 + + api_version="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' + )" + api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" + node_rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' + )" + + if [[ -z "${node_rows}" ]]; then + echo "No Kubernetes nodes found." >&2 + exit 1 + fi + + printf 'API server: %s\n\n' "${api_version}" + printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME" + printf '%s\n' "${node_rows}" | + awk -F '\t' '{ + version = $2 + gsub(/^v/, "", version) + split(version, parts, ".") + printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3 + }' + + kubelet_minor_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | + sort -u | + wc -l | + tr -d ' ' + )" + containerd_major_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | + sort -u | + wc -l | + tr -d ' ' + )" + + if [[ "${kubelet_minor_count}" != "1" ]]; then + echo + echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2 + status=1 + fi + + if printf '%s\n' "${node_rows}" | + awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then + echo + echo "At least one kubelet does not match the API server minor (${api_minor})." >&2 + status=1 + fi + + if [[ "${containerd_major_count}" -gt 1 ]]; then + echo + echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2 + status=1 + fi + + if [[ "${status}" -eq 0 ]]; then + echo + echo "Node Kubernetes and containerd versions are aligned." + fi + + exit "${status}" +} + truthy() { case "${1,,}" in 1 | true | yes | on) @@ -3172,6 +3250,9 @@ case "${1:-}" in move-prometheus-stack-workers) move_prometheus_stack_workers ;; + doctor-versions) + doctor_versions + ;; openwrt) openwrt ;; @@ -3179,7 +3260,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|apps|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|apps|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" exit 1 ;; esac