From d94ae2619ba49eb097de7272a29b2f42a7b1f7c2 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Thu, 4 Jun 2026 21:10:43 -0600 Subject: [PATCH] Repair node version drift in pipeline --- .gitea/workflows/homelab-main.yml | 16 ++++- README.md | 14 +++-- bootstrap/cluster/main.tf | 58 ++++++++++++++++++- bootstrap/provisioning/README.md | 2 + bootstrap/provisioning/main.tf | 1 - .../templates/golden-node-prepare.sh.tftpl | 26 +++++++++ lab.sh | 17 +++++- 7 files changed, 122 insertions(+), 12 deletions(-) diff --git a/.gitea/workflows/homelab-main.yml b/.gitea/workflows/homelab-main.yml index 1197606..4ff874c 100644 --- a/.gitea/workflows/homelab-main.yml +++ b/.gitea/workflows/homelab-main.yml @@ -159,7 +159,21 @@ jobs: action_command="${HOMELAB_ACTION_COMMAND:-auto}" if [[ "${action_command}" == "auto" ]]; then if [[ -s "${kubeconfig_probe}" ]]; then - action_command="apps" + set +e + version_report="$("${deploy_dir}/lab.sh" doctor-versions 2>&1)" + version_status=$? + set -e + printf '%s\n' "${version_report}" + + if ((version_status == 0)); then + action_command="apps" + elif printf '%s\n' "${version_report}" | grep -Eq 'Kubernetes kubelet minors are mixed|At least one kubelet does not match'; then + echo "Kubernetes node version drift detected; switching automatic deploy to rebuild-cluster." + action_command="rebuild-cluster" + else + echo "Version guard failed without Kubernetes minor drift; refusing automatic app deploy." >&2 + exit "${version_status}" + fi else action_command="rebuild-cluster" fi diff --git a/README.md b/README.md index ff8cc84..622963d 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,11 @@ Check node versions from the Debian host: The check prints the API server, kubelet versions, kubelet minors, and containerd runtimes. It exits nonzero when kubelet minors differ from the API server minor -or when containerd major versions are mixed. For Pimox workers, prefer rebuilding -from the corrected golden template and rejoining the worker over ad hoc live -package upgrades. +or when containerd major versions are mixed. The Gitea Actions auto path runs +this check before app deploys; kubelet minor drift switches the job to +`./lab.sh rebuild-cluster` so the Pimox template and worker VMs are replaced +from the pinned image. For Pimox workers, prefer rebuilding from the corrected +golden template and rejoining the worker over ad hoc live package upgrades. ## Deploying @@ -575,8 +577,10 @@ The workflow only blocks automatic deploy for Raspberry Pi Gitea service changes: files under `infra/gitea/`, or edits inside the `deploy_gitea`, `install_gitea_backup_timer`, `backup_gitea`, or `drill_gitea_restore` functions in `lab.sh`. Other changes use `HOMELAB_ACTION_COMMAND=auto` by -default: Actions runs `./lab.sh apps` when the Debian runner already has a -cluster kubeconfig, or `./lab.sh rebuild-cluster` when it does not. +default: Actions runs `./lab.sh doctor-versions` when the Debian runner already +has a cluster kubeconfig. If node versions are aligned it runs `./lab.sh apps`; +if kubelet minor drift is detected it runs `./lab.sh rebuild-cluster`; if no +kubeconfig exists it also runs `./lab.sh rebuild-cluster`. Set `HOMELAB_ACTION_COMMAND=apps` or `HOMELAB_ACTION_COMMAND=rebuild-cluster` on the runner to force one path. diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index ee9f0ef..e69e489 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -21,6 +21,7 @@ resource "null_resource" "kubeadm_control_plane" { kubeconfig_owner = var.kubeconfig_owner registry_endpoint = var.registry_endpoint registry_config_version = "8" + containerd_runtime = "docker-containerd-io" cni_plugins_version = "2" node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) @@ -44,6 +45,31 @@ install_missing_packages() { fi } +install_containerd_runtime() { + codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")" + arch="$(dpkg --print-architecture)" + + sudo install -d -m 0755 /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null </dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then + sudo apt-get remove -y containerd runc || true + fi + + sudo apt-get update + sudo apt-get install -y --no-install-recommends containerd.io + sudo apt-mark hold containerd.io +} + configure_node_dns() { dns_servers="${self.triggers.node_dns_servers}" if [ -z "$dns_servers" ]; then @@ -269,7 +295,8 @@ REGISTRY_EOT } configure_node_dns -install_missing_packages containernetworking-plugins open-iscsi nfs-common +install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common +install_containerd_runtime install_cni_plugins sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true @@ -364,6 +391,7 @@ resource "null_resource" "kubeadm_worker" { ssh_key_path = each.value.ssh_key_path registry_endpoint = var.registry_endpoint registry_config_version = "8" + containerd_runtime = "docker-containerd-io" cni_plugins_version = "2" node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) @@ -404,6 +432,31 @@ install_missing_packages() { fi } +install_containerd_runtime() { + codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")" + arch="$(dpkg --print-architecture)" + + sudo install -d -m 0755 /etc/apt/keyrings + sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + sudo chmod a+r /etc/apt/keyrings/docker.asc + sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null </dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then + sudo apt-get remove -y containerd runc || true + fi + + sudo apt-get update + sudo apt-get install -y --no-install-recommends containerd.io + sudo apt-mark hold containerd.io +} + configure_node_dns() { dns_servers="${self.triggers.node_dns_servers}" if [ -z "$dns_servers" ]; then @@ -629,7 +682,8 @@ REGISTRY_EOT } configure_node_dns -install_missing_packages containernetworking-plugins open-iscsi nfs-common +install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common +install_containerd_runtime install_cni_plugins sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index ad52317..804e0f9 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -15,6 +15,8 @@ to edit Orange Pi host networking. - a preseed file for unattended Debian install - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path - Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36` +- `containerd.io` from Docker's Debian repository so worker runtimes use the + same package source as the Debian control-plane host - kernel boot options for cgroup support through `TF_VAR_kernel_cgroup_boot_options` - a template sealing script at `/usr/local/sbin/homelab-prepare-template.sh` inside the installed VM that verifies cgroup boot state before sealing diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index 3684e2f..aaa22d6 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -24,7 +24,6 @@ locals { "apt-transport-https", "qemu-guest-agent", "cloud-init", - "containerd", "open-iscsi", "nfs-common", "iptables", diff --git a/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl b/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl index 5ca9d77..1aedb7d 100644 --- a/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl +++ b/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl @@ -100,6 +100,31 @@ install_kubernetes_tools() { apt-mark hold kubelet kubeadm kubectl } +install_containerd_runtime() { + codename="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")" + arch="$(dpkg --print-architecture)" + + install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc + chmod a+r /etc/apt/keyrings/docker.asc + cat >/etc/apt/sources.list.d/docker.sources </dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then + apt-get remove -y containerd runc || true + fi + + apt-get update + apt-get install -y --no-install-recommends containerd.io + apt-mark hold containerd.io +} + configure_containerd() { mkdir -p /etc/containerd /etc/containerd/certs.d/${registry_endpoint} containerd config default >/etc/containerd/config.toml @@ -170,5 +195,6 @@ configure_dns configure_kubernetes_prereqs configure_kernel_boot_options install_kubernetes_tools +install_containerd_runtime configure_containerd enable_services diff --git a/lab.sh b/lab.sh index 8c7740b..6026623 100755 --- a/lab.sh +++ b/lab.sh @@ -309,7 +309,7 @@ move_prometheus_stack_workers() { kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide } -doctor_versions() { +doctor_versions_report() { local api_version local api_minor local containerd_major_count @@ -332,7 +332,7 @@ doctor_versions() { if [[ -z "${node_rows}" ]]; then echo "No Kubernetes nodes found." >&2 - exit 1 + return 1 fi printf 'API server: %s\n\n' "${api_version}" @@ -384,7 +384,12 @@ doctor_versions() { echo "Node Kubernetes and containerd versions are aligned." fi - exit "${status}" + return "${status}" +} + +doctor_versions() { + doctor_versions_report + exit "$?" } truthy() { @@ -3053,6 +3058,7 @@ up() { prepare_cluster_worker_var_file true fi run_tofu_stack "bootstrap/cluster" + doctor_versions_report run_tofu_stack "bootstrap/platform" apps run_tofu_stack "bootstrap/edge" @@ -3064,6 +3070,10 @@ rebuild_cluster() { require_debian_server "rebuild-cluster" export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}" + export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-true}" + export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}" + export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}" + export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}" echo "Rebuilding the Kubernetes cluster without touching external Raspberry Pi Gitea..." @@ -3074,6 +3084,7 @@ rebuild_cluster() { prepare_cluster_worker_var_file true fi run_tofu_stack "bootstrap/cluster" + doctor_versions_report run_tofu_stack "bootstrap/platform" apps run_tofu_stack "bootstrap/edge"