From a006cfc3fec33aaee33b38706b9669ba28564206 Mon Sep 17 00:00:00 2001
From: juvdiaz <juvenal.diaz@oracle.com>
Date: Thu, 4 Jun 2026 23:09:30 -0600
Subject: [PATCH] Stabilize worker rejoin reset

---
 bootstrap/cluster/main.tf | 39 ++++++++++++++++++++++++++++++---------
 1 file changed, 30 insertions(+), 9 deletions(-)

diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf
index c1b51e3..7da4685 100644
--- a/bootstrap/cluster/main.tf
+++ b/bootstrap/cluster/main.tf
@@ -395,6 +395,7 @@ resource "null_resource" "kubeadm_worker" {
     containerd_runtime                   = "docker-containerd-io"
     cni_plugins_version                  = "2"
     force_rejoin                         = tostring(var.force_worker_rejoin)
+    worker_rejoin_reset_version          = "2"
     node_dns_servers                     = join(" ", var.node_dns_servers)
     persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
     tailscale_nodeport_version           = "3"
@@ -824,6 +825,25 @@ configure_tailscale_subnet_routes \
 
 configure_containerd_registry "${self.triggers.registry_endpoint}"
 
+reset_worker_join_state() {
+  sudo systemctl stop kubelet 2>/dev/null || true
+  sudo kubeadm reset --force || true
+  sudo systemctl stop kubelet 2>/dev/null || true
+  sudo systemctl stop containerd 2>/dev/null || true
+
+  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/
+  sudo mkdir -p /etc/cni/net.d
+  sudo rm -rf /etc/cni/net.d/*
+  sudo mkdir -p /var/lib/kubelet /var/lib/cni
+
+  sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
+  if ! sudo systemctl start containerd; then
+    sudo systemctl status containerd --no-pager -l || true
+    sudo journalctl -u containerd --no-pager -n 160 || true
+    exit 1
+  fi
+}
+
 pv_dirs="${self.triggers.persistent_volume_dirs}"
 IFS=','
 for path in $pv_dirs; do
@@ -832,21 +852,15 @@ for path in $pv_dirs; do
 done
 
 if [ "${self.triggers.force_rejoin}" = "true" ]; then
-  sudo kubeadm reset --force || true
-  sudo systemctl stop kubelet 2>/dev/null || true
-  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
+  reset_worker_join_state
 fi
 
 if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then
-  sudo kubeadm reset --force || true
-  sudo systemctl stop kubelet 2>/dev/null || true
-  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
+  reset_worker_join_state
 fi
 
 if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
-  sudo kubeadm reset --force || true
-  sudo systemctl stop kubelet 2>/dev/null || true
-  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
+  reset_worker_join_state
 fi
 
 if [ ! -f /etc/kubernetes/kubelet.conf ]; then
@@ -856,6 +870,13 @@ if [ ! -f /etc/kubernetes/kubelet.conf ]; then
     sudo journalctl -u kubelet --no-pager -n 160 || true
     exit 1
   fi
+  if [ ! -f /etc/kubernetes/kubelet.conf ] || [ ! -f /var/lib/kubelet/config.yaml ]; then
+    echo "kubeadm join completed without creating kubelet configuration" >&2
+    sudo ls -la /etc/kubernetes /var/lib/kubelet 2>/dev/null || true
+    sudo systemctl status kubelet containerd --no-pager -l || true
+    sudo journalctl -u kubelet --no-pager -n 160 || true
+    exit 1
+  fi
 fi
 EOT
     ]