Fix stop cluster worker discovery

jeannie

@@ -4337,7 +4337,6 @@ nuke_for_rebuild() {
 cluster_worker_targets() {
     local worker_ssh_targets="${WORKER_SSH_TARGETS-}"
     local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
-    local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
 
     CLUSTER_WORKER_TARGETS=()
     read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}"
@@ -4348,17 +4347,15 @@ cluster_worker_targets() {
             if [[ ! " ${CLUSTER_WORKER_TARGETS[*]} " =~ [[:space:]]${target}[[:space:]] ]]; then
                 CLUSTER_WORKER_TARGETS+=("${target}")
             fi
-        done < <(python3 - "${var_file}" "${worker_key_prefix}" <<'PY'
+        done < <(python3 - "${var_file}" <<'PY'
 import json
 import sys
 
-var_file, worker_key_prefix = sys.argv[1:3]
+var_file = sys.argv[1]
 with open(var_file, encoding="utf-8") as handle:
     document = json.load(handle)
 
 for key, node in sorted((document.get("worker_nodes") or {}).items()):
-    if key.startswith(worker_key_prefix):
-        continue
     host = node.get("host")
     user = node.get("user")
     if host and user:
@@ -4373,6 +4370,19 @@ stop_local_kubernetes_services() {
     sudo systemctl stop kubelet 2>/dev/null || true
     if command -v crictl >/dev/null 2>&1; then
         sudo crictl stop --all 2>/dev/null || true
+    elif command -v ctr >/dev/null 2>&1; then
+        sudo ctr -n k8s.io tasks ls -q 2>/dev/null |
+            while IFS= read -r task; do
+                [[ -n "${task}" ]] || continue
+                sudo ctr -n k8s.io tasks kill -s SIGTERM "${task}" 2>/dev/null || true
+            done
+        sleep 3
+        sudo ctr -n k8s.io tasks ls -q 2>/dev/null |
+            while IFS= read -r task; do
+                [[ -n "${task}" ]] || continue
+                sudo ctr -n k8s.io tasks kill -s SIGKILL "${task}" 2>/dev/null || true
+                sudo ctr -n k8s.io tasks rm -f "${task}" 2>/dev/null || true
+            done
     fi
     if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then
         sudo systemctl stop containerd 2>/dev/null || true
@@ -4393,7 +4403,23 @@ stop_remote_kubernetes_services() {
 
     echo "--> Stopping Kubernetes services on remote worker ${target}..."
     ssh -o ConnectTimeout=5 "${target}" \
-        "sudo systemctl stop kubelet 2>/dev/null || true; if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true; fi; if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi; sudo systemctl reset-failed kubelet 2>/dev/null || true"
+        "sudo systemctl stop kubelet 2>/dev/null || true
+if command -v crictl >/dev/null 2>&1; then
+  sudo crictl stop --all 2>/dev/null || true
+elif command -v ctr >/dev/null 2>&1; then
+  sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do
+    [ -n \"\$task\" ] || continue
+    sudo ctr -n k8s.io tasks kill -s SIGTERM \"\$task\" 2>/dev/null || true
+  done
+  sleep 3
+  sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do
+    [ -n \"\$task\" ] || continue
+    sudo ctr -n k8s.io tasks kill -s SIGKILL \"\$task\" 2>/dev/null || true
+    sudo ctr -n k8s.io tasks rm -f \"\$task\" 2>/dev/null || true
+  done
+fi
+if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi
+sudo systemctl reset-failed kubelet 2>/dev/null || true"
 }
 
 start_remote_kubernetes_services() {
@@ -4470,12 +4496,24 @@ PY
     printf '%s\n' "${max_count}"
 }
 
+pimox_worker_count_effective() {
+    local configured="${LAB_PIMOX_WORKER_COUNT:-}"
+    local detected
+
+    detected="$(pimox_worker_count_default)"
+    if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then
+        printf '%s\n' "${configured}"
+        return 0
+    fi
+    printf '%s\n' "${detected}"
+}
+
 stop_pimox_worker_vms() {
     local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
     local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
     local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
     local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
-    local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
+    local worker_count
     local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
     local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
     local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}"
@@ -4483,6 +4521,7 @@ stop_pimox_worker_vms() {
     local index
     local vmid
 
+    worker_count="$(pimox_worker_count_effective)"
     if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
         echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
         exit 1
@@ -4525,12 +4564,39 @@ fi"
     done
 }
 
+verify_pimox_worker_vms_stopped() {
+    local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
+    local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
+    local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
+    local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
+    local worker_count
+    local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
+    local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
+    local index
+    local vmid
+    local failures=0
+
+    worker_count="$(pimox_worker_count_effective)"
+    for ((index = 1; index <= worker_count; index++)); do
+        if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
+            continue
+        fi
+        vmid=$((worker_base_vmid + index - 1))
+        if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' 2>/dev/null | grep -q 'status: stopped'"; then
+            echo "Pimox worker VM ${vmid} is not stopped." >&2
+            failures=$((failures + 1))
+        fi
+    done
+
+    ((failures == 0))
+}
+
 destroy_pimox_worker_vms() {
     local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
     local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
     local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
     local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
-    local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
+    local worker_count
     local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
     local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
     local index
@@ -4540,6 +4606,7 @@ destroy_pimox_worker_vms() {
         echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}."
         return 0
     fi
+    worker_count="$(pimox_worker_count_effective)"
     if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
         echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
         exit 1
@@ -4574,12 +4641,13 @@ start_pimox_worker_vms() {
     local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
     local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
     local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
-    local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
+    local worker_count
     local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
     local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
     local index
     local vmid
 
+    worker_count="$(pimox_worker_count_effective)"
     if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
         echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
         exit 1
@@ -4611,6 +4679,7 @@ sudo '${qm_bin}' start '${vmid}'"
 
 stop_cluster() {
     local target
+    local failures=0
     declare -a CLUSTER_WORKER_TARGETS=()
 
     require_debian_server "stop-cluster"
@@ -4622,6 +4691,17 @@ stop_cluster() {
     done
     stop_pimox_worker_vms
     stop_local_kubernetes_services
+    if kubernetes_api_reachable; then
+        echo "Kubernetes API is still reachable after stop; control-plane containers may still be running." >&2
+        failures=$((failures + 1))
+    fi
+    if ! verify_pimox_worker_vms_stopped; then
+        failures=$((failures + 1))
+    fi
+    if ((failures > 0)); then
+        echo "Cluster stop finished with ${failures} verification failure(s)." >&2
+        return 1
+    fi
     echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact."
 }
 
@@ -5044,13 +5124,14 @@ status_pimox_workers_running() {
     local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
     local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
     local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
-    local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
+    local worker_count
     local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
     local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
     local index
     local vmid
     local failures=0
 
+    worker_count="$(pimox_worker_count_effective)"
     if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
         echo "invalid worker count ${worker_count}" >&2
         return 1
@@ -5303,7 +5384,7 @@ status_pimox_workers() {
     local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
     local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
     local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
-    local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
+    local worker_count
     local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
     local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
     local index
@@ -5311,6 +5392,7 @@ status_pimox_workers() {
 
     status_section "Pimox Workers"
 
+    worker_count="$(pimox_worker_count_effective)"
     if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
         printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}"
         return 0