From a5bf0c2f019d643304d6cccf556f5c4b60b1d5a4 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 23:36:24 -0600 Subject: [PATCH] Fix stop cluster worker discovery --- jeannie | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 11 deletions(-) diff --git a/jeannie b/jeannie index 8c0fab7..02a37e8 100755 --- a/jeannie +++ b/jeannie @@ -4337,7 +4337,6 @@ nuke_for_rebuild() { cluster_worker_targets() { local worker_ssh_targets="${WORKER_SSH_TARGETS-}" local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" CLUSTER_WORKER_TARGETS=() read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}" @@ -4348,17 +4347,15 @@ cluster_worker_targets() { if [[ ! " ${CLUSTER_WORKER_TARGETS[*]} " =~ [[:space:]]${target}[[:space:]] ]]; then CLUSTER_WORKER_TARGETS+=("${target}") fi - done < <(python3 - "${var_file}" "${worker_key_prefix}" <<'PY' + done < <(python3 - "${var_file}" <<'PY' import json import sys -var_file, worker_key_prefix = sys.argv[1:3] +var_file = sys.argv[1] with open(var_file, encoding="utf-8") as handle: document = json.load(handle) for key, node in sorted((document.get("worker_nodes") or {}).items()): - if key.startswith(worker_key_prefix): - continue host = node.get("host") user = node.get("user") if host and user: @@ -4373,6 +4370,19 @@ stop_local_kubernetes_services() { sudo systemctl stop kubelet 2>/dev/null || true if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true + elif command -v ctr >/dev/null 2>&1; then + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | + while IFS= read -r task; do + [[ -n "${task}" ]] || continue + sudo ctr -n k8s.io tasks kill -s SIGTERM "${task}" 2>/dev/null || true + done + sleep 3 + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | + while IFS= read -r task; do + [[ -n "${task}" ]] || continue + sudo ctr -n k8s.io tasks kill -s SIGKILL "${task}" 2>/dev/null || true + sudo ctr -n k8s.io tasks rm -f "${task}" 2>/dev/null || true + done fi if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then sudo systemctl stop containerd 2>/dev/null || true @@ -4393,7 +4403,23 @@ stop_remote_kubernetes_services() { echo "--> Stopping Kubernetes services on remote worker ${target}..." ssh -o ConnectTimeout=5 "${target}" \ - "sudo systemctl stop kubelet 2>/dev/null || true; if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true; fi; if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi; sudo systemctl reset-failed kubelet 2>/dev/null || true" + "sudo systemctl stop kubelet 2>/dev/null || true +if command -v crictl >/dev/null 2>&1; then + sudo crictl stop --all 2>/dev/null || true +elif command -v ctr >/dev/null 2>&1; then + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do + [ -n \"\$task\" ] || continue + sudo ctr -n k8s.io tasks kill -s SIGTERM \"\$task\" 2>/dev/null || true + done + sleep 3 + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do + [ -n \"\$task\" ] || continue + sudo ctr -n k8s.io tasks kill -s SIGKILL \"\$task\" 2>/dev/null || true + sudo ctr -n k8s.io tasks rm -f \"\$task\" 2>/dev/null || true + done +fi +if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi +sudo systemctl reset-failed kubelet 2>/dev/null || true" } start_remote_kubernetes_services() { @@ -4470,12 +4496,24 @@ PY printf '%s\n' "${max_count}" } +pimox_worker_count_effective() { + local configured="${LAB_PIMOX_WORKER_COUNT:-}" + local detected + + detected="$(pimox_worker_count_default)" + if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then + printf '%s\n' "${configured}" + return 0 + fi + printf '%s\n' "${detected}" +} + stop_pimox_worker_vms() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_count local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}" @@ -4483,6 +4521,7 @@ stop_pimox_worker_vms() { local index local vmid + worker_count="$(pimox_worker_count_effective)" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 @@ -4525,12 +4564,39 @@ fi" done } +verify_pimox_worker_vms_stopped() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + local failures=0 + + worker_count="$(pimox_worker_count_effective)" + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + continue + fi + vmid=$((worker_base_vmid + index - 1)) + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' 2>/dev/null | grep -q 'status: stopped'"; then + echo "Pimox worker VM ${vmid} is not stopped." >&2 + failures=$((failures + 1)) + fi + done + + ((failures == 0)) +} + destroy_pimox_worker_vms() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_count local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index @@ -4540,6 +4606,7 @@ destroy_pimox_worker_vms() { echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}." return 0 fi + worker_count="$(pimox_worker_count_effective)" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 @@ -4574,12 +4641,13 @@ start_pimox_worker_vms() { local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_count local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid + worker_count="$(pimox_worker_count_effective)" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 @@ -4611,6 +4679,7 @@ sudo '${qm_bin}' start '${vmid}'" stop_cluster() { local target + local failures=0 declare -a CLUSTER_WORKER_TARGETS=() require_debian_server "stop-cluster" @@ -4622,6 +4691,17 @@ stop_cluster() { done stop_pimox_worker_vms stop_local_kubernetes_services + if kubernetes_api_reachable; then + echo "Kubernetes API is still reachable after stop; control-plane containers may still be running." >&2 + failures=$((failures + 1)) + fi + if ! verify_pimox_worker_vms_stopped; then + failures=$((failures + 1)) + fi + if ((failures > 0)); then + echo "Cluster stop finished with ${failures} verification failure(s)." >&2 + return 1 + fi echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact." } @@ -5044,13 +5124,14 @@ status_pimox_workers_running() { local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_count local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid local failures=0 + worker_count="$(pimox_worker_count_effective)" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "invalid worker count ${worker_count}" >&2 return 1 @@ -5303,7 +5384,7 @@ status_pimox_workers() { local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_count local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index @@ -5311,6 +5392,7 @@ status_pimox_workers() { status_section "Pimox Workers" + worker_count="$(pimox_worker_count_effective)" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}" return 0