Fix stop cluster worker discovery
This commit is contained in:
juvdiaz
parent
19964cfc20
commit
a5bf0c2f01
104
jeannie
104
jeannie
|
|
@ -4337,7 +4337,6 @@ nuke_for_rebuild() {
|
|||
cluster_worker_targets() {
|
||||
local worker_ssh_targets="${WORKER_SSH_TARGETS-}"
|
||||
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
|
||||
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
|
||||
|
||||
CLUSTER_WORKER_TARGETS=()
|
||||
read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}"
|
||||
|
|
@ -4348,17 +4347,15 @@ cluster_worker_targets() {
|
|||
if [[ ! " ${CLUSTER_WORKER_TARGETS[*]} " =~ [[:space:]]${target}[[:space:]] ]]; then
|
||||
CLUSTER_WORKER_TARGETS+=("${target}")
|
||||
fi
|
||||
done < <(python3 - "${var_file}" "${worker_key_prefix}" <<'PY'
|
||||
done < <(python3 - "${var_file}" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
var_file, worker_key_prefix = sys.argv[1:3]
|
||||
var_file = sys.argv[1]
|
||||
with open(var_file, encoding="utf-8") as handle:
|
||||
document = json.load(handle)
|
||||
|
||||
for key, node in sorted((document.get("worker_nodes") or {}).items()):
|
||||
if key.startswith(worker_key_prefix):
|
||||
continue
|
||||
host = node.get("host")
|
||||
user = node.get("user")
|
||||
if host and user:
|
||||
|
|
@ -4373,6 +4370,19 @@ stop_local_kubernetes_services() {
|
|||
sudo systemctl stop kubelet 2>/dev/null || true
|
||||
if command -v crictl >/dev/null 2>&1; then
|
||||
sudo crictl stop --all 2>/dev/null || true
|
||||
elif command -v ctr >/dev/null 2>&1; then
|
||||
sudo ctr -n k8s.io tasks ls -q 2>/dev/null |
|
||||
while IFS= read -r task; do
|
||||
[[ -n "${task}" ]] || continue
|
||||
sudo ctr -n k8s.io tasks kill -s SIGTERM "${task}" 2>/dev/null || true
|
||||
done
|
||||
sleep 3
|
||||
sudo ctr -n k8s.io tasks ls -q 2>/dev/null |
|
||||
while IFS= read -r task; do
|
||||
[[ -n "${task}" ]] || continue
|
||||
sudo ctr -n k8s.io tasks kill -s SIGKILL "${task}" 2>/dev/null || true
|
||||
sudo ctr -n k8s.io tasks rm -f "${task}" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then
|
||||
sudo systemctl stop containerd 2>/dev/null || true
|
||||
|
|
@ -4393,7 +4403,23 @@ stop_remote_kubernetes_services() {
|
|||
|
||||
echo "--> Stopping Kubernetes services on remote worker ${target}..."
|
||||
ssh -o ConnectTimeout=5 "${target}" \
|
||||
"sudo systemctl stop kubelet 2>/dev/null || true; if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true; fi; if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi; sudo systemctl reset-failed kubelet 2>/dev/null || true"
|
||||
"sudo systemctl stop kubelet 2>/dev/null || true
|
||||
if command -v crictl >/dev/null 2>&1; then
|
||||
sudo crictl stop --all 2>/dev/null || true
|
||||
elif command -v ctr >/dev/null 2>&1; then
|
||||
sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do
|
||||
[ -n \"\$task\" ] || continue
|
||||
sudo ctr -n k8s.io tasks kill -s SIGTERM \"\$task\" 2>/dev/null || true
|
||||
done
|
||||
sleep 3
|
||||
sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do
|
||||
[ -n \"\$task\" ] || continue
|
||||
sudo ctr -n k8s.io tasks kill -s SIGKILL \"\$task\" 2>/dev/null || true
|
||||
sudo ctr -n k8s.io tasks rm -f \"\$task\" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi
|
||||
sudo systemctl reset-failed kubelet 2>/dev/null || true"
|
||||
}
|
||||
|
||||
start_remote_kubernetes_services() {
|
||||
|
|
@ -4470,12 +4496,24 @@ PY
|
|||
printf '%s\n' "${max_count}"
|
||||
}
|
||||
|
||||
pimox_worker_count_effective() {
|
||||
local configured="${LAB_PIMOX_WORKER_COUNT:-}"
|
||||
local detected
|
||||
|
||||
detected="$(pimox_worker_count_default)"
|
||||
if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then
|
||||
printf '%s\n' "${configured}"
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' "${detected}"
|
||||
}
|
||||
|
||||
stop_pimox_worker_vms() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}"
|
||||
|
|
@ -4483,6 +4521,7 @@ stop_pimox_worker_vms() {
|
|||
local index
|
||||
local vmid
|
||||
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
|
||||
exit 1
|
||||
|
|
@ -4525,12 +4564,39 @@ fi"
|
|||
done
|
||||
}
|
||||
|
||||
verify_pimox_worker_vms_stopped() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
local vmid
|
||||
local failures=0
|
||||
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
for ((index = 1; index <= worker_count; index++)); do
|
||||
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
||||
continue
|
||||
fi
|
||||
vmid=$((worker_base_vmid + index - 1))
|
||||
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' 2>/dev/null | grep -q 'status: stopped'"; then
|
||||
echo "Pimox worker VM ${vmid} is not stopped." >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
((failures == 0))
|
||||
}
|
||||
|
||||
destroy_pimox_worker_vms() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
|
|
@ -4540,6 +4606,7 @@ destroy_pimox_worker_vms() {
|
|||
echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}."
|
||||
return 0
|
||||
fi
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
|
||||
exit 1
|
||||
|
|
@ -4574,12 +4641,13 @@ start_pimox_worker_vms() {
|
|||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
local vmid
|
||||
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
|
||||
exit 1
|
||||
|
|
@ -4611,6 +4679,7 @@ sudo '${qm_bin}' start '${vmid}'"
|
|||
|
||||
stop_cluster() {
|
||||
local target
|
||||
local failures=0
|
||||
declare -a CLUSTER_WORKER_TARGETS=()
|
||||
|
||||
require_debian_server "stop-cluster"
|
||||
|
|
@ -4622,6 +4691,17 @@ stop_cluster() {
|
|||
done
|
||||
stop_pimox_worker_vms
|
||||
stop_local_kubernetes_services
|
||||
if kubernetes_api_reachable; then
|
||||
echo "Kubernetes API is still reachable after stop; control-plane containers may still be running." >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
if ! verify_pimox_worker_vms_stopped; then
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
if ((failures > 0)); then
|
||||
echo "Cluster stop finished with ${failures} verification failure(s)." >&2
|
||||
return 1
|
||||
fi
|
||||
echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact."
|
||||
}
|
||||
|
||||
|
|
@ -5044,13 +5124,14 @@ status_pimox_workers_running() {
|
|||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
local vmid
|
||||
local failures=0
|
||||
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "invalid worker count ${worker_count}" >&2
|
||||
return 1
|
||||
|
|
@ -5303,7 +5384,7 @@ status_pimox_workers() {
|
|||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_count
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
|
|
@ -5311,6 +5392,7 @@ status_pimox_workers() {
|
|||
|
||||
status_section "Pimox Workers"
|
||||
|
||||
worker_count="$(pimox_worker_count_effective)"
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}"
|
||||
return 0
|
||||
|
|
|
|||
Loading…
Reference in New Issue