diff --git a/README.md b/README.md index d06abd9..f7be138 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,26 @@ disabled by default because the Bullseye-pinned Pimox `qm` does not support it. The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. +To pause the Kubernetes cluster without deleting kubeadm files, OpenTofu state, +PV data, or VM disks, run: + +```bash +./lab.sh stop-cluster +``` + +This stops `kubelet` and `containerd` on the Debian control plane, stops any +workers listed in `WORKER_SSH_TARGETS`, and gracefully shuts down automated +Pimox worker VMs. It does not run `kubeadm reset`, delete CNI files, or remove +state. Use `LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS` to change the Pimox shutdown +timeout, `LAB_CLUSTER_STOP_FORCE=false` to avoid a hard `qm stop` after timeout, +and the usual `LAB_PIMOX_WORKER_COUNT`, `LAB_PIMOX_WORKER_BASE_VMID`, and +`LAB_PIMOX_SKIP_WORKER_INDEXES` variables to select VM workers. Resume the +runtime with: + +```bash +./lab.sh start-cluster +``` + To exclude the Raspberry Pi from the Kubernetes cluster, set `LAB_INCLUDE_RASPBERRY_WORKER=false`. To manage workers manually instead, add entries to diff --git a/lab.sh b/lab.sh index e17d474..88d2409 100755 --- a/lab.sh +++ b/lab.sh @@ -3250,6 +3250,179 @@ rebuild_cluster() { echo "Cluster rebuild successfully completed." } +cluster_worker_targets() { + local worker_ssh_targets="${WORKER_SSH_TARGETS-}" + + read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}" +} + +stop_local_kubernetes_services() { + echo "--> Stopping local Kubernetes services on the Debian control plane..." + sudo systemctl stop kubelet 2>/dev/null || true + sudo systemctl stop containerd 2>/dev/null || true + sudo systemctl reset-failed kubelet containerd 2>/dev/null || true +} + +start_local_kubernetes_services() { + echo "--> Starting local Kubernetes services on the Debian control plane..." + sudo systemctl start containerd + sudo systemctl start kubelet +} + +stop_remote_kubernetes_services() { + local target="$1" + + echo "--> Stopping Kubernetes services on remote worker ${target}..." + ssh -o ConnectTimeout=5 "${target}" \ + "sudo systemctl stop kubelet 2>/dev/null || true; sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed kubelet containerd 2>/dev/null || true" +} + +start_remote_kubernetes_services() { + local target="$1" + + echo "--> Starting Kubernetes services on remote worker ${target}..." + ssh -o ConnectTimeout=5 "${target}" \ + "sudo systemctl start containerd && sudo systemctl start kubelet" +} + +pimox_worker_count_default() { + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + + if [[ -s "${spec_file}" ]]; then + awk 'NF { count++ } END { print count + 0 }' "${spec_file}" + return 0 + fi + + printf '1\n' +} + +stop_pimox_worker_vms() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}" + local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}" + local index + local vmid + + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 + exit 1 + fi + if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then + echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2 + exit 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + echo "--> Stopping Pimox worker VMs on ${pimox_host}..." + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + vmid=$((worker_base_vmid + index - 1)) + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then + echo 'Pimox worker VM ${vmid} does not exist; skipping.' + exit 0 +fi +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + echo 'Pimox worker VM ${vmid} is already stopped.' + exit 0 +fi +echo 'Gracefully shutting down Pimox worker VM ${vmid}...' +if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then + exit 0 +fi +if [ '${force_stop}' = 'true' ]; then + echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.' + sudo '${qm_bin}' stop '${vmid}' +else + echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2 + exit 1 +fi" + done +} + +start_pimox_worker_vms() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 + exit 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + echo "--> Starting Pimox worker VMs on ${pimox_host}..." + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + vmid=$((worker_base_vmid + index - 1)) + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then + echo 'Pimox worker VM ${vmid} does not exist; skipping.' + exit 0 +fi +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then + echo 'Pimox worker VM ${vmid} is already running.' + exit 0 +fi +echo 'Starting Pimox worker VM ${vmid}...' +sudo '${qm_bin}' start '${vmid}'" + done +} + +stop_cluster() { + local target + declare -a CLUSTER_WORKER_TARGETS=() + + require_debian_server "stop-cluster" + cluster_worker_targets + + echo "Stopping Kubernetes cluster runtime without destroying state..." + for target in "${CLUSTER_WORKER_TARGETS[@]}"; do + stop_remote_kubernetes_services "${target}" + done + stop_pimox_worker_vms + stop_local_kubernetes_services + echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact." +} + +start_cluster() { + local target + declare -a CLUSTER_WORKER_TARGETS=() + + require_debian_server "start-cluster" + cluster_worker_targets + + echo "Starting Kubernetes cluster runtime without rebuilding state..." + start_local_kubernetes_services + start_pimox_worker_vms + for target in "${CLUSTER_WORKER_TARGETS[@]}"; do + start_remote_kubernetes_services "${target}" + done + echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness." +} + nuke() { local worker_ssh_targets local worker_targets @@ -3398,6 +3571,12 @@ case "${1:-}" in rebuild-cluster) rebuild_cluster ;; + stop-cluster) + stop_cluster + ;; + start-cluster) + start_cluster + ;; apps) apps ;; @@ -3435,7 +3614,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|apps|website-translation-model|website-ollama-listen|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|apps|website-translation-model|website-ollama-listen|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" exit 1 ;; esac