Add cluster stop and start commands
This commit is contained in:
juvdiaz
parent
8ebc87f06f
commit
09db6370b0
20
README.md
20
README.md
|
|
@ -258,6 +258,26 @@ disabled by default because the Bullseye-pinned Pimox `qm` does not support it.
|
|||
The Raspberry Pi is still included as a Kubernetes worker by default; `nuke`
|
||||
does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`.
|
||||
|
||||
To pause the Kubernetes cluster without deleting kubeadm files, OpenTofu state,
|
||||
PV data, or VM disks, run:
|
||||
|
||||
```bash
|
||||
./lab.sh stop-cluster
|
||||
```
|
||||
|
||||
This stops `kubelet` and `containerd` on the Debian control plane, stops any
|
||||
workers listed in `WORKER_SSH_TARGETS`, and gracefully shuts down automated
|
||||
Pimox worker VMs. It does not run `kubeadm reset`, delete CNI files, or remove
|
||||
state. Use `LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS` to change the Pimox shutdown
|
||||
timeout, `LAB_CLUSTER_STOP_FORCE=false` to avoid a hard `qm stop` after timeout,
|
||||
and the usual `LAB_PIMOX_WORKER_COUNT`, `LAB_PIMOX_WORKER_BASE_VMID`, and
|
||||
`LAB_PIMOX_SKIP_WORKER_INDEXES` variables to select VM workers. Resume the
|
||||
runtime with:
|
||||
|
||||
```bash
|
||||
./lab.sh start-cluster
|
||||
```
|
||||
|
||||
To exclude the Raspberry Pi from the Kubernetes cluster, set
|
||||
`LAB_INCLUDE_RASPBERRY_WORKER=false`. To manage workers manually instead, add
|
||||
entries to
|
||||
|
|
|
|||
181
lab.sh
181
lab.sh
|
|
@ -3250,6 +3250,179 @@ rebuild_cluster() {
|
|||
echo "Cluster rebuild successfully completed."
|
||||
}
|
||||
|
||||
cluster_worker_targets() {
|
||||
local worker_ssh_targets="${WORKER_SSH_TARGETS-}"
|
||||
|
||||
read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}"
|
||||
}
|
||||
|
||||
stop_local_kubernetes_services() {
|
||||
echo "--> Stopping local Kubernetes services on the Debian control plane..."
|
||||
sudo systemctl stop kubelet 2>/dev/null || true
|
||||
sudo systemctl stop containerd 2>/dev/null || true
|
||||
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
|
||||
}
|
||||
|
||||
start_local_kubernetes_services() {
|
||||
echo "--> Starting local Kubernetes services on the Debian control plane..."
|
||||
sudo systemctl start containerd
|
||||
sudo systemctl start kubelet
|
||||
}
|
||||
|
||||
stop_remote_kubernetes_services() {
|
||||
local target="$1"
|
||||
|
||||
echo "--> Stopping Kubernetes services on remote worker ${target}..."
|
||||
ssh -o ConnectTimeout=5 "${target}" \
|
||||
"sudo systemctl stop kubelet 2>/dev/null || true; sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed kubelet containerd 2>/dev/null || true"
|
||||
}
|
||||
|
||||
start_remote_kubernetes_services() {
|
||||
local target="$1"
|
||||
|
||||
echo "--> Starting Kubernetes services on remote worker ${target}..."
|
||||
ssh -o ConnectTimeout=5 "${target}" \
|
||||
"sudo systemctl start containerd && sudo systemctl start kubelet"
|
||||
}
|
||||
|
||||
pimox_worker_count_default() {
|
||||
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
|
||||
|
||||
if [[ -s "${spec_file}" ]]; then
|
||||
awk 'NF { count++ } END { print count + 0 }' "${spec_file}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '1\n'
|
||||
}
|
||||
|
||||
stop_pimox_worker_vms() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}"
|
||||
local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}"
|
||||
local index
|
||||
local vmid
|
||||
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ((worker_count == 0)); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "--> Stopping Pimox worker VMs on ${pimox_host}..."
|
||||
for ((index = 1; index <= worker_count; index++)); do
|
||||
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
||||
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
|
||||
continue
|
||||
fi
|
||||
vmid=$((worker_base_vmid + index - 1))
|
||||
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
||||
if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then
|
||||
echo 'Pimox worker VM ${vmid} does not exist; skipping.'
|
||||
exit 0
|
||||
fi
|
||||
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
|
||||
echo 'Pimox worker VM ${vmid} is already stopped.'
|
||||
exit 0
|
||||
fi
|
||||
echo 'Gracefully shutting down Pimox worker VM ${vmid}...'
|
||||
if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then
|
||||
exit 0
|
||||
fi
|
||||
if [ '${force_stop}' = 'true' ]; then
|
||||
echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.'
|
||||
sudo '${qm_bin}' stop '${vmid}'
|
||||
else
|
||||
echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2
|
||||
exit 1
|
||||
fi"
|
||||
done
|
||||
}
|
||||
|
||||
start_pimox_worker_vms() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||
local index
|
||||
local vmid
|
||||
|
||||
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
|
||||
exit 1
|
||||
fi
|
||||
if ((worker_count == 0)); then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "--> Starting Pimox worker VMs on ${pimox_host}..."
|
||||
for ((index = 1; index <= worker_count; index++)); do
|
||||
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
||||
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
|
||||
continue
|
||||
fi
|
||||
vmid=$((worker_base_vmid + index - 1))
|
||||
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
||||
if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then
|
||||
echo 'Pimox worker VM ${vmid} does not exist; skipping.'
|
||||
exit 0
|
||||
fi
|
||||
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then
|
||||
echo 'Pimox worker VM ${vmid} is already running.'
|
||||
exit 0
|
||||
fi
|
||||
echo 'Starting Pimox worker VM ${vmid}...'
|
||||
sudo '${qm_bin}' start '${vmid}'"
|
||||
done
|
||||
}
|
||||
|
||||
stop_cluster() {
|
||||
local target
|
||||
declare -a CLUSTER_WORKER_TARGETS=()
|
||||
|
||||
require_debian_server "stop-cluster"
|
||||
cluster_worker_targets
|
||||
|
||||
echo "Stopping Kubernetes cluster runtime without destroying state..."
|
||||
for target in "${CLUSTER_WORKER_TARGETS[@]}"; do
|
||||
stop_remote_kubernetes_services "${target}"
|
||||
done
|
||||
stop_pimox_worker_vms
|
||||
stop_local_kubernetes_services
|
||||
echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact."
|
||||
}
|
||||
|
||||
start_cluster() {
|
||||
local target
|
||||
declare -a CLUSTER_WORKER_TARGETS=()
|
||||
|
||||
require_debian_server "start-cluster"
|
||||
cluster_worker_targets
|
||||
|
||||
echo "Starting Kubernetes cluster runtime without rebuilding state..."
|
||||
start_local_kubernetes_services
|
||||
start_pimox_worker_vms
|
||||
for target in "${CLUSTER_WORKER_TARGETS[@]}"; do
|
||||
start_remote_kubernetes_services "${target}"
|
||||
done
|
||||
echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness."
|
||||
}
|
||||
|
||||
nuke() {
|
||||
local worker_ssh_targets
|
||||
local worker_targets
|
||||
|
|
@ -3398,6 +3571,12 @@ case "${1:-}" in
|
|||
rebuild-cluster)
|
||||
rebuild_cluster
|
||||
;;
|
||||
stop-cluster)
|
||||
stop_cluster
|
||||
;;
|
||||
start-cluster)
|
||||
start_cluster
|
||||
;;
|
||||
apps)
|
||||
apps
|
||||
;;
|
||||
|
|
@ -3435,7 +3614,7 @@ case "${1:-}" in
|
|||
nuke
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {up|rebuild-cluster|apps|website-translation-model|website-ollama-listen|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}"
|
||||
echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|apps|website-translation-model|website-ollama-listen|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue