diff --git a/README.md b/README.md index 7b2505c..dd893c5 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie cert-check ./jeannie release-snapshot ./jeannie backup-status +./jeannie workers list ./jeannie map ``` @@ -320,6 +321,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups. restore drill reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`workers` provides named Kubernetes/Pimox worker lifecycle operations: +`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. +Use `recreate-plan ` before replacing one broken Pimox worker so the +cluster drain, VM stop, and `./jeannie up` path stay ordered. + `map` prints a dependency map from the canonical inventory, covering public entry, GitOps automation, cluster foundation, DNS, observability, and security learning paths. `./jeannie map --dot` emits Graphviz DOT. diff --git a/README.md.tmpl b/README.md.tmpl index b8b6ffd..1528aa7 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} cert-check ./{{ main_script }} release-snapshot ./{{ main_script }} backup-status +./{{ main_script }} workers list ./{{ main_script }} map ``` @@ -320,6 +321,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups. restore drill reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`workers` provides named Kubernetes/Pimox worker lifecycle operations: +`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. +Use `recreate-plan ` before replacing one broken Pimox worker so the +cluster drain, VM stop, and `./{{ main_script }} up` path stay ordered. + `map` prints a dependency map from the canonical inventory, covering public entry, GitOps automation, cluster foundation, DNS, observability, and security learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT. diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index f03672b..e4b9951 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -81,6 +81,10 @@ pointers. reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`workers ` +: Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`, +`start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. + `map` : Print a dependency map from the canonical inventory. Use `map --dot` for a Graphviz DOT graph. diff --git a/jeannie b/jeannie index e166629..bc99ac9 100755 --- a/jeannie +++ b/jeannie @@ -5992,6 +5992,11 @@ backup_status() { "${REPO_ROOT}/scripts/backup-status" } +workers_manage() { + require_debian_server "workers" + "${REPO_ROOT}/scripts/workers" "${@:2}" +} + homelab_map() { "${REPO_ROOT}/scripts/homelab-map" "${@:2}" } @@ -6033,6 +6038,9 @@ case "${1:-}" in backup-status) backup_status ;; + workers) + workers_manage "$@" + ;; map) homelab_map "$@" ;; @@ -6180,7 +6188,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|workers |map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/workers b/scripts/workers new file mode 100755 index 0000000..c2d386e --- /dev/null +++ b/scripts/workers @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +set -euo pipefail + +KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" +PIMOX_HOST="${LAB_PIMOX_HOST:-192.168.100.80}" +PIMOX_USER="${LAB_PIMOX_USER:-jv}" +PIMOX_KEY="${LAB_PIMOX_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}" +PIMOX_QM_BIN="${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}" +WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-1}" +WORKER_BASE_VMID="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" +WORKER_NODE_PREFIX="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" +WORKER_SKIP_INDEXES="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + +usage() { + cat < [args] + +Commands: + list List desired Pimox workers, VM status, and node status + start [index|all] Start Pimox worker VM(s) + stop [index|all] Gracefully stop Pimox worker VM(s) + drain Drain a Kubernetes worker node + uncordon Mark a Kubernetes worker node schedulable + recreate-plan Print the safe recreate sequence for one Pimox worker + rebalance Restart evicted/pending pods and show scheduling pressure +EOF +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +is_skipped() { + local index="$1" + local item + local normalized="${WORKER_SKIP_INDEXES//,/ }" + + for item in $normalized; do + if [ "$item" = "$index" ]; then + return 0 + fi + done + return 1 +} + +validate_index() { + local index="$1" + + if ! [[ "$index" =~ ^[0-9]+$ ]] || [ "$index" -lt 1 ]; then + echo "worker index must be a positive integer, got '$index'" >&2 + exit 1 + fi +} + +vmid_for_index() { + local index="$1" + echo $((WORKER_BASE_VMID + index - 1)) +} + +node_for_index() { + local index="$1" + printf '%s-%02d\n' "$WORKER_NODE_PREFIX" "$index" +} + +pimox_ssh() { + ssh -i "$PIMOX_KEY" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${PIMOX_USER}@${PIMOX_HOST}" "$@" +} + +kubectl_cmd() { + kubectl --kubeconfig "$KUBECONFIG_PATH" "$@" +} + +target_indexes() { + local target="${1:-all}" + local index + + if [ "$target" = "all" ]; then + for ((index = 1; index <= WORKER_COUNT; index++)); do + if ! is_skipped "$index"; then + echo "$index" + fi + done + else + validate_index "$target" + echo "$target" + fi +} + +list_workers() { + local index + local vmid + local node + local status + local node_status + + printf '%-8s %-8s %-24s %-16s %-16s\n' "INDEX" "VMID" "NODE" "VM" "K8S" + for ((index = 1; index <= WORKER_COUNT; index++)); do + vmid="$(vmid_for_index "$index")" + node="$(node_for_index "$index")" + if is_skipped "$index"; then + printf '%-8s %-8s %-24s %-16s %-16s\n' "$index" "$vmid" "$node" "skipped" "skipped" + continue + fi + status="$(pimox_ssh "sudo '$PIMOX_QM_BIN' status '$vmid'" 2>/dev/null | awk '{print $2}' || echo unknown)" + if have kubectl && [ -s "$KUBECONFIG_PATH" ]; then + node_status="$(kubectl_cmd get node "$node" --no-headers 2>/dev/null | awk '{print $2}' || echo missing)" + else + node_status="kubectl-unavailable" + fi + printf '%-8s %-8s %-24s %-16s %-16s\n' "$index" "$vmid" "$node" "$status" "$node_status" + done +} + +start_workers() { + local index + local vmid + + for index in $(target_indexes "${1:-all}"); do + vmid="$(vmid_for_index "$index")" + echo "Starting Pimox worker index $index VM $vmid..." + pimox_ssh "sudo '$PIMOX_QM_BIN' start '$vmid'" + done +} + +stop_workers() { + local index + local vmid + + for index in $(target_indexes "${1:-all}"); do + vmid="$(vmid_for_index "$index")" + echo "Gracefully stopping Pimox worker index $index VM $vmid..." + pimox_ssh "sudo '$PIMOX_QM_BIN' shutdown '$vmid' --timeout '${LAB_WORKER_SHUTDOWN_TIMEOUT:-90}'" || { + echo "Graceful shutdown failed for VM $vmid. Use LAB_CLUSTER_STOP_FORCE=true ./jeannie stop-cluster if a force stop is intended." >&2 + exit 1 + } + done +} + +drain_node() { + local node="${1:-}" + + if [ -z "$node" ]; then + echo "workers drain requires a node name." >&2 + exit 1 + fi + kubectl_cmd drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout="${LAB_WORKER_DRAIN_TIMEOUT:-10m}" +} + +uncordon_node() { + local node="${1:-}" + + if [ -z "$node" ]; then + echo "workers uncordon requires a node name." >&2 + exit 1 + fi + kubectl_cmd uncordon "$node" +} + +recreate_plan() { + local index="${1:-}" + local vmid + local node + + validate_index "$index" + vmid="$(vmid_for_index "$index")" + node="$(node_for_index "$index")" + cat <&2 + exit 1 + fi + echo "Current node pressure:" + kubectl_cmd top nodes 2>/dev/null || kubectl_cmd get nodes -o wide + echo + echo "Problem pods:" + kubectl_cmd get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded -o wide || true + echo + echo "Restarting pending/failed pods is intentionally manual. Use kubectl delete pod for specific pods after checking events." +} + +case "${1:-}" in + list) + list_workers + ;; + start) + start_workers "${2:-all}" + ;; + stop) + stop_workers "${2:-all}" + ;; + drain) + drain_node "${2:-}" + ;; + uncordon) + uncordon_node "${2:-}" + ;; + recreate-plan) + recreate_plan "${2:-}" + ;; + rebalance) + rebalance + ;; + -h|--help|help|"") + usage + ;; + *) + usage >&2 + exit 1 + ;; +esac