Add worker lifecycle command

README.md

@@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
 ./jeannie cert-check
 ./jeannie release-snapshot
 ./jeannie backup-status
+./jeannie workers list
 ./jeannie map
 ```
 
@@ -320,6 +321,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
 restore drill reports, and repo-managed Pi-hole restore inputs. The main
 `status` cascade includes this as a warning signal.
 
+`workers` provides named Kubernetes/Pimox worker lifecycle operations:
+`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+Use `recreate-plan <index>` before replacing one broken Pimox worker so the
+cluster drain, VM stop, and `./jeannie up` path stay ordered.
+
 `map` prints a dependency map from the canonical inventory, covering public
 entry, GitOps automation, cluster foundation, DNS, observability, and security
 learning paths. `./jeannie map --dot` emits Graphviz DOT.

README.md.tmpl

@@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
 ./{{ main_script }} cert-check
 ./{{ main_script }} release-snapshot
 ./{{ main_script }} backup-status
+./{{ main_script }} workers list
 ./{{ main_script }} map
 ```
 
@@ -320,6 +321,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
 restore drill reports, and repo-managed Pi-hole restore inputs. The main
 `status` cascade includes this as a warning signal.
 
+`workers` provides named Kubernetes/Pimox worker lifecycle operations:
+`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+Use `recreate-plan <index>` before replacing one broken Pimox worker so the
+cluster drain, VM stop, and `./{{ main_script }} up` path stay ordered.
+
 `map` prints a dependency map from the canonical inventory, covering public
 entry, GitOps automation, cluster foundation, DNS, observability, and security
 learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT.

docs/jeannie.1.md

@@ -81,6 +81,10 @@ pointers.
 reports, and repo-managed Pi-hole restore inputs. The main `status` cascade
 includes this as a warning signal.
 
+`workers <command>`
+: Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
+`start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+
 `map`
 : Print a dependency map from the canonical inventory. Use `map --dot` for a
 Graphviz DOT graph.

jeannie

@@ -5992,6 +5992,11 @@ backup_status() {
     "${REPO_ROOT}/scripts/backup-status"
 }
 
+workers_manage() {
+    require_debian_server "workers"
+    "${REPO_ROOT}/scripts/workers" "${@:2}"
+}
+
 homelab_map() {
     "${REPO_ROOT}/scripts/homelab-map" "${@:2}"
 }
@@ -6033,6 +6038,9 @@ case "${1:-}" in
     backup-status)
         backup_status
         ;;
+    workers)
+        workers_manage "$@"
+        ;;
     map)
         homelab_map "$@"
         ;;
@@ -6180,7 +6188,7 @@ case "${1:-}" in
         echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     *)
-        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
+        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
         exit 1
         ;;
 esac

scripts/workers

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
+PIMOX_HOST="${LAB_PIMOX_HOST:-192.168.100.80}"
+PIMOX_USER="${LAB_PIMOX_USER:-jv}"
+PIMOX_KEY="${LAB_PIMOX_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}"
+PIMOX_QM_BIN="${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}"
+WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-1}"
+WORKER_BASE_VMID="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
+WORKER_NODE_PREFIX="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
+WORKER_SKIP_INDEXES="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
+
+usage() {
+    cat <<EOF
+Usage: ./jeannie workers <command> [args]
+
+Commands:
+  list                       List desired Pimox workers, VM status, and node status
+  start [index|all]          Start Pimox worker VM(s)
+  stop [index|all]           Gracefully stop Pimox worker VM(s)
+  drain <node>               Drain a Kubernetes worker node
+  uncordon <node>            Mark a Kubernetes worker node schedulable
+  recreate-plan <index>      Print the safe recreate sequence for one Pimox worker
+  rebalance                  Restart evicted/pending pods and show scheduling pressure
+EOF
+}
+
+have() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+is_skipped() {
+    local index="$1"
+    local item
+    local normalized="${WORKER_SKIP_INDEXES//,/ }"
+
+    for item in $normalized; do
+        if [ "$item" = "$index" ]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+validate_index() {
+    local index="$1"
+
+    if ! [[ "$index" =~ ^[0-9]+$ ]] || [ "$index" -lt 1 ]; then
+        echo "worker index must be a positive integer, got '$index'" >&2
+        exit 1
+    fi
+}
+
+vmid_for_index() {
+    local index="$1"
+    echo $((WORKER_BASE_VMID + index - 1))
+}
+
+node_for_index() {
+    local index="$1"
+    printf '%s-%02d\n' "$WORKER_NODE_PREFIX" "$index"
+}
+
+pimox_ssh() {
+    ssh -i "$PIMOX_KEY" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${PIMOX_USER}@${PIMOX_HOST}" "$@"
+}
+
+kubectl_cmd() {
+    kubectl --kubeconfig "$KUBECONFIG_PATH" "$@"
+}
+
+target_indexes() {
+    local target="${1:-all}"
+    local index
+
+    if [ "$target" = "all" ]; then
+        for ((index = 1; index <= WORKER_COUNT; index++)); do
+            if ! is_skipped "$index"; then
+                echo "$index"
+            fi
+        done
+    else
+        validate_index "$target"
+        echo "$target"
+    fi
+}
+
+list_workers() {
+    local index
+    local vmid
+    local node
+    local status
+    local node_status
+
+    printf '%-8s %-8s %-24s %-16s %-16s\n' "INDEX" "VMID" "NODE" "VM" "K8S"
+    for ((index = 1; index <= WORKER_COUNT; index++)); do
+        vmid="$(vmid_for_index "$index")"
+        node="$(node_for_index "$index")"
+        if is_skipped "$index"; then
+            printf '%-8s %-8s %-24s %-16s %-16s\n' "$index" "$vmid" "$node" "skipped" "skipped"
+            continue
+        fi
+        status="$(pimox_ssh "sudo '$PIMOX_QM_BIN' status '$vmid'" 2>/dev/null | awk '{print $2}' || echo unknown)"
+        if have kubectl && [ -s "$KUBECONFIG_PATH" ]; then
+            node_status="$(kubectl_cmd get node "$node" --no-headers 2>/dev/null | awk '{print $2}' || echo missing)"
+        else
+            node_status="kubectl-unavailable"
+        fi
+        printf '%-8s %-8s %-24s %-16s %-16s\n' "$index" "$vmid" "$node" "$status" "$node_status"
+    done
+}
+
+start_workers() {
+    local index
+    local vmid
+
+    for index in $(target_indexes "${1:-all}"); do
+        vmid="$(vmid_for_index "$index")"
+        echo "Starting Pimox worker index $index VM $vmid..."
+        pimox_ssh "sudo '$PIMOX_QM_BIN' start '$vmid'"
+    done
+}
+
+stop_workers() {
+    local index
+    local vmid
+
+    for index in $(target_indexes "${1:-all}"); do
+        vmid="$(vmid_for_index "$index")"
+        echo "Gracefully stopping Pimox worker index $index VM $vmid..."
+        pimox_ssh "sudo '$PIMOX_QM_BIN' shutdown '$vmid' --timeout '${LAB_WORKER_SHUTDOWN_TIMEOUT:-90}'" || {
+            echo "Graceful shutdown failed for VM $vmid. Use LAB_CLUSTER_STOP_FORCE=true ./jeannie stop-cluster if a force stop is intended." >&2
+            exit 1
+        }
+    done
+}
+
+drain_node() {
+    local node="${1:-}"
+
+    if [ -z "$node" ]; then
+        echo "workers drain requires a node name." >&2
+        exit 1
+    fi
+    kubectl_cmd drain "$node" --ignore-daemonsets --delete-emptydir-data --timeout="${LAB_WORKER_DRAIN_TIMEOUT:-10m}"
+}
+
+uncordon_node() {
+    local node="${1:-}"
+
+    if [ -z "$node" ]; then
+        echo "workers uncordon requires a node name." >&2
+        exit 1
+    fi
+    kubectl_cmd uncordon "$node"
+}
+
+recreate_plan() {
+    local index="${1:-}"
+    local vmid
+    local node
+
+    validate_index "$index"
+    vmid="$(vmid_for_index "$index")"
+    node="$(node_for_index "$index")"
+    cat <<EOF
+Safe recreate plan for Pimox worker index $index:
+
+1. Snapshot current state:
+   ./jeannie release-snapshot
+
+2. Drain node if it exists:
+   ./jeannie workers drain $node
+
+3. Stop only this VM:
+   ./jeannie workers stop $index
+
+4. Recreate through the normal pipeline so topology and cluster vars stay aligned:
+   LAB_PIMOX_WORKER_COUNT=$WORKER_COUNT LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=false LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
+
+5. Validate:
+   ./jeannie workers list
+   ./jeannie status
+
+Target VMID: $vmid
+Target node: $node
+EOF
+}
+
+rebalance() {
+    if ! have kubectl || [ ! -s "$KUBECONFIG_PATH" ]; then
+        echo "kubectl and kubeconfig are required for workers rebalance." >&2
+        exit 1
+    fi
+    echo "Current node pressure:"
+    kubectl_cmd top nodes 2>/dev/null || kubectl_cmd get nodes -o wide
+    echo
+    echo "Problem pods:"
+    kubectl_cmd get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded -o wide || true
+    echo
+    echo "Restarting pending/failed pods is intentionally manual. Use kubectl delete pod for specific pods after checking events."
+}
+
+case "${1:-}" in
+    list)
+        list_workers
+        ;;
+    start)
+        start_workers "${2:-all}"
+        ;;
+    stop)
+        stop_workers "${2:-all}"
+        ;;
+    drain)
+        drain_node "${2:-}"
+        ;;
+    uncordon)
+        uncordon_node "${2:-}"
+        ;;
+    recreate-plan)
+        recreate_plan "${2:-}"
+        ;;
+    rebalance)
+        rebalance
+        ;;
+    -h|--help|help|"")
+        usage
+        ;;
+    *)
+        usage >&2
+        exit 1
+        ;;
+esac