From 00f8801fd79651576d095fe89998461bcfb06932 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 17:20:40 -0600 Subject: [PATCH] Add homelab recovery plan command --- README.md | 5 +++ README.md.tmpl | 5 +++ docs/jeannie.1.md | 5 +++ jeannie | 10 ++++- scripts/recover-plan | 93 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 117 insertions(+), 1 deletion(-) create mode 100755 scripts/recover-plan diff --git a/README.md b/README.md index 6c156d0..ee9c767 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ Run a read-only health snapshot from the Debian server with: ```bash ./jeannie status ./jeannie capacity +./jeannie recover-plan ``` It reports host memory/disk, systemd services, Docker Compose stacks, @@ -295,6 +296,10 @@ summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement guidance for deciding what can safely run next. +`recover-plan` prints the disaster recovery order and lightweight prerequisite +checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and +edge verification. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/README.md.tmpl b/README.md.tmpl index 4f18cde..c89ae7d 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -271,6 +271,7 @@ Run a read-only health snapshot from the Debian server with: ```bash ./{{ main_script }} status ./{{ main_script }} capacity +./{{ main_script }} recover-plan ``` It reports host memory/disk, systemd services, Docker Compose stacks, @@ -295,6 +296,10 @@ summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement guidance for deciding what can safely run next. +`recover-plan` prints the disaster recovery order and lightweight prerequisite +checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and +edge verification. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 3c06e9c..2e85aec 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -57,6 +57,11 @@ deployment readiness, pod restarts, node pressure, disk pressure, Traefik Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk state, and practical placement guidance. +`recover-plan` +: Print the ordered disaster recovery checklist and lightweight prerequisite +checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge +verification. + Prometheus alerts : High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app. They cover node readiness, restart storms, unavailable deployments, storage diff --git a/jeannie b/jeannie index 3b16b28..58f4204 100755 --- a/jeannie +++ b/jeannie @@ -5967,6 +5967,11 @@ capacity_report() { "${REPO_ROOT}/scripts/capacity-report" } +recover_plan() { + require_debian_server "recover-plan" + "${REPO_ROOT}/scripts/recover-plan" +} + case "${1:-}" in up) up @@ -5989,6 +5994,9 @@ case "${1:-}" in capacity) capacity_report ;; + recover-plan) + recover_plan + ;; validate) validate_homelab ;; @@ -6133,7 +6141,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/recover-plan b/scripts/recover-plan new file mode 100755 index 0000000..4413566 --- /dev/null +++ b/scripts/recover-plan @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}" + +check() { + local label="$1" + shift + + printf '%-36s ' "${label}" + if "$@" >/tmp/homelab-recover-check.$$ 2>&1; then + printf 'ok\n' + else + printf 'check\n' + sed -n '1,3p' /tmp/homelab-recover-check.$$ | sed 's/^/ /' + fi + rm -f /tmp/homelab-recover-check.$$ +} + +has_gitea_backup() { + find "${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" -maxdepth 1 -type f -name 'gitea-*.zip' -print -quit | grep -q . +} + +has_state_backup() { + find "${HOMELAB_TOFU_STATE_BACKUP_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab/tofu-state-backups}" -maxdepth 1 -type f -name 'tofu-state-*.tgz' -print -quit | grep -q . +} + +main() { + cat <<'EOF' +== Homelab Recovery Order == + +1. Restore Debian control host baseline. + Command: ./jeannie validate + Check: Docker, containerd, Tailscale, kubeconfig path, repo checkout. + +2. Restore or verify Gitea. + Commands: + ./jeannie deploy-gitea + ./jeannie drill-gitea-restore + ./jeannie bootstrap-gitea-repo + +3. Restore RPi DNS services. + Commands: + ./jeannie rpi-services + ./jeannie doctor-rpi + ./jeannie drill-pihole-restore + +4. Verify Pimox template and worker storage. + Commands: + ./jeannie preflight + ./jeannie doctor-preapply + +5. Rebuild or start Kubernetes. + If state exists and nodes are stopped: ./jeannie start-cluster + If cluster must be rebuilt: ./jeannie rebuild-cluster + +6. Restore platform and apps from GitOps. + Commands: + ./jeannie apps + ./jeannie gitops-status + +7. Verify public edge. + Commands: + ./jeannie doctor-edge + ./jeannie cert-check + ./jeannie status + +8. Capture final recovery evidence. + Commands: + ./jeannie release-snapshot + ./jeannie backup-gitea + ./jeannie state-backup + +EOF + + echo "== Current Recovery Inputs ==" + check "repo validation" "${REPO_ROOT}/scripts/validate-homelab" + check "Gitea backup archive" has_gitea_backup + check "OpenTofu state backup" has_state_backup + check "Pi-hole restore inputs" "${REPO_ROOT}/scripts/restore-drill" pihole + check "Kubernetes API" kubectl --kubeconfig "${KUBECONFIG}" get --raw=/readyz + + cat <<'EOF' + +== Recovery Rule == +Prefer restoring source-of-truth first: repo, Gitea, DNS, state backups, then +rebuild compute. Treat Pimox worker disks as rebuildable unless a service has an +explicit backup and restore path. +EOF +} + +main "$@"