From 7281143253ef2817eb9b36264d9bd3597490a7a4 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 17:52:41 -0600 Subject: [PATCH] Add power loss recovery mode --- README.md | 5 +++++ README.md.tmpl | 5 +++++ docs/jeannie.1.md | 4 ++++ jeannie | 11 +++++++++- scripts/recover-power | 50 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) create mode 100755 scripts/recover-power diff --git a/README.md b/README.md index f9f3f69..2d17198 100644 --- a/README.md +++ b/README.md @@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie status ./jeannie capacity ./jeannie recover-plan +./jeannie recover-power --dry-run ./jeannie gitops-status ./jeannie cert-check ./jeannie release-snapshot @@ -312,6 +313,10 @@ guidance for deciding what can safely run next. checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and edge verification. +`recover-power` runs the post-outage recovery sequence in dependency order: +Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and +edge/public checks. Use `recover-power --dry-run` to preview the sequence. + `gitops-status` prints a focused Argo CD view: application sync and health, out-of-sync or degraded apps, repository secret presence, recent Argo CD events, and recent repo-server/application-controller errors. diff --git a/README.md.tmpl b/README.md.tmpl index 7913cb6..4e5ace8 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} status ./{{ main_script }} capacity ./{{ main_script }} recover-plan +./{{ main_script }} recover-power --dry-run ./{{ main_script }} gitops-status ./{{ main_script }} cert-check ./{{ main_script }} release-snapshot @@ -312,6 +313,10 @@ guidance for deciding what can safely run next. checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and edge verification. +`recover-power` runs the post-outage recovery sequence in dependency order: +Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and +edge/public checks. Use `recover-power --dry-run` to preview the sequence. + `gitops-status` prints a focused Argo CD view: application sync and health, out-of-sync or degraded apps, repository secret presence, recent Argo CD events, and recent repo-server/application-controller errors. diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index e64f90f..663ead2 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -62,6 +62,10 @@ state, and practical placement guidance. checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge verification. +`recover-power [--dry-run]` +: Run the post-outage recovery sequence in dependency order: Debian runtime, +Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks. + `gitops-status` : Print a focused Argo CD status view with application sync/health, out-of-sync or degraded apps, repository secret presence, recent events, and recent diff --git a/jeannie b/jeannie index 4103b06..6368a91 100755 --- a/jeannie +++ b/jeannie @@ -6016,6 +6016,12 @@ route_inventory() { "${REPO_ROOT}/scripts/route-inventory" } +recover_power() { + require_debian_server "recover-power" + record_change_journal "recover-power" "$@" + "${REPO_ROOT}/scripts/recover-power" "${@:2}" +} + workers_manage() { require_debian_server "workers" "${REPO_ROOT}/scripts/workers" "${@:2}" @@ -6093,6 +6099,9 @@ case "${1:-}" in route-inventory) route_inventory ;; + recover-power) + recover_power "$@" + ;; workers) workers_manage "$@" ;; @@ -6250,7 +6259,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/recover-power b/scripts/recover-power new file mode 100755 index 0000000..920893e --- /dev/null +++ b/scripts/recover-power @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DRY_RUN=false + +if [ "${1:-}" = "--dry-run" ]; then + DRY_RUN=true +fi + +run_step() { + local label="$1" + shift + + printf '\n== %s ==\n' "$label" + printf '+ %s\n' "$*" + if [ "$DRY_RUN" = "true" ]; then + return 0 + fi + "$@" +} + +run_optional() { + local label="$1" + shift + + if ! run_step "$label" "$@"; then + printf 'warn: step failed: %s\n' "$label" >&2 + fi +} + +cat < Gitea -> RPi DNS -> Pimox workers -> Kubernetes -> GitOps/apps -> edge/public URLs. +EOF + +run_optional "Debian runtime status" systemctl is-active docker containerd +run_optional "Start Docker runtime" sudo systemctl start docker containerd +run_optional "Deploy or verify Gitea" "${REPO_ROOT}/jeannie" deploy-gitea +run_optional "Verify RPi services" "${REPO_ROOT}/jeannie" doctor-rpi +run_optional "Start Kubernetes cluster" "${REPO_ROOT}/jeannie" start-cluster +run_optional "Cluster doctor" "${REPO_ROOT}/jeannie" doctor-cluster +run_optional "GitOps status" "${REPO_ROOT}/jeannie" gitops-status +run_optional "Public edge doctor" "${REPO_ROOT}/jeannie" doctor-edge +run_optional "Final status cascade" "${REPO_ROOT}/jeannie" status + +printf '\nRecovery sequence complete%s.\n' "$([ "$DRY_RUN" = "true" ] && printf ' dry-run' || true)"