Add power loss recovery mode

This commit is contained in:
juvdiaz 2026-06-29 17:52:41 -06:00
parent 479be7fc47
commit 7281143253
5 changed files with 74 additions and 1 deletions

View File

@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
./jeannie status
./jeannie capacity
./jeannie recover-plan
./jeannie recover-power --dry-run
./jeannie gitops-status
./jeannie cert-check
./jeannie release-snapshot
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
edge verification.
`recover-power` runs the post-outage recovery sequence in dependency order:
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
`gitops-status` prints a focused Argo CD view: application sync and health,
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors.

View File

@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
./{{ main_script }} status
./{{ main_script }} capacity
./{{ main_script }} recover-plan
./{{ main_script }} recover-power --dry-run
./{{ main_script }} gitops-status
./{{ main_script }} cert-check
./{{ main_script }} release-snapshot
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
edge verification.
`recover-power` runs the post-outage recovery sequence in dependency order:
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
`gitops-status` prints a focused Argo CD view: application sync and health,
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors.

View File

@ -62,6 +62,10 @@ state, and practical placement guidance.
checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge
verification.
`recover-power [--dry-run]`
: Run the post-outage recovery sequence in dependency order: Debian runtime,
Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks.
`gitops-status`
: Print a focused Argo CD status view with application sync/health, out-of-sync
or degraded apps, repository secret presence, recent events, and recent

11
jeannie
View File

@ -6016,6 +6016,12 @@ route_inventory() {
"${REPO_ROOT}/scripts/route-inventory"
}
recover_power() {
require_debian_server "recover-power"
record_change_journal "recover-power" "$@"
"${REPO_ROOT}/scripts/recover-power" "${@:2}"
}
workers_manage() {
require_debian_server "workers"
"${REPO_ROOT}/scripts/workers" "${@:2}"
@ -6093,6 +6099,9 @@ case "${1:-}" in
route-inventory)
route_inventory
;;
recover-power)
recover_power "$@"
;;
workers)
workers_manage "$@"
;;
@ -6250,7 +6259,7 @@ case "${1:-}" in
echo "Log: ${JEANNIE_LOG_FILE}"
;;
*)
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
exit 1
;;
esac

50
scripts/recover-power Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DRY_RUN=false
if [ "${1:-}" = "--dry-run" ]; then
DRY_RUN=true
fi
run_step() {
local label="$1"
shift
printf '\n== %s ==\n' "$label"
printf '+ %s\n' "$*"
if [ "$DRY_RUN" = "true" ]; then
return 0
fi
"$@"
}
run_optional() {
local label="$1"
shift
if ! run_step "$label" "$@"; then
printf 'warn: step failed: %s\n' "$label" >&2
fi
}
cat <<EOF
Power-loss recovery order
=========================
This sequence restores dependencies before dependents:
Debian runtime -> Gitea -> RPi DNS -> Pimox workers -> Kubernetes -> GitOps/apps -> edge/public URLs.
EOF
run_optional "Debian runtime status" systemctl is-active docker containerd
run_optional "Start Docker runtime" sudo systemctl start docker containerd
run_optional "Deploy or verify Gitea" "${REPO_ROOT}/jeannie" deploy-gitea
run_optional "Verify RPi services" "${REPO_ROOT}/jeannie" doctor-rpi
run_optional "Start Kubernetes cluster" "${REPO_ROOT}/jeannie" start-cluster
run_optional "Cluster doctor" "${REPO_ROOT}/jeannie" doctor-cluster
run_optional "GitOps status" "${REPO_ROOT}/jeannie" gitops-status
run_optional "Public edge doctor" "${REPO_ROOT}/jeannie" doctor-edge
run_optional "Final status cascade" "${REPO_ROOT}/jeannie" status
printf '\nRecovery sequence complete%s.\n' "$([ "$DRY_RUN" = "true" ] && printf ' dry-run' || true)"