Add power loss recovery mode

This commit is contained in:
juvdiaz 2026-06-29 17:52:41 -06:00
parent 479be7fc47
commit 7281143253
5 changed files with 74 additions and 1 deletions

View File

@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
./jeannie status ./jeannie status
./jeannie capacity ./jeannie capacity
./jeannie recover-plan ./jeannie recover-plan
./jeannie recover-power --dry-run
./jeannie gitops-status ./jeannie gitops-status
./jeannie cert-check ./jeannie cert-check
./jeannie release-snapshot ./jeannie release-snapshot
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
edge verification. edge verification.
`recover-power` runs the post-outage recovery sequence in dependency order:
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
`gitops-status` prints a focused Argo CD view: application sync and health, `gitops-status` prints a focused Argo CD view: application sync and health,
out-of-sync or degraded apps, repository secret presence, recent Argo CD events, out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors. and recent repo-server/application-controller errors.

View File

@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
./{{ main_script }} status ./{{ main_script }} status
./{{ main_script }} capacity ./{{ main_script }} capacity
./{{ main_script }} recover-plan ./{{ main_script }} recover-plan
./{{ main_script }} recover-power --dry-run
./{{ main_script }} gitops-status ./{{ main_script }} gitops-status
./{{ main_script }} cert-check ./{{ main_script }} cert-check
./{{ main_script }} release-snapshot ./{{ main_script }} release-snapshot
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
edge verification. edge verification.
`recover-power` runs the post-outage recovery sequence in dependency order:
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
`gitops-status` prints a focused Argo CD view: application sync and health, `gitops-status` prints a focused Argo CD view: application sync and health,
out-of-sync or degraded apps, repository secret presence, recent Argo CD events, out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors. and recent repo-server/application-controller errors.

View File

@ -62,6 +62,10 @@ state, and practical placement guidance.
checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge
verification. verification.
`recover-power [--dry-run]`
: Run the post-outage recovery sequence in dependency order: Debian runtime,
Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks.
`gitops-status` `gitops-status`
: Print a focused Argo CD status view with application sync/health, out-of-sync : Print a focused Argo CD status view with application sync/health, out-of-sync
or degraded apps, repository secret presence, recent events, and recent or degraded apps, repository secret presence, recent events, and recent

11
jeannie
View File

@ -6016,6 +6016,12 @@ route_inventory() {
"${REPO_ROOT}/scripts/route-inventory" "${REPO_ROOT}/scripts/route-inventory"
} }
recover_power() {
require_debian_server "recover-power"
record_change_journal "recover-power" "$@"
"${REPO_ROOT}/scripts/recover-power" "${@:2}"
}
workers_manage() { workers_manage() {
require_debian_server "workers" require_debian_server "workers"
"${REPO_ROOT}/scripts/workers" "${@:2}" "${REPO_ROOT}/scripts/workers" "${@:2}"
@ -6093,6 +6099,9 @@ case "${1:-}" in
route-inventory) route-inventory)
route_inventory route_inventory
;; ;;
recover-power)
recover_power "$@"
;;
workers) workers)
workers_manage "$@" workers_manage "$@"
;; ;;
@ -6250,7 +6259,7 @@ case "${1:-}" in
echo "Log: ${JEANNIE_LOG_FILE}" echo "Log: ${JEANNIE_LOG_FILE}"
;; ;;
*) *)
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
exit 1 exit 1
;; ;;
esac esac

50
scripts/recover-power Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DRY_RUN=false
if [ "${1:-}" = "--dry-run" ]; then
DRY_RUN=true
fi
run_step() {
local label="$1"
shift
printf '\n== %s ==\n' "$label"
printf '+ %s\n' "$*"
if [ "$DRY_RUN" = "true" ]; then
return 0
fi
"$@"
}
run_optional() {
local label="$1"
shift
if ! run_step "$label" "$@"; then
printf 'warn: step failed: %s\n' "$label" >&2
fi
}
cat <<EOF
Power-loss recovery order
=========================
This sequence restores dependencies before dependents:
Debian runtime -> Gitea -> RPi DNS -> Pimox workers -> Kubernetes -> GitOps/apps -> edge/public URLs.
EOF
run_optional "Debian runtime status" systemctl is-active docker containerd
run_optional "Start Docker runtime" sudo systemctl start docker containerd
run_optional "Deploy or verify Gitea" "${REPO_ROOT}/jeannie" deploy-gitea
run_optional "Verify RPi services" "${REPO_ROOT}/jeannie" doctor-rpi
run_optional "Start Kubernetes cluster" "${REPO_ROOT}/jeannie" start-cluster
run_optional "Cluster doctor" "${REPO_ROOT}/jeannie" doctor-cluster
run_optional "GitOps status" "${REPO_ROOT}/jeannie" gitops-status
run_optional "Public edge doctor" "${REPO_ROOT}/jeannie" doctor-edge
run_optional "Final status cascade" "${REPO_ROOT}/jeannie" status
printf '\nRecovery sequence complete%s.\n' "$([ "$DRY_RUN" = "true" ] && printf ' dry-run' || true)"