Add power loss recovery mode
This commit is contained in:
parent
479be7fc47
commit
7281143253
|
|
@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
./jeannie status
|
./jeannie status
|
||||||
./jeannie capacity
|
./jeannie capacity
|
||||||
./jeannie recover-plan
|
./jeannie recover-plan
|
||||||
|
./jeannie recover-power --dry-run
|
||||||
./jeannie gitops-status
|
./jeannie gitops-status
|
||||||
./jeannie cert-check
|
./jeannie cert-check
|
||||||
./jeannie release-snapshot
|
./jeannie release-snapshot
|
||||||
|
|
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
|
||||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||||
edge verification.
|
edge verification.
|
||||||
|
|
||||||
|
`recover-power` runs the post-outage recovery sequence in dependency order:
|
||||||
|
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
|
||||||
|
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
|
||||||
|
|
||||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
and recent repo-server/application-controller errors.
|
and recent repo-server/application-controller errors.
|
||||||
|
|
|
||||||
|
|
@ -272,6 +272,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
./{{ main_script }} status
|
./{{ main_script }} status
|
||||||
./{{ main_script }} capacity
|
./{{ main_script }} capacity
|
||||||
./{{ main_script }} recover-plan
|
./{{ main_script }} recover-plan
|
||||||
|
./{{ main_script }} recover-power --dry-run
|
||||||
./{{ main_script }} gitops-status
|
./{{ main_script }} gitops-status
|
||||||
./{{ main_script }} cert-check
|
./{{ main_script }} cert-check
|
||||||
./{{ main_script }} release-snapshot
|
./{{ main_script }} release-snapshot
|
||||||
|
|
@ -312,6 +313,10 @@ guidance for deciding what can safely run next.
|
||||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||||
edge verification.
|
edge verification.
|
||||||
|
|
||||||
|
`recover-power` runs the post-outage recovery sequence in dependency order:
|
||||||
|
Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and
|
||||||
|
edge/public checks. Use `recover-power --dry-run` to preview the sequence.
|
||||||
|
|
||||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
and recent repo-server/application-controller errors.
|
and recent repo-server/application-controller errors.
|
||||||
|
|
|
||||||
|
|
@ -62,6 +62,10 @@ state, and practical placement guidance.
|
||||||
checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge
|
checks for Debian, Gitea, RPi DNS, Pimox, Kubernetes, GitOps apps, and edge
|
||||||
verification.
|
verification.
|
||||||
|
|
||||||
|
`recover-power [--dry-run]`
|
||||||
|
: Run the post-outage recovery sequence in dependency order: Debian runtime,
|
||||||
|
Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks.
|
||||||
|
|
||||||
`gitops-status`
|
`gitops-status`
|
||||||
: Print a focused Argo CD status view with application sync/health, out-of-sync
|
: Print a focused Argo CD status view with application sync/health, out-of-sync
|
||||||
or degraded apps, repository secret presence, recent events, and recent
|
or degraded apps, repository secret presence, recent events, and recent
|
||||||
|
|
|
||||||
11
jeannie
11
jeannie
|
|
@ -6016,6 +6016,12 @@ route_inventory() {
|
||||||
"${REPO_ROOT}/scripts/route-inventory"
|
"${REPO_ROOT}/scripts/route-inventory"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recover_power() {
|
||||||
|
require_debian_server "recover-power"
|
||||||
|
record_change_journal "recover-power" "$@"
|
||||||
|
"${REPO_ROOT}/scripts/recover-power" "${@:2}"
|
||||||
|
}
|
||||||
|
|
||||||
workers_manage() {
|
workers_manage() {
|
||||||
require_debian_server "workers"
|
require_debian_server "workers"
|
||||||
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
||||||
|
|
@ -6093,6 +6099,9 @@ case "${1:-}" in
|
||||||
route-inventory)
|
route-inventory)
|
||||||
route_inventory
|
route_inventory
|
||||||
;;
|
;;
|
||||||
|
recover-power)
|
||||||
|
recover_power "$@"
|
||||||
|
;;
|
||||||
workers)
|
workers)
|
||||||
workers_manage "$@"
|
workers_manage "$@"
|
||||||
;;
|
;;
|
||||||
|
|
@ -6250,7 +6259,7 @@ case "${1:-}" in
|
||||||
echo "Log: ${JEANNIE_LOG_FILE}"
|
echo "Log: ${JEANNIE_LOG_FILE}"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
DRY_RUN=false
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--dry-run" ]; then
|
||||||
|
DRY_RUN=true
|
||||||
|
fi
|
||||||
|
|
||||||
|
run_step() {
|
||||||
|
local label="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
printf '\n== %s ==\n' "$label"
|
||||||
|
printf '+ %s\n' "$*"
|
||||||
|
if [ "$DRY_RUN" = "true" ]; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"$@"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_optional() {
|
||||||
|
local label="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
if ! run_step "$label" "$@"; then
|
||||||
|
printf 'warn: step failed: %s\n' "$label" >&2
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cat <<EOF
|
||||||
|
Power-loss recovery order
|
||||||
|
=========================
|
||||||
|
|
||||||
|
This sequence restores dependencies before dependents:
|
||||||
|
Debian runtime -> Gitea -> RPi DNS -> Pimox workers -> Kubernetes -> GitOps/apps -> edge/public URLs.
|
||||||
|
EOF
|
||||||
|
|
||||||
|
run_optional "Debian runtime status" systemctl is-active docker containerd
|
||||||
|
run_optional "Start Docker runtime" sudo systemctl start docker containerd
|
||||||
|
run_optional "Deploy or verify Gitea" "${REPO_ROOT}/jeannie" deploy-gitea
|
||||||
|
run_optional "Verify RPi services" "${REPO_ROOT}/jeannie" doctor-rpi
|
||||||
|
run_optional "Start Kubernetes cluster" "${REPO_ROOT}/jeannie" start-cluster
|
||||||
|
run_optional "Cluster doctor" "${REPO_ROOT}/jeannie" doctor-cluster
|
||||||
|
run_optional "GitOps status" "${REPO_ROOT}/jeannie" gitops-status
|
||||||
|
run_optional "Public edge doctor" "${REPO_ROOT}/jeannie" doctor-edge
|
||||||
|
run_optional "Final status cascade" "${REPO_ROOT}/jeannie" status
|
||||||
|
|
||||||
|
printf '\nRecovery sequence complete%s.\n' "$([ "$DRY_RUN" = "true" ] && printf ' dry-run' || true)"
|
||||||
Loading…
Reference in New Issue