From f7cf1423f97c0903b75d8721e662a24067b5d562 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 17:08:16 -0600 Subject: [PATCH] Add restore drill checklist --- README.md | 6 ++ README.md.tmpl | 6 ++ docs/jeannie.1.md | 7 +++ jeannie | 17 +++++- scripts/restore-drill | 138 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100755 scripts/restore-drill diff --git a/README.md b/README.md index 857cfc3..298558d 100644 --- a/README.md +++ b/README.md @@ -750,8 +750,14 @@ Run the restore drill manually with: ```bash ./jeannie drill-gitea-restore +./jeannie drill-restore +./jeannie drill-pihole-restore ``` +`drill-restore` validates the latest Gitea backup archive, Pi-hole repo-managed +config inputs, the latest OpenTofu state backup archive when present, and the +GitOps rebuild path. It does not replace running services or mutate Kubernetes. + Useful checks: ```bash diff --git a/README.md.tmpl b/README.md.tmpl index 8fccc33..d7d661a 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -750,8 +750,14 @@ Run the restore drill manually with: ```bash ./{{ main_script }} drill-gitea-restore +./{{ main_script }} drill-restore +./{{ main_script }} drill-pihole-restore ``` +`drill-restore` validates the latest Gitea backup archive, Pi-hole repo-managed +config inputs, the latest OpenTofu state backup archive when present, and the +GitOps rebuild path. It does not replace running services or mutate Kubernetes. + Useful checks: ```bash diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 9569c95..9d45567 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -95,9 +95,16 @@ ready. `backup-gitea` : Run a Gitea backup on the Debian host. +`drill-restore` +: Run a read-only restore checklist for Gitea backups, Pi-hole repo-managed +config, OpenTofu state backup archives, and GitOps rebuild inputs. + `drill-gitea-restore` : Run the Gitea restore drill workflow. +`drill-pihole-restore` +: Validate that repo-managed Pi-hole restore inputs exist and are non-empty. + `install-gitea-runner [TOKEN]` : Install or repair the Gitea Actions runner. If `TOKEN` is omitted, the command uses the configured token source. diff --git a/jeannie b/jeannie index b9aaf7f..d8a57b2 100755 --- a/jeannie +++ b/jeannie @@ -5948,6 +5948,15 @@ access_audit() { "${REPO_ROOT}/scripts/access-audit" } +drill_restore() { + require_debian_server "drill-restore" + "${REPO_ROOT}/scripts/restore-drill" all +} + +drill_pihole_restore() { + "${REPO_ROOT}/scripts/restore-drill" pihole +} + case "${1:-}" in up) up @@ -6001,9 +6010,15 @@ case "${1:-}" in backup-gitea) backup_gitea ;; + drill-restore) + drill_restore + ;; drill-gitea-restore) drill_gitea_restore ;; + drill-pihole-restore) + drill_pihole_restore + ;; install-gitea-runner) install_gitea_runner "${2:-}" ;; @@ -6102,7 +6117,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|validate|access-audit|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|validate|access-audit|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/restore-drill b/scripts/restore-drill new file mode 100755 index 0000000..dfe9b43 --- /dev/null +++ b/scripts/restore-drill @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}" +KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}" + +section() { + printf '\n== %s ==\n' "$1" +} + +check_file() { + local label="$1" + local path="$2" + + printf '%-34s ' "${label}" + if [[ -s "${path}" ]]; then + printf 'ok - %s\n' "${path}" + else + printf 'fail - missing or empty: %s\n' "${path}" + return 1 + fi +} + +latest_file() { + local directory="$1" + local glob="$2" + + find "${directory}" -maxdepth 1 -type f -name "${glob}" -print0 2>/dev/null | + xargs -0 ls -t 2>/dev/null | + sed -n '1p' +} + +drill_gitea() { + local backup_dir="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" + local latest + + section "Gitea Restore Drill" + latest="$(latest_file "${backup_dir}" 'gitea-*.zip' || true)" + if [[ -z "${latest}" ]]; then + printf 'fail - no Gitea backup archive found in %s\n' "${backup_dir}" + return 1 + fi + + python3 - "${latest}" <<'PY' +import sys +import zipfile + +archive = sys.argv[1] +with zipfile.ZipFile(archive) as handle: + bad = handle.testzip() + if bad: + raise SystemExit(f"fail - corrupt archive member: {bad}") + members = handle.namelist() + if not members: + raise SystemExit("fail - archive is empty") +print(f"ok - archive valid: {archive} ({len(members)} members)") +PY +} + +drill_pihole() { + section "Pi-hole Restore Drill" + check_file "adlists" "${REPO_ROOT}/infra/rpi-services/adlists.txt" + check_file "local DNS records" "${REPO_ROOT}/infra/rpi-services/local-dns-records.txt" + check_file "CNAME records" "${REPO_ROOT}/infra/rpi-services/cname-records.txt" + check_file "static DHCP notes" "${REPO_ROOT}/infra/rpi-services/static-dhcp-hosts.txt" + check_file "RPi compose" "${REPO_ROOT}/infra/rpi-services/docker-compose.yml" + check_file "RPi bootstrap" "${REPO_ROOT}/infra/rpi-services/bootstrap.sh" +} + +drill_state() { + local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}" + local latest + + section "Jeannie/OpenTofu State Restore Drill" + latest="$(latest_file "${backup_dir}" 'tofu-state-*.tgz' || true)" + if [[ -z "${latest}" ]]; then + printf 'warn - no local OpenTofu state backup archive found in %s\n' "${backup_dir}" + printf 'hint - run ./jeannie state-backup after important changes\n' + return 0 + fi + tar -tzf "${latest}" >/dev/null + printf 'ok - state archive can be listed: %s\n' "${latest}" +} + +drill_gitops() { + section "GitOps Rebuild Drill" + git -C "${REPO_ROOT}" status --short + git -C "${REPO_ROOT}" rev-parse --verify HEAD >/dev/null + printf 'ok - repository HEAD: %s\n' "$(git -C "${REPO_ROOT}" rev-parse --short HEAD)" + + if command -v kubectl >/dev/null 2>&1 && [[ -s "${KUBECONFIG}" ]]; then + kubectl --kubeconfig "${KUBECONFIG}" -n argocd get applications.argoproj.io 2>/dev/null || true + else + printf 'info - kubeconfig unavailable; skipping Argo CD application list\n' + fi +} + +usage() { + cat <<'EOF' +Usage: scripts/restore-drill {all|gitea|pihole|state|gitops} + +Read-only restore drills. These checks prove that restore inputs exist and can +be inspected, but they do not replace running services or mutate Kubernetes. +EOF +} + +main() { + case "${1:-all}" in + all) + drill_gitea + drill_pihole + drill_state + drill_gitops + ;; + gitea) + drill_gitea + ;; + pihole) + drill_pihole + ;; + state) + drill_state + ;; + gitops) + drill_gitops + ;; + -h | --help | help) + usage + ;; + *) + usage >&2 + return 2 + ;; + esac +} + +main "$@"