Add backup verification status

README.md

@@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
 ./jeannie gitops-status
 ./jeannie cert-check
 ./jeannie release-snapshot
+./jeannie backup-status
 ./jeannie map
 ```
 
@@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
 state directory with Git state, Kubernetes/Argo CD/Helm state, workload images,
 public URL status, and pointers to the latest Gitea/OpenTofu backups.
 
+`backup-status` reports freshness for Gitea backups, OpenTofu state backups,
+restore drill reports, and repo-managed Pi-hole restore inputs. The main
+`status` cascade includes this as a warning signal.
+
 `map` prints a dependency map from the canonical inventory, covering public
 entry, GitOps automation, cluster foundation, DNS, observability, and security
 learning paths. `./jeannie map --dot` emits Graphviz DOT.
@@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes.
 Useful checks:
 
 ```bash
+./jeannie backup-status
 systemctl list-timers homelab-gitea-backup.timer
 systemctl list-timers homelab-gitea-restore-drill.timer
 sudo systemctl start homelab-gitea-backup.service

README.md.tmpl

@@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
 ./{{ main_script }} gitops-status
 ./{{ main_script }} cert-check
 ./{{ main_script }} release-snapshot
+./{{ main_script }} backup-status
 ./{{ main_script }} map
 ```
 
@@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
 state directory with Git state, Kubernetes/Argo CD/Helm state, workload images,
 public URL status, and pointers to the latest Gitea/OpenTofu backups.
 
+`backup-status` reports freshness for Gitea backups, OpenTofu state backups,
+restore drill reports, and repo-managed Pi-hole restore inputs. The main
+`status` cascade includes this as a warning signal.
+
 `map` prints a dependency map from the canonical inventory, covering public
 entry, GitOps automation, cluster foundation, DNS, observability, and security
 learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT.
@@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes.
 Useful checks:
 
 ```bash
+./{{ main_script }} backup-status
 systemctl list-timers homelab-gitea-backup.timer
 systemctl list-timers homelab-gitea-restore-drill.timer
 sudo systemctl start homelab-gitea-backup.service

docs/jeannie.1.md

@@ -76,6 +76,11 @@ and DuckDNS-to-OCI edge IP drift from the canonical inventory.
 state, workload images, public URL status, and latest Gitea/OpenTofu backup
 pointers.
 
+`backup-status`
+: Report freshness for Gitea backups, OpenTofu state backups, restore drill
+reports, and repo-managed Pi-hole restore inputs. The main `status` cascade
+includes this as a warning signal.
+
 `map`
 : Print a dependency map from the canonical inventory. Use `map --dot` for a
 Graphviz DOT graph.

jeannie

@@ -4941,6 +4941,7 @@ status_cascade_report() {
     status_cascade_warn "No problem pods" status_no_problem_pods
 
     status_section "What Broke Signals"
+    status_cascade_warn "Backup freshness" backup_status
     status_cascade_warn "Recent deployments healthy" status_recent_deployments
     status_cascade_warn "Pod restart pressure" status_pod_restart_pressure
     status_cascade_warn "Node pressure" status_node_pressure
@@ -5986,6 +5987,11 @@ release_snapshot() {
     "${REPO_ROOT}/scripts/release-snapshot"
 }
 
+backup_status() {
+    require_debian_server "backup-status"
+    "${REPO_ROOT}/scripts/backup-status"
+}
+
 homelab_map() {
     "${REPO_ROOT}/scripts/homelab-map" "${@:2}"
 }
@@ -6024,6 +6030,9 @@ case "${1:-}" in
     release-snapshot)
         release_snapshot
         ;;
+    backup-status)
+        backup_status
+        ;;
     map)
         homelab_map "$@"
         ;;
@@ -6171,7 +6180,7 @@ case "${1:-}" in
         echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     *)
-        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
+        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
         exit 1
         ;;
 esac

scripts/backup-status

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}"
+GITEA_BACKUP_DIR="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
+GITEA_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}"
+TOFU_BACKUP_DIR="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}"
+PIHOLE_CONFIG_FILES="${PIHOLE_CONFIG_FILES:-infra/rpi-services/adlists.txt infra/rpi-services/local-dns-records.txt infra/rpi-services/cname-records.txt infra/rpi-services/static-dhcp-hosts.txt}"
+MAX_GITEA_AGE_HOURS="${LAB_BACKUP_MAX_GITEA_AGE_HOURS:-36}"
+MAX_TOFU_AGE_HOURS="${LAB_BACKUP_MAX_TOFU_AGE_HOURS:-168}"
+MAX_DRILL_AGE_HOURS="${LAB_BACKUP_MAX_DRILL_AGE_HOURS:-744}"
+
+failures=0
+warnings=0
+
+section() {
+    printf '\n== %s ==\n' "$1"
+}
+
+latest_file() {
+    local dir="$1"
+    local pattern="$2"
+
+    if [ -d "$dir" ]; then
+        find "$dir" -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' 2>/dev/null |
+            sort -n |
+            tail -1 |
+            sed 's/^[^ ]* //'
+    fi
+}
+
+file_age_hours() {
+    local path="$1"
+    python3 - "$path" <<'PY'
+import os
+import sys
+import time
+
+path = sys.argv[1]
+age = int((time.time() - os.path.getmtime(path)) / 3600)
+print(age)
+PY
+}
+
+check_latest() {
+    local label="$1"
+    local dir="$2"
+    local pattern="$3"
+    local max_age_hours="$4"
+    local severity="${5:-fail}"
+    local latest
+    local age_hours
+
+    latest="$(latest_file "$dir" "$pattern" || true)"
+    printf '%-30s ' "$label"
+    if [ -z "$latest" ]; then
+        printf '%s - missing in %s\n' "$severity" "$dir"
+        if [ "$severity" = "fail" ]; then
+            failures=$((failures + 1))
+        else
+            warnings=$((warnings + 1))
+        fi
+        return 0
+    fi
+
+    age_hours="$(file_age_hours "$latest")"
+    if [ "$age_hours" -le "$max_age_hours" ]; then
+        printf 'ok - age=%sh path=%s\n' "$age_hours" "$latest"
+    else
+        printf '%s - age=%sh max=%sh path=%s\n' "$severity" "$age_hours" "$max_age_hours" "$latest"
+        if [ "$severity" = "fail" ]; then
+            failures=$((failures + 1))
+        else
+            warnings=$((warnings + 1))
+        fi
+    fi
+}
+
+section "Backup Freshness"
+check_latest "Gitea backup" "$GITEA_BACKUP_DIR" 'gitea-*.zip' "$MAX_GITEA_AGE_HOURS" fail
+check_latest "OpenTofu state backup" "$TOFU_BACKUP_DIR" 'tofu-state-*.tgz' "$MAX_TOFU_AGE_HOURS" warn
+check_latest "Restore drill report" "$GITEA_DRILL_DIR" 'restore-drill-*.txt' "$MAX_DRILL_AGE_HOURS" warn
+
+section "Repo Managed Pi-hole Inputs"
+for file in $PIHOLE_CONFIG_FILES; do
+    printf '%-30s ' "$file"
+    if [ -s "$file" ]; then
+        printf 'ok\n'
+    else
+        printf 'warn - missing or empty\n'
+        warnings=$((warnings + 1))
+    fi
+done
+
+section "Summary"
+printf 'failures=%s warnings=%s\n' "$failures" "$warnings"
+
+if [ "$failures" -gt 0 ]; then
+    exit 1
+fi