diff --git a/README.md b/README.md index ab70398..7b2505c 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie gitops-status ./jeannie cert-check ./jeannie release-snapshot +./jeannie backup-status ./jeannie map ``` @@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory. state directory with Git state, Kubernetes/Argo CD/Helm state, workload images, public URL status, and pointers to the latest Gitea/OpenTofu backups. +`backup-status` reports freshness for Gitea backups, OpenTofu state backups, +restore drill reports, and repo-managed Pi-hole restore inputs. The main +`status` cascade includes this as a warning signal. + `map` prints a dependency map from the canonical inventory, covering public entry, GitOps automation, cluster foundation, DNS, observability, and security learning paths. `./jeannie map --dot` emits Graphviz DOT. @@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes. Useful checks: ```bash +./jeannie backup-status systemctl list-timers homelab-gitea-backup.timer systemctl list-timers homelab-gitea-restore-drill.timer sudo systemctl start homelab-gitea-backup.service diff --git a/README.md.tmpl b/README.md.tmpl index 942faa9..b8b6ffd 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} gitops-status ./{{ main_script }} cert-check ./{{ main_script }} release-snapshot +./{{ main_script }} backup-status ./{{ main_script }} map ``` @@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory. state directory with Git state, Kubernetes/Argo CD/Helm state, workload images, public URL status, and pointers to the latest Gitea/OpenTofu backups. +`backup-status` reports freshness for Gitea backups, OpenTofu state backups, +restore drill reports, and repo-managed Pi-hole restore inputs. The main +`status` cascade includes this as a warning signal. + `map` prints a dependency map from the canonical inventory, covering public entry, GitOps automation, cluster foundation, DNS, observability, and security learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT. @@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes. Useful checks: ```bash +./{{ main_script }} backup-status systemctl list-timers homelab-gitea-backup.timer systemctl list-timers homelab-gitea-restore-drill.timer sudo systemctl start homelab-gitea-backup.service diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index bc4c957..f03672b 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -76,6 +76,11 @@ and DuckDNS-to-OCI edge IP drift from the canonical inventory. state, workload images, public URL status, and latest Gitea/OpenTofu backup pointers. +`backup-status` +: Report freshness for Gitea backups, OpenTofu state backups, restore drill +reports, and repo-managed Pi-hole restore inputs. The main `status` cascade +includes this as a warning signal. + `map` : Print a dependency map from the canonical inventory. Use `map --dot` for a Graphviz DOT graph. diff --git a/jeannie b/jeannie index ccd2cda..e166629 100755 --- a/jeannie +++ b/jeannie @@ -4941,6 +4941,7 @@ status_cascade_report() { status_cascade_warn "No problem pods" status_no_problem_pods status_section "What Broke Signals" + status_cascade_warn "Backup freshness" backup_status status_cascade_warn "Recent deployments healthy" status_recent_deployments status_cascade_warn "Pod restart pressure" status_pod_restart_pressure status_cascade_warn "Node pressure" status_node_pressure @@ -5986,6 +5987,11 @@ release_snapshot() { "${REPO_ROOT}/scripts/release-snapshot" } +backup_status() { + require_debian_server "backup-status" + "${REPO_ROOT}/scripts/backup-status" +} + homelab_map() { "${REPO_ROOT}/scripts/homelab-map" "${@:2}" } @@ -6024,6 +6030,9 @@ case "${1:-}" in release-snapshot) release_snapshot ;; + backup-status) + backup_status + ;; map) homelab_map "$@" ;; @@ -6171,7 +6180,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/backup-status b/scripts/backup-status new file mode 100755 index 0000000..4a50553 --- /dev/null +++ b/scripts/backup-status @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}" +GITEA_BACKUP_DIR="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" +GITEA_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}" +TOFU_BACKUP_DIR="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}" +PIHOLE_CONFIG_FILES="${PIHOLE_CONFIG_FILES:-infra/rpi-services/adlists.txt infra/rpi-services/local-dns-records.txt infra/rpi-services/cname-records.txt infra/rpi-services/static-dhcp-hosts.txt}" +MAX_GITEA_AGE_HOURS="${LAB_BACKUP_MAX_GITEA_AGE_HOURS:-36}" +MAX_TOFU_AGE_HOURS="${LAB_BACKUP_MAX_TOFU_AGE_HOURS:-168}" +MAX_DRILL_AGE_HOURS="${LAB_BACKUP_MAX_DRILL_AGE_HOURS:-744}" + +failures=0 +warnings=0 + +section() { + printf '\n== %s ==\n' "$1" +} + +latest_file() { + local dir="$1" + local pattern="$2" + + if [ -d "$dir" ]; then + find "$dir" -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' 2>/dev/null | + sort -n | + tail -1 | + sed 's/^[^ ]* //' + fi +} + +file_age_hours() { + local path="$1" + python3 - "$path" <<'PY' +import os +import sys +import time + +path = sys.argv[1] +age = int((time.time() - os.path.getmtime(path)) / 3600) +print(age) +PY +} + +check_latest() { + local label="$1" + local dir="$2" + local pattern="$3" + local max_age_hours="$4" + local severity="${5:-fail}" + local latest + local age_hours + + latest="$(latest_file "$dir" "$pattern" || true)" + printf '%-30s ' "$label" + if [ -z "$latest" ]; then + printf '%s - missing in %s\n' "$severity" "$dir" + if [ "$severity" = "fail" ]; then + failures=$((failures + 1)) + else + warnings=$((warnings + 1)) + fi + return 0 + fi + + age_hours="$(file_age_hours "$latest")" + if [ "$age_hours" -le "$max_age_hours" ]; then + printf 'ok - age=%sh path=%s\n' "$age_hours" "$latest" + else + printf '%s - age=%sh max=%sh path=%s\n' "$severity" "$age_hours" "$max_age_hours" "$latest" + if [ "$severity" = "fail" ]; then + failures=$((failures + 1)) + else + warnings=$((warnings + 1)) + fi + fi +} + +section "Backup Freshness" +check_latest "Gitea backup" "$GITEA_BACKUP_DIR" 'gitea-*.zip' "$MAX_GITEA_AGE_HOURS" fail +check_latest "OpenTofu state backup" "$TOFU_BACKUP_DIR" 'tofu-state-*.tgz' "$MAX_TOFU_AGE_HOURS" warn +check_latest "Restore drill report" "$GITEA_DRILL_DIR" 'restore-drill-*.txt' "$MAX_DRILL_AGE_HOURS" warn + +section "Repo Managed Pi-hole Inputs" +for file in $PIHOLE_CONFIG_FILES; do + printf '%-30s ' "$file" + if [ -s "$file" ]; then + printf 'ok\n' + else + printf 'warn - missing or empty\n' + warnings=$((warnings + 1)) + fi +done + +section "Summary" +printf 'failures=%s warnings=%s\n' "$failures" "$warnings" + +if [ "$failures" -gt 0 ]; then + exit 1 +fi