jv
/
my-homelab-configs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add backup verification status

This commit is contained in:
juvdiaz 2026-06-29 17:38:53 -06:00
parent 7efa7a0452
commit bfe4627c56
5 changed files with 127 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
./jeannie gitops-status
./jeannie cert-check
./jeannie release-snapshot
./jeannie backup-status
./jeannie map
```
@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
state directory with Git state, Kubernetes/Argo CD/Helm state, workload images,
public URL status, and pointers to the latest Gitea/OpenTofu backups.
`backup-status` reports freshness for Gitea backups, OpenTofu state backups,
restore drill reports, and repo-managed Pi-hole restore inputs. The main
`status` cascade includes this as a warning signal.
`map` prints a dependency map from the canonical inventory, covering public
entry, GitOps automation, cluster foundation, DNS, observability, and security
learning paths. `./jeannie map --dot` emits Graphviz DOT.
@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes.
Useful checks:
```bash
./jeannie backup-status
systemctl list-timers homelab-gitea-backup.timer
systemctl list-timers homelab-gitea-restore-drill.timer
sudo systemctl start homelab-gitea-backup.service

6
README.md.tmpl
View File

@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
./{{ main_script }} gitops-status
./{{ main_script }} cert-check
./{{ main_script }} release-snapshot
./{{ main_script }} backup-status
./{{ main_script }} map
```
@ -315,6 +316,10 @@ HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
state directory with Git state, Kubernetes/Argo CD/Helm state, workload images,
public URL status, and pointers to the latest Gitea/OpenTofu backups.
`backup-status` reports freshness for Gitea backups, OpenTofu state backups,
restore drill reports, and repo-managed Pi-hole restore inputs. The main
`status` cascade includes this as a warning signal.
`map` prints a dependency map from the canonical inventory, covering public
entry, GitOps automation, cluster foundation, DNS, observability, and security
learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT.
@ -797,6 +802,7 @@ GitOps rebuild path. It does not replace running services or mutate Kubernetes.
Useful checks:
```bash
./{{ main_script }} backup-status
systemctl list-timers homelab-gitea-backup.timer
systemctl list-timers homelab-gitea-restore-drill.timer
sudo systemctl start homelab-gitea-backup.service

5
docs/jeannie.1.md
View File

@ -76,6 +76,11 @@ and DuckDNS-to-OCI edge IP drift from the canonical inventory.
state, workload images, public URL status, and latest Gitea/OpenTofu backup
pointers.
`backup-status`
: Report freshness for Gitea backups, OpenTofu state backups, restore drill
reports, and repo-managed Pi-hole restore inputs. The main `status` cascade
includes this as a warning signal.
`map`
: Print a dependency map from the canonical inventory. Use `map --dot` for a
Graphviz DOT graph.

11
jeannie
View File

@ -4941,6 +4941,7 @@ status_cascade_report() {
status_cascade_warn "No problem pods" status_no_problem_pods
status_section "What Broke Signals"
status_cascade_warn "Backup freshness" backup_status
status_cascade_warn "Recent deployments healthy" status_recent_deployments
status_cascade_warn "Pod restart pressure" status_pod_restart_pressure
status_cascade_warn "Node pressure" status_node_pressure
@ -5986,6 +5987,11 @@ release_snapshot() {
"${REPO_ROOT}/scripts/release-snapshot"
}
backup_status() {
require_debian_server "backup-status"
"${REPO_ROOT}/scripts/backup-status"
}
homelab_map() {
"${REPO_ROOT}/scripts/homelab-map" "${@:2}"
}
@ -6024,6 +6030,9 @@ case "${1:-}" in
release-snapshot)
release_snapshot
;;
backup-status)
backup_status
;;
map)
homelab_map "$@"
;;
@ -6171,7 +6180,7 @@ case "${1:-}" in
echo "Log: ${JEANNIE_LOG_FILE}"
;;
*)
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
exit 1
;;
esac

100
scripts/backup-status Executable file
View File

@ -0,0 +1,100 @@
#!/usr/bin/env bash
set -euo pipefail
HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}"
GITEA_BACKUP_DIR="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
GITEA_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}"
TOFU_BACKUP_DIR="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}"
PIHOLE_CONFIG_FILES="${PIHOLE_CONFIG_FILES:-infra/rpi-services/adlists.txt infra/rpi-services/local-dns-records.txt infra/rpi-services/cname-records.txt infra/rpi-services/static-dhcp-hosts.txt}"
MAX_GITEA_AGE_HOURS="${LAB_BACKUP_MAX_GITEA_AGE_HOURS:-36}"
MAX_TOFU_AGE_HOURS="${LAB_BACKUP_MAX_TOFU_AGE_HOURS:-168}"
MAX_DRILL_AGE_HOURS="${LAB_BACKUP_MAX_DRILL_AGE_HOURS:-744}"
failures=0
warnings=0
section() {
printf '\n== %s ==\n' "$1"
}
latest_file() {
local dir="$1"
local pattern="$2"
if [ -d "$dir" ]; then
find "$dir" -maxdepth 1 -type f -name "$pattern" -printf '%T@ %p\n' 2>/dev/null |
sort -n |
tail -1 |
sed 's/^[^ ]* //'
fi
}
file_age_hours() {
local path="$1"
python3 - "$path" <<'PY'
import os
import sys
import time
path = sys.argv[1]
age = int((time.time() - os.path.getmtime(path)) / 3600)
print(age)
PY
}
check_latest() {
local label="$1"
local dir="$2"
local pattern="$3"
local max_age_hours="$4"
local severity="${5:-fail}"
local latest
local age_hours
latest="$(latest_file "$dir" "$pattern" || true)"
printf '%-30s ' "$label"
if [ -z "$latest" ]; then
printf '%s - missing in %s\n' "$severity" "$dir"
if [ "$severity" = "fail" ]; then
failures=$((failures + 1))
else
warnings=$((warnings + 1))
fi
return 0
fi
age_hours="$(file_age_hours "$latest")"
if [ "$age_hours" -le "$max_age_hours" ]; then
printf 'ok - age=%sh path=%s\n' "$age_hours" "$latest"
else
printf '%s - age=%sh max=%sh path=%s\n' "$severity" "$age_hours" "$max_age_hours" "$latest"
if [ "$severity" = "fail" ]; then
failures=$((failures + 1))
else
warnings=$((warnings + 1))
fi
fi
}
section "Backup Freshness"
check_latest "Gitea backup" "$GITEA_BACKUP_DIR" 'gitea-*.zip' "$MAX_GITEA_AGE_HOURS" fail
check_latest "OpenTofu state backup" "$TOFU_BACKUP_DIR" 'tofu-state-*.tgz' "$MAX_TOFU_AGE_HOURS" warn
check_latest "Restore drill report" "$GITEA_DRILL_DIR" 'restore-drill-*.txt' "$MAX_DRILL_AGE_HOURS" warn
section "Repo Managed Pi-hole Inputs"
for file in $PIHOLE_CONFIG_FILES; do
printf '%-30s ' "$file"
if [ -s "$file" ]; then
printf 'ok\n'
else
printf 'warn - missing or empty\n'
warnings=$((warnings + 1))
fi
done
section "Summary"
printf 'failures=%s warnings=%s\n' "$failures" "$warnings"
if [ "$failures" -gt 0 ]; then
exit 1
fi