From 047aee8481904b1f5dcfce2904c63cfb219a7bdb Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 26 May 2026 23:07:35 -0600 Subject: [PATCH] Add Gitea backup restore drill --- README.md | 14 ++++++ lab.sh | 124 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a45b5fe..7082b88 100644 --- a/README.md +++ b/README.md @@ -318,18 +318,32 @@ the Gitea pod, copies the dump out of Kubernetes, and stores it under `/var/backups/homelab/gitea` on the Debian server. The default retention is 30 days. +The same install step also creates `homelab-gitea-restore-drill.timer`. The +monthly drill is non-destructive: it verifies the latest backup ZIP, extracts it +to a temporary directory, records a report under +`/var/backups/homelab/gitea-restore-drills`, and removes the temporary extract. +It does not write into the live Gitea PVC. + Run a manual backup from the Debian server with: ```bash ./lab.sh backup-gitea ``` +Run the restore drill manually with: + +```bash +./lab.sh drill-gitea-restore +``` + Useful checks: ```bash systemctl list-timers homelab-gitea-backup.timer +systemctl list-timers homelab-gitea-restore-drill.timer sudo systemctl start homelab-gitea-backup.service sudo ls -lh /var/backups/homelab/gitea +sudo ls -lh /var/backups/homelab/gitea-restore-drills ``` ## Gitea Actions diff --git a/lab.sh b/lab.sh index ae67048..ac0d0bc 100755 --- a/lab.sh +++ b/lab.sh @@ -1178,6 +1178,7 @@ apply_gitea_bootstrap_manifests() { install_gitea_backup_timer() { local backup_script="/usr/local/sbin/homelab-gitea-backup.sh" + local restore_drill_script="/usr/local/sbin/homelab-gitea-restore-drill.sh" sudo tee "${backup_script}" >/dev/null </dev/null <<'RESTORE_DRILL_SCRIPT_EOT' +#!/usr/bin/env bash +set -euo pipefail + +GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/var/backups/homelab/gitea}" +GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/var/backups/homelab/gitea-restore-drills}" +GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}" + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required for Gitea restore drills." >&2 + exit 1 +fi + +latest_archive="$( + { find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } | + sort -nr | + awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }' +)" + +if [[ -z "${latest_archive}" ]]; then + echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}." + exit 0 +fi + +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")" +tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")" +report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt" + +cleanup() { + rm -rf "${tmp_dir}" + rm -f "${tmp_report}" +} +trap cleanup EXIT + +python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY' +import os +import sys +import zipfile + +archive_path, extract_dir, report_path = sys.argv[1:4] + +with zipfile.ZipFile(archive_path) as archive: + bad_member = archive.testzip() + if bad_member: + raise SystemExit(f"ZIP integrity check failed at {bad_member}") + + members = archive.infolist() + if not members: + raise SystemExit("ZIP archive is empty") + + extract_root = os.path.abspath(extract_dir) + for member in members: + target = os.path.abspath(os.path.join(extract_root, member.filename)) + if target != extract_root and not target.startswith(extract_root + os.sep): + raise SystemExit(f"Unsafe archive path: {member.filename}") + + archive.extractall(extract_root) + +file_count = 0 +total_bytes = 0 +for root, _, files in os.walk(extract_dir): + for name in files: + file_count += 1 + total_bytes += os.path.getsize(os.path.join(root, name)) + +if file_count == 0: + raise SystemExit("Archive extracted no files") + +with open(report_path, "w", encoding="utf-8") as handle: + handle.write("Gitea restore drill report\n") + handle.write(f"archive={archive_path}\n") + handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n") + handle.write(f"extracted_files={file_count}\n") + handle.write(f"extracted_bytes={total_bytes}\n") + handle.write("result=ok\n") +PY + +sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}" +sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}" +sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete + +echo "Created ${report_path}" +RESTORE_DRILL_SCRIPT_EOT + sudo chmod 0755 "${restore_drill_script}" + + sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT' +[Unit] +Description=Run a non-destructive Gitea backup restore drill +After=network-online.target homelab-gitea-backup.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh +RESTORE_DRILL_SERVICE_EOT + + sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT' +[Unit] +Description=Run monthly Homelab Gitea restore drills + +[Timer] +OnCalendar=monthly +RandomizedDelaySec=2h +Persistent=true + +[Install] +WantedBy=timers.target +RESTORE_DRILL_TIMER_EOT + sudo systemctl daemon-reload sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null + sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null } backup_gitea() { @@ -1273,6 +1385,13 @@ backup_gitea() { sudo /usr/local/sbin/homelab-gitea-backup.sh } +drill_gitea_restore() { + require_debian_server "drill-gitea-restore" + + install_gitea_backup_timer + sudo /usr/local/sbin/homelab-gitea-restore-drill.sh +} + install_gitea_runner() { local runner_arch local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}" @@ -1677,6 +1796,9 @@ case "${1:-}" in backup-gitea) backup_gitea ;; + drill-gitea-restore) + drill_gitea_restore + ;; install-gitea-runner) install_gitea_runner "${2:-}" ;; @@ -1684,7 +1806,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|apps|backup-gitea|install-gitea-runner|nuke}" + echo "Usage: $0 {up|apps|backup-gitea|drill-gitea-restore|install-gitea-runner|nuke}" exit 1 ;; esac