From 06c963d8e6accc6cefd7f7ebdde74e6c64d05c23 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 25 May 2026 11:42:13 -0600 Subject: [PATCH] Adding gitea backups and exposing it to the internet read only --- README.md | 42 ++++++++ apps/gitea/deployment.yaml | 18 +++- bootstrap/cluster/main.tf | 20 ++-- bootstrap/cluster/variables.tf | 7 +- bootstrap/edge/main.tf | 1 + bootstrap/edge/templates/default.conf.tftpl | 25 +++++ bootstrap/edge/variables.tf | 5 + bootstrap/platform/main.tf | 15 ++- lab.sh | 113 +++++++++++++++++++- 9 files changed, 233 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d63e7a8..6f97fec 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,48 @@ For the current lab, `/var/openebs/local` and `/var/lib/docker` are expected to live on larger storage than the root filesystem. This keeps retained PVs, container layers, Buildx state, and image caches from filling `/`. +## Gitea + +Gitea is deployed from `apps/gitea`, stores data in the retained local PV at +`/var/openebs/local/gitea`, and is exposed through the public edge path at +`https://lab2025.duckdns.org/git/`. HTTP clone and push traffic goes through the +same path. The NodePort remains available inside the lab at port `30300`. + +`./lab.sh up` applies the Gitea manifests directly before creating Argo CD +Applications. This keeps the Git service bootstrap-safe if the GitOps repo is +later moved into in-cluster Gitea. + +After the repo exists in Gitea, Argo CD can be pointed at the internal service +URL so it no longer depends on the old external Git server: + +```bash +export TF_VAR_gitops_repo_url='http://gitea.gitea-system.svc.cluster.local:3000/jv/my-homelab-configs.git' +tofu -chdir=bootstrap/platform apply -auto-approve +tofu -chdir=bootstrap/apps apply -auto-approve +``` + +## Gitea Backups + +`./lab.sh up` installs a Debian-host systemd timer named +`homelab-gitea-backup.timer`. The timer runs daily, executes `gitea dump` inside +the Gitea pod, copies the dump out of Kubernetes, and stores it under +`/var/backups/homelab/gitea` on the Debian server. The default retention is 30 +days. + +Run a manual backup from the Debian server with: + +```bash +./lab.sh backup-gitea +``` + +Useful checks: + +```bash +systemctl list-timers homelab-gitea-backup.timer +sudo systemctl start homelab-gitea-backup.service +sudo ls -lh /var/backups/homelab/gitea +``` + ## Destructive Rebuilds `./lab.sh nuke` resets kubeadm, containerd runtime state, CNI files, Calico diff --git a/apps/gitea/deployment.yaml b/apps/gitea/deployment.yaml index 871c364..a794512 100644 --- a/apps/gitea/deployment.yaml +++ b/apps/gitea/deployment.yaml @@ -43,22 +43,36 @@ spec: value: "true" - name: GITEA__migrations__ALLOW_LOCALNETWORKS value: "true" + - name: GITEA__repository__DEFAULT_PRIVATE + value: public + - name: GITEA__security__INSTALL_LOCK + value: "true" + - name: GITEA__server__DOMAIN + value: lab2025.duckdns.org + - name: GITEA__server__ROOT_URL + value: https://lab2025.duckdns.org/git/ + - name: GITEA__server__SERVE_FROM_SUB_PATH + value: "true" - name: GITEA__server__SSH_PORT value: "32222" - name: GITEA__server__SSH_LISTEN_PORT value: "22" + - name: GITEA__service__DISABLE_REGISTRATION + value: "true" + - name: GITEA__service__REQUIRE_SIGNIN_VIEW + value: "false" volumeMounts: - name: gitea-data mountPath: /data readinessProbe: httpGet: - path: / + path: /git/ port: http initialDelaySeconds: 20 periodSeconds: 10 livenessProbe: httpGet: - path: / + path: /git/ port: http initialDelaySeconds: 60 periodSeconds: 30 diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index 7732996..b92a043 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -298,13 +298,13 @@ resource "null_resource" "kubeadm_worker" { registry_config_version = "6" node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) - tailscale_nodeport_version = "2" + tailscale_nodeport_version = "3" tailscale_nodeport_enabled = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false" tailscale_nodeport_peer_ip = var.tailscale_nodeport_access.peer_ip tailscale_nodeport_node_tailscale_ip = var.tailscale_nodeport_access.node_tailscale_ip tailscale_nodeport_pod_cidr = var.tailscale_nodeport_access.pod_cidr tailscale_nodeport_node_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.node_port], var.tailscale_nodeport_extra_ports))) - tailscale_nodeport_target_port = tostring(var.tailscale_nodeport_access.target_port) + tailscale_nodeport_target_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.target_port], var.tailscale_nodeport_extra_target_ports))) } connection { @@ -531,7 +531,7 @@ configure_tailscale_nodeport_access() { local node_tailscale_ip="$3" local pod_cidr="$4" local node_ports="$5" - local target_port="$6" + local target_ports="$6" if [ "$enabled" != "true" ]; then return 0 @@ -561,12 +561,16 @@ for node_port in $node_ports; do iptables -C INPUT -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT done -iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "$target_port" -j ACCEPT 2>/dev/null || - iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "$target_port" -j ACCEPT +for target_port in $target_ports; do + iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT 2>/dev/null || + iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT +done iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT -iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null || - iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE +for target_port in $target_ports; do + iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null || + iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE +done NODEPORT_SCRIPT_EOT sudo chmod 0755 /usr/local/sbin/homelab-tailscale-nodeport.sh @@ -595,7 +599,7 @@ configure_tailscale_nodeport_access \ "${self.triggers.tailscale_nodeport_node_tailscale_ip}" \ "${self.triggers.tailscale_nodeport_pod_cidr}" \ "${self.triggers.tailscale_nodeport_node_ports}" \ - "${self.triggers.tailscale_nodeport_target_port}" + "${self.triggers.tailscale_nodeport_target_ports}" configure_containerd_registry "${self.triggers.registry_endpoint}" diff --git a/bootstrap/cluster/variables.tf b/bootstrap/cluster/variables.tf index 741c5af..fda3bd6 100644 --- a/bootstrap/cluster/variables.tf +++ b/bootstrap/cluster/variables.tf @@ -86,5 +86,10 @@ variable "tailscale_nodeport_access" { variable "tailscale_nodeport_extra_ports" { type = list(number) - default = [30081] + default = [30081, 30300] +} + +variable "tailscale_nodeport_extra_target_ports" { + type = list(number) + default = [3000] } diff --git a/bootstrap/edge/main.tf b/bootstrap/edge/main.tf index 5a1cb3d..9429317 100644 --- a/bootstrap/edge/main.tf +++ b/bootstrap/edge/main.tf @@ -14,6 +14,7 @@ locals { server_name = var.server_name backend_host = var.backend_host demos_backend_port = var.demos_backend_port + gitea_backend_port = var.gitea_backend_port }) default_vcl = templatefile("${path.module}/templates/default.vcl.tftpl", { backend_host = var.backend_host diff --git a/bootstrap/edge/templates/default.conf.tftpl b/bootstrap/edge/templates/default.conf.tftpl index 193e66d..79738a7 100644 --- a/bootstrap/edge/templates/default.conf.tftpl +++ b/bootstrap/edge/templates/default.conf.tftpl @@ -75,6 +75,31 @@ server { return 204; } + location = /git { + return 301 /git/; + } + + location ^~ /git/ { + limit_req zone=one burst=20 nodelay; + client_max_body_size 512m; + + proxy_pass http://${backend_host}:${gitea_backend_port}; + proxy_http_version 1.1; + proxy_request_buffering off; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-Prefix /git; + proxy_set_header CF-Connecting-IP $http_cf_connecting_ip; + proxy_redirect off; + add_header Cache-Control "no-store"; + } + location ^~ /demo-apps/ { limit_req zone=one burst=20 nodelay; diff --git a/bootstrap/edge/variables.tf b/bootstrap/edge/variables.tf index 877148c..0d28eff 100644 --- a/bootstrap/edge/variables.tf +++ b/bootstrap/edge/variables.tf @@ -53,6 +53,11 @@ variable "demos_backend_port" { default = 30081 } +variable "gitea_backend_port" { + type = number + default = 30300 +} + variable "haproxy_stats_user" { type = string default = "admin" diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf index 8b494a0..7139c3a 100644 --- a/bootstrap/platform/main.tf +++ b/bootstrap/platform/main.tf @@ -281,6 +281,20 @@ resource "null_resource" "argocd_private_repo" { set -euo pipefail repo_url="${self.triggers.repo_url}" + +case "$${repo_url}" in + http://*|https://*) + kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" create secret generic "${self.triggers.secret_name}" \ + --from-literal=type=git \ + --from-literal=url="${self.triggers.repo_url}" \ + --dry-run=client -o yaml | kubectl --kubeconfig "${self.triggers.kubeconfig_path}" apply -f - + + kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" label secret "${self.triggers.secret_name}" \ + argocd.argoproj.io/secret-type=repository --overwrite + exit 0 + ;; +esac + repo_target="$${repo_url#ssh://}" repo_target="$${repo_target#*@}" repo_target="$${repo_target%%/*}" @@ -335,4 +349,3 @@ resource "helm_release" "extra_tools" { } } } - diff --git a/lab.sh b/lab.sh index 6b964fe..f3c4ce9 100755 --- a/lab.sh +++ b/lab.sh @@ -410,6 +410,112 @@ wait_for_deployment_ready() { done } +apply_gitea_bootstrap_manifests() { + kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/namespace.yaml" + kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/storage.yaml" + kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/service.yaml" + kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/deployment.yaml" + + wait_for_namespace gitea-system gitea 300 + wait_for_namespaced_resource gitea-system deployment gitea gitea 300 + wait_for_deployment_ready gitea-system gitea gitea 300 +} + +install_gitea_backup_timer() { + local backup_script="/usr/local/sbin/homelab-gitea-backup.sh" + + sudo tee "${backup_script}" >/dev/null </dev/null 2>&1; then + echo "kubectl is required for Gitea backups." >&2 + exit 1 +fi + +pod="\$(kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" get pods \ + -l "\${GITEA_SELECTOR}" \ + --field-selector=status.phase=Running \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + +if [[ -z "\${pod}" ]]; then + echo "Skipping Gitea backup: no running Gitea pod found." + exit 0 +fi + +timestamp="\$(date -u +%Y%m%dT%H%M%SZ)" +tmp_archive="\$(mktemp "/tmp/gitea-\${timestamp}.XXXXXX.zip")" +backup_archive="\${GITEA_BACKUP_DIR}/gitea-\${timestamp}.zip" + +cleanup() { + rm -f "\${tmp_archive}" + kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- rm -f "\${REMOTE_ARCHIVE}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- rm -f "\${REMOTE_ARCHIVE}" >/dev/null 2>&1 || true +kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- \ + gitea dump -c /data/gitea/conf/app.ini --file "\${REMOTE_ARCHIVE}" +kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" cp -c "\${GITEA_CONTAINER}" \ + "\${GITEA_NAMESPACE}/\${pod}:\${REMOTE_ARCHIVE}" "\${tmp_archive}" + +sudo mkdir -p "\${GITEA_BACKUP_DIR}" +sudo install -m 0640 -o root -g root "\${tmp_archive}" "\${backup_archive}" +sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete + +echo "Created \${backup_archive}" +BACKUP_SCRIPT_EOT + sudo chmod 0755 "${backup_script}" + + sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT' +[Unit] +Description=Back up in-cluster Gitea to Debian host storage +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-gitea-backup.sh +SERVICE_EOT + + sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT' +[Unit] +Description=Run daily Homelab Gitea backups + +[Timer] +OnCalendar=*-*-* 02:35:00 +RandomizedDelaySec=20m +Persistent=true + +[Install] +WantedBy=timers.target +TIMER_EOT + + sudo systemctl daemon-reload + sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null +} + +backup_gitea() { + require_debian_server "backup-gitea" + + export KUBECONFIG="${KUBECONFIG_PATH}" + install_gitea_backup_timer + sudo /usr/local/sbin/homelab-gitea-backup.sh +} + recreate_pods_for_selector() { local namespace="$1" local selector="$2" @@ -474,6 +580,8 @@ up() { run_tofu_stack "bootstrap/cluster" run_tofu_stack "bootstrap/platform" + apply_gitea_bootstrap_manifests + install_gitea_backup_timer run_tofu_stack "bootstrap/apps" refresh_argocd_application container-registry @@ -707,11 +815,14 @@ case "${1:-}" in up) up ;; + backup-gitea) + backup_gitea + ;; nuke) nuke ;; *) - echo "Usage: $0 {up|nuke}" + echo "Usage: $0 {up|backup-gitea|nuke}" exit 1 ;; esac