From cc657fad6cc5a9989ba76e121cb8e8b55ab8a52c Mon Sep 17 00:00:00 2001 From: jv Date: Wed, 27 May 2026 14:15:10 -0500 Subject: [PATCH] Bootstrap external Gitea --- .gitea/workflows/homelab-main.yml | 3 +- .gitignore | 2 +- .trivyignore.yaml | 9 - README.md | 124 +++--- apps/gitea/deployment.yaml | 90 ----- apps/gitea/namespace.yaml | 4 - apps/gitea/service.yaml | 18 - apps/gitea/storage.yaml | 36 -- apps/website/blog.php | 10 +- apps/website/homelab-tree.php | 6 +- apps/website/lang/en.php | 14 +- apps/website/lang/nah.php | 2 +- bootstrap/apps/variables.tf | 9 - bootstrap/cluster/variables.tf | 7 +- bootstrap/edge/variables.tf | 2 +- bootstrap/provisioning/README.md | 16 +- bootstrap/provisioning/main.tf | 2 + bootstrap/provisioning/variables.tf | 7 +- infra/gitea/README.md | 26 ++ infra/gitea/docker-compose.yml | 26 ++ lab.sh | 559 +++++++++++++++++++++++++--- renovate.json | 1 + 22 files changed, 676 insertions(+), 297 deletions(-) delete mode 100644 .trivyignore.yaml delete mode 100644 apps/gitea/deployment.yaml delete mode 100644 apps/gitea/namespace.yaml delete mode 100644 apps/gitea/service.yaml delete mode 100644 apps/gitea/storage.yaml create mode 100644 infra/gitea/README.md create mode 100644 infra/gitea/docker-compose.yml diff --git a/.gitea/workflows/homelab-main.yml b/.gitea/workflows/homelab-main.yml index 2aa58cc..9d10479 100644 --- a/.gitea/workflows/homelab-main.yml +++ b/.gitea/workflows/homelab-main.yml @@ -129,6 +129,7 @@ jobs: set -euo pipefail bash -n lab.sh + docker compose -f infra/gitea/docker-compose.yml config >/dev/null kubectl --kubeconfig "${KUBECONFIG:-/home/jv/.kube/config}" apply --dry-run=server --recursive -f apps @@ -161,7 +162,7 @@ jobs: fi printf '%s\n' "${changed_files}" - if printf '%s\n' "${changed_files}" | grep -Eq '^(bootstrap/(provisioning|cluster|platform|edge)/|lab[.]sh|[.]gitea/workflows/)'; then + if printf '%s\n' "${changed_files}" | grep -Eq '^(bootstrap/(provisioning|cluster|platform|edge)/|infra/gitea/|lab[.]sh|[.]gitea/workflows/)'; then echo "High-impact bootstrap, runner, or workflow changes require a manual Debian run." exit 1 fi diff --git a/.gitignore b/.gitignore index 169808c..d26c15f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ # Ignore local archive dumps and backups *.tar *.zip -apps/gitea/gitea-docker-backup +infra/gitea/data/ # Ignore decrypted secret material *.dec.yaml diff --git a/.trivyignore.yaml b/.trivyignore.yaml deleted file mode 100644 index f8db970..0000000 --- a/.trivyignore.yaml +++ /dev/null @@ -1,9 +0,0 @@ -misconfigurations: - - id: KSV-0014 - paths: - - apps/gitea/deployment.yaml - statement: Gitea needs a separate tested migration to the rootless image because its current persistent volume layout uses the standard image /data path and OpenSSH setup. - - id: KSV-0118 - paths: - - apps/gitea/deployment.yaml - statement: Gitea needs a separate tested migration to the rootless image because its current persistent volume layout uses the standard image /data path and OpenSSH setup. diff --git a/README.md b/README.md index ee6df90..facfef2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,10 @@ The lab is intentionally small but production-shaped: - a Debian amd64 host runs the kubeadm control plane and local deployment tools - a Raspberry Pi arm64 node runs selected workloads +- the Raspberry Pi also runs the always-on Gitea Docker service outside + Kubernetes +- the Debian host keeps a bare GitOps mirror under + `/home/jv/git-server/my-homelab-configs.git` - a provisioning layer can PXE boot Debian 13 arm64 VMs for Pimox worker templates - OpenTofu owns the bootstrap layers for cluster, platform, apps, and edge @@ -55,7 +59,7 @@ accidentally modify the cluster. 4. `bootstrap/apps` - registers Argo CD Applications from the `applications` map - - default apps are `container-registry`, `gitea`, `website-production`, and + - default apps are `container-registry`, `website-production`, and `demos-static` 5. `bootstrap/edge` @@ -87,14 +91,16 @@ cd ~/my-homelab-configs ./lab.sh up ``` -The script detects the Pimox host at `192.168.100.80` in auto mode. When SSH, -`qm`, and `vmbr0` are available, it applies `bootstrap/provisioning`, creates or -reuses the Debian 13 arm64 template, creates or reuses one worker VM clone, -discovers the guest IP through qemu-guest-agent, and passes that worker into the -cluster layer. It then applies the remaining OpenTofu stacks, refreshes Argo CD -apps, waits for the local registry, builds the website and demos images when -their source changed, pushes them to the registry, recreates pods only after a -new image is built, and applies the edge stack. +The script first deploys external Gitea to the Raspberry Pi with Docker Compose +so Git stays outside the Kubernetes rebuild blast radius. It then detects the +Pimox host at `192.168.100.80` in auto mode. When SSH, `qm`, and `vmbr0` are +available, it applies `bootstrap/provisioning`, creates or reuses the Debian 13 +arm64 template, creates or reuses one worker VM clone, discovers the guest IP +through qemu-guest-agent, and passes that worker into the cluster layer. It then +applies the remaining OpenTofu stacks, refreshes Argo CD apps, waits for the +local registry, builds the website and demos images when their source changed, +pushes them to the registry, recreates pods only after a new image is built, and +applies the edge stack. Set `LAB_PIMOX_PIPELINE=false` to skip Pimox automation. Set `LAB_PIMOX_WORKER_COUNT=0` to create or refresh only the template. The pipeline @@ -125,6 +131,11 @@ Build metadata is written under `.lab/` so repeat runs can skip the website or demos image build when the source hash, platform, image reference, and registry manifest still match. +Set `LAB_GITEA_DEPLOY=false` to skip the external Gitea deployment step when the +Raspberry Pi service is already managed manually. The default Gitea target is +`jv@192.168.100.89`, install directory `/opt/homelab-gitea`, HTTP port `3000`, +and SSH port `32222`. + ## Validation Useful checks after a rebuild: @@ -135,10 +146,11 @@ export KUBECONFIG=/home/jv/.kube/config kubectl get nodes kubectl -n argocd get applications kubectl -n container-registry get pods -kubectl -n gitea-system get pods kubectl -n website-production get pods -o wide kubectl -n demos-static get pods -o wide +ssh jv@192.168.100.89 'cd /opt/homelab-gitea && sudo docker compose ps' + docker info --format '{{.DockerRootDir}}' df -h / /var/openebs/local /var/lib/docker ``` @@ -190,10 +202,11 @@ duplicate those PV manifests when you want storage on another node. Override `control_plane_node_labels`, `worker_node_labels`, `LAB_RASPBERRY_NODE_LABELS_JSON`, or `LAB_PIMOX_WORKER_NODE_LABELS_JSON` when -the physical layout changes. The current website, demos, registry, and Gitea -manifests are not moved automatically because the public NodePort path and -retained OpenEBS hostpath PVs are node-local. Move workloads only after their -storage and edge path are ready on the target node. +the physical layout changes. The current website, demos, and registry manifests +are not moved automatically because the public NodePort path and retained +OpenEBS hostpath PVs are node-local. Move workloads only after their storage and +edge path are ready on the target node. Gitea is outside Kubernetes and is moved +by changing the Raspberry Pi Docker install target instead. The website and demos NodePorts are reachable from the OCI jump box through the Raspberry Pi Tailscale interface. `bootstrap/cluster` installs a persistent @@ -211,7 +224,7 @@ tailscale_nodeport_access = { node_tailscale_ip = "100.77.80.72" pod_cidr = "10.244.0.0/16" node_port = 30080 - target_port = 80 + target_port = 8080 } tailscale_nodeport_extra_ports = [30081] @@ -260,8 +273,9 @@ export TF_VAR_metallb='{ }' ``` -The current website, demos, registry, and Gitea services remain `NodePort` -services until the LAN address pool and edge route are tested manually. +The current website, demos, and registry services remain `NodePort` services +until the LAN address pool and edge route are tested manually. Gitea is not a +Kubernetes service; it runs on the Raspberry Pi Docker host. ## Secrets @@ -284,6 +298,11 @@ deploys them to `/opt/homelab-edge` on the OCI host. Defaults are in file when the public host, SSH key, server name, backend Tailscale IP, or NodePort changes. +The `/git/` route is intentionally different from the Kubernetes app routes: it +proxies to Gitea on the Raspberry Pi at the configured `backend_host` and +`gitea_backend_port` instead of a Kubernetes NodePort. This keeps public +read-only source browsing available even when the cluster has been destroyed. + Use the configured `server_name` in the browser, for example `https://lab2025.duckdns.org`. A raw OCI IP address will still show a browser certificate warning because the trusted certificate is issued for the hostname. @@ -302,12 +321,12 @@ self-healing for the app. ## Storage -OpenEBS provides the platform storage provisioner. Stateful homelab apps use -retained local PV paths such as `/var/openebs/local/gitea` and -`/var/openebs/local/registry`; these paths are intentionally outside kubeadm -reset paths so data can survive cluster destroy/create cycles. Those critical -volumes are declared explicitly as retained local PVs so a rebuilt cluster binds -back to the same host paths instead of creating fresh directories. +OpenEBS provides the platform storage provisioner. Stateful Kubernetes apps use +retained local PV paths such as `/var/openebs/local/registry`; these paths are +intentionally outside kubeadm reset paths so data can survive cluster +destroy/create cycles. Those critical volumes are declared explicitly as +retained local PVs so a rebuilt cluster binds back to the same host paths +instead of creating fresh directories. For the current lab, `/var/openebs/local` and `/var/lib/docker` are expected to live on larger storage than the root filesystem. This keeps retained PVs, @@ -315,37 +334,49 @@ container layers, Buildx state, and image caches from filling `/`. ## Gitea -Gitea is deployed from `apps/gitea`, stores data in the retained local PV at -`/var/openebs/local/gitea`, and is exposed through the public edge path at -`https://lab2025.duckdns.org/git/`. HTTP clone and push traffic goes through the -same path. The NodePort remains available inside the lab at port `30300`. +Gitea is external bootstrap infrastructure. It runs on the Raspberry Pi as an +always-on Docker Compose service from `infra/gitea/docker-compose.yml`, not as a +Kubernetes workload. This keeps Git available when the Kubernetes cluster is +destroyed and rebuilt. -`./lab.sh up` applies the Gitea manifests directly before creating Argo CD -Applications. This keeps the Git service bootstrap-safe if the GitOps repo is -later moved into in-cluster Gitea. +The default data path is `/opt/homelab-gitea/data` on the Raspberry Pi SD card. +That is acceptable for the current temporary setup; move +`LAB_GITEA_INSTALL_DIR` to an SSD mount when the SSD is added. -After the repo exists in Gitea, Argo CD can be pointed at the internal service -URL so it no longer depends on the old external Git server: +Public source browsing stays available through +`https://lab2025.duckdns.org/git/`. Registration is disabled and anonymous users +can view public repositories, so the blog can link to code read-only while +writes still require an authenticated Gitea account. + +The Debian bare repo remains the GitOps mirror: + +```text +/home/jv/git-server/my-homelab-configs.git +``` + +Argo CD consumes that Debian mirror through the default `gitops_repo_url`. +Gitea Actions pushes the validated `main` commit into the mirror before running +`./lab.sh apps`. + +Deploy or refresh the external Gitea container from the Debian host with: ```bash -export TF_VAR_gitops_repo_url='http://gitea.gitea-system.svc.cluster.local:3000/jv/my-homelab-configs.git' -tofu -chdir=bootstrap/platform apply -auto-approve -tofu -chdir=bootstrap/apps apply -auto-approve +./lab.sh deploy-gitea ``` ## Gitea Backups `./lab.sh up` installs a Debian-host systemd timer named -`homelab-gitea-backup.timer`. The timer runs daily, executes `gitea dump` inside -the Gitea pod, copies the dump out of Kubernetes, and stores it under -`/var/backups/homelab/gitea` on the Debian server. The default retention is 30 -days. +`homelab-gitea-backup.timer`. The timer runs daily, SSHes to the Raspberry Pi, +executes `gitea dump` inside the Gitea Docker container, copies the dump back to +Debian, and stores it under `/home/jv/backups/gitea`. The default retention is +30 days. The same install step also creates `homelab-gitea-restore-drill.timer`. The monthly drill is non-destructive: it verifies the latest backup ZIP, extracts it to a temporary directory, records a report under -`/var/backups/homelab/gitea-restore-drills`, and removes the temporary extract. -It does not write into the live Gitea PVC. +`/home/jv/backups/gitea-restore-drills`, and removes the temporary extract. It +does not write into the live Raspberry Pi Gitea data directory. Run a manual backup from the Debian server with: @@ -365,8 +396,8 @@ Useful checks: systemctl list-timers homelab-gitea-backup.timer systemctl list-timers homelab-gitea-restore-drill.timer sudo systemctl start homelab-gitea-backup.service -sudo ls -lh /var/backups/homelab/gitea -sudo ls -lh /var/backups/homelab/gitea-restore-drills +ls -lh /home/jv/backups/gitea +ls -lh /home/jv/backups/gitea-restore-drills ``` ## Gitea Actions @@ -378,9 +409,10 @@ a repository-scoped Debian host runner with the label `homelab-debian`. The workflow validates shell syntax, Kubernetes manifests, and all OpenTofu stacks before deployment. It automatically stops when high-impact files under `bootstrap/provisioning`, `bootstrap/cluster`, `bootstrap/platform`, -`bootstrap/edge`, `lab.sh`, or `.gitea/workflows` change; those changes still -require a manual Debian run. Lower-risk app changes proceed to `./lab.sh apps` -after validation passes, which skips Pimox, cluster, platform, and edge changes. +`bootstrap/edge`, `infra/gitea`, `lab.sh`, or `.gitea/workflows` change; those +changes still require a manual Debian run. Lower-risk app changes proceed to +`./lab.sh apps` after validation passes, which skips Gitea, Pimox, cluster, +platform, and edge changes. Enable Actions for the repository in Gitea, then create a repository-level runner token from: diff --git a/apps/gitea/deployment.yaml b/apps/gitea/deployment.yaml deleted file mode 100644 index c1c026f..0000000 --- a/apps/gitea/deployment.yaml +++ /dev/null @@ -1,90 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: gitea - namespace: gitea-system - labels: - app: gitea -spec: - replicas: 1 - selector: - matchLabels: - app: gitea - template: - metadata: - labels: - app: gitea - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - debian - containers: - - name: gitea - image: gitea/gitea:1.21.7 - ports: - - containerPort: 3000 - name: http - - containerPort: 22 - name: ssh - env: - - name: USER_UID - value: "1000" - - name: USER_GID - value: "1000" - - name: GITEA__database__DB_TYPE - value: sqlite3 - - name: GITEA__repository__ENABLE_PUSH_MIRROR - value: "true" - - name: GITEA__migrations__ALLOW_LOCALNETWORKS - value: "true" - - name: GITEA__actions__ENABLED - value: "true" - - name: GITEA__repository__DEFAULT_PRIVATE - value: public - - name: GITEA__security__INSTALL_LOCK - value: "true" - - name: GITEA__server__DOMAIN - value: lab2025.duckdns.org - - name: GITEA__server__ROOT_URL - value: https://lab2025.duckdns.org/git/ - - name: GITEA__server__SERVE_FROM_SUB_PATH - value: "true" - - name: GITEA__server__SSH_PORT - value: "32222" - - name: GITEA__server__SSH_LISTEN_PORT - value: "22" - - name: GITEA__service__DISABLE_REGISTRATION - value: "true" - - name: GITEA__service__REQUIRE_SIGNIN_VIEW - value: "false" - volumeMounts: - - name: gitea-data - mountPath: /data - readinessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 20 - periodSeconds: 10 - livenessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 60 - periodSeconds: 30 - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - memory: 1Gi - volumes: - - name: gitea-data - persistentVolumeClaim: - claimName: gitea-data diff --git a/apps/gitea/namespace.yaml b/apps/gitea/namespace.yaml deleted file mode 100644 index a2b56dc..0000000 --- a/apps/gitea/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: gitea-system diff --git a/apps/gitea/service.yaml b/apps/gitea/service.yaml deleted file mode 100644 index 069f5e3..0000000 --- a/apps/gitea/service.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: gitea - namespace: gitea-system -spec: - type: NodePort - selector: - app: gitea - ports: - - name: http - port: 3000 - targetPort: http - nodePort: 30300 - - name: ssh - port: 22 - targetPort: ssh - nodePort: 32222 diff --git a/apps/gitea/storage.yaml b/apps/gitea/storage.yaml deleted file mode 100644 index 8a9fdf9..0000000 --- a/apps/gitea/storage.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: gitea-data-debian -spec: - capacity: - storage: 20Gi - volumeMode: Filesystem - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Retain - storageClassName: openebs-hostpath-retain - local: - path: /var/openebs/local/gitea - nodeAffinity: - required: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/hostname - operator: In - values: - - debian ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: gitea-data - namespace: gitea-system -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath-retain - volumeName: gitea-data-debian - resources: - requests: - storage: 20Gi diff --git a/apps/website/blog.php b/apps/website/blog.php index 176bf21..d6ace97 100644 --- a/apps/website/blog.php +++ b/apps/website/blog.php @@ -308,7 +308,7 @@ function renderStackSourceLinks(string $stackKey, array $sourceLinks, string $so Argo CD - registry, gitea, monitoring + registry and monitoring website and demos-static apps @@ -336,17 +336,17 @@ function renderStackSourceLinks(string $stackKey, array $sourceLinks, string $so - Tailscale + NodePorts + Tailscale + edge routes 30080 website, 30081 demos - 30300 Gitea service path + 3000 Gitea on Raspberry Pi Raspberry Pi 192.168.100.89 arm64 Kubernetes worker - website-production pods - demos-static and lab apps + external Gitea Docker service + website and demos pods diff --git a/apps/website/homelab-tree.php b/apps/website/homelab-tree.php index b4254ae..de88787 100644 --- a/apps/website/homelab-tree.php +++ b/apps/website/homelab-tree.php @@ -156,8 +156,8 @@ $blogHref = 'blog.php?lang=' . urlencode($lang); - Gitea app - Git service + Registry + image cache @@ -243,7 +243,7 @@ $blogHref = 'blog.php?lang=' . urlencode($lang);
  • Star: public DNS, TLS, and the entry point users actually type.
  • Garlands: Tailscale routing, NodePorts, and the GitOps sync loop connecting the layers.
  • Branches: Kubernetes namespaces and workloads that carry the visible services.
    • -
  • Ornaments: Gitea, Actions, Argo CD, Buildx, Trivy, Gitleaks, Calico, registry, website, demos, and the Gitea app.
    • +
  • Ornaments: external Gitea, Actions, Argo CD, Buildx, Trivy, Gitleaks, Calico, registry, website, and demos.
  • Bells: probes and health checks that make noise before users do.
  • Trunk: the Debian control-plane node that holds the platform upright.
  • Roots: OpenEBS retained volumes, external SSD storage, Gitea dumps, and restore discipline.
    • diff --git a/apps/website/lang/en.php b/apps/website/lang/en.php index 02e7937..646cd96 100644 --- a/apps/website/lang/en.php +++ b/apps/website/lang/en.php @@ -65,7 +65,7 @@ return [ 'blog_q3' => 'So where is the CI/CD part hiding?', 'blog_a3' => 'It is small, but it is real. OpenTofu brings up the cluster, platform, apps, and edge layers. Argo CD watches Git and keeps the cluster honest. Docker Buildx builds the PHP website for linux/arm64, pushes it to the local registry, and then the workload rolls forward. No enterprise dashboard fireworks, just a clean loop that says: Git changed, image built, cluster updated, nobody had to kubectl-edit anything at 2 AM.', 'blog_q4' => 'Why run your own registry and Gitea? Was the simple option unavailable?', - 'blog_a4' => 'The simple option was very available, which is why I heroically ignored it. The registry means experiments do not need to go to a public image repo, and Gitea gives the lab its own Git service. Together they make the setup feel less like "some containers under the stairs" and more like a tiny platform with opinions, responsibilities, and occasionally dramatic storage needs.', + 'blog_a4' => 'The simple option was very available, which is why I heroically ignored it. The registry means experiments do not need to go to a public image repo, and external Gitea gives the lab its own Git service without making Kubernetes responsible for its own source of truth. Together they make the setup feel less like "some containers under the stairs" and more like a tiny platform with opinions, responsibilities, and occasionally dramatic storage needs.', 'blog_q5' => 'What actually hurt the most?', 'blog_a5' => 'Storage. Always storage. Kubernetes, Docker, retained volumes, and build caches can fill a small root disk with the quiet confidence of a bad decision. Moving OpenEBS local volumes and Docker data to the external SSD turned the lab from "why is everything on fire?" into "okay, this is usable now." Growth, allegedly.', 'blog_q6' => 'And now the website has demos and a weirdly expressive CV?', @@ -103,14 +103,14 @@ return [ 'blog_activity_kicker' => 'Recent activity log', 'blog_activity_title' => 'What changed since the first build', 'blog_activity_intro' => 'The lab moved from a working Kubernetes experiment into a more complete self-hosted delivery system. The latest work focused on trust, repeatability, VM-based expansion, and making deploys match the exact commit that passed validation.', - 'blog_activity_1' => 'Brought Gitea online as the local Git service, including persistent storage and the public /git/ route through the edge stack.', + 'blog_activity_1' => 'Moved Gitea out of Kubernetes and onto the Raspberry Pi as the local Git service, while keeping the public /git/ route through the edge stack.', 'blog_activity_2' => 'Installed and validated a Debian-hosted Gitea Actions runner so pushes to main can build, scan, and deploy without depending on a laptop session.', 'blog_activity_3' => 'Added a custom checkout flow for the /git/ subpath and kept a persistent Debian checkout for the deployment scripts.', - 'blog_activity_4' => 'Added Gitleaks secret scanning and Trivy scanning, with scoped exceptions only where the lab intentionally accepts a known Gitea workload shape.', + 'blog_activity_4' => 'Added Gitleaks secret scanning and Trivy scanning for the app and infrastructure tree.', 'blog_activity_5' => 'Changed deployment so the validated commit is pushed into the local GitOps mirror before lab.sh runs, preventing Argo CD from reconciling an older tree.', 'blog_activity_6' => 'Hardened the website, demos-static, and registry workloads with non-root containers, read-only root filesystems, resource limits, and explicit writable volumes.', 'blog_activity_7' => 'Split the demos into a dedicated demos-static image and Argo CD application so the PHP website stays small and boring.', - 'blog_activity_8' => 'Fixed Gitea operational details around probes, service paths, backup dumps, and the user context used for safe backup execution.', + 'blog_activity_8' => 'Changed Gitea backups to dump from the Raspberry Pi Docker container and store archives on the Debian host.', 'blog_activity_9' => 'Validated the full main-branch deployment path: fetch main, apply OpenTofu layers, build and push arm64 images, refresh Argo CD, and confirm the runner completes successfully.', 'blog_activity_10' => 'Built the Debian 13 arm64 Pimox template end to end with PXE, preseed, qemu-guest-agent discovery, cgroup validation, swap disabled, and a final seal step.', 'blog_activity_11' => 'Added NVMe-backed Pimox worker clone automation so VM 9000 stays on local storage while worker nodes are created on nvme_thin_pool.', @@ -119,15 +119,15 @@ return [ 'blog_todo_kicker' => 'Improvement backlog', 'blog_todo_title' => 'Todo list for the next homelab pass', 'blog_todo_intro' => 'These are improvement proposals, not chores for the sake of chores. Each item either reduces rebuild risk, tightens supply-chain hygiene, or makes the platform easier to operate when something fails.', - 'blog_todo_1' => 'Move Gitea to a rootless runtime image and remove the remaining privileged assumptions from the Git service.', - 'blog_todo_2' => 'Point Argo CD directly at Gitea once bootstrap is stable, then retire or simplify the local bare GitOps mirror.', + 'blog_todo_1' => 'Move Gitea data from the Raspberry Pi SD card to SSD-backed storage.', + 'blog_todo_2' => 'Keep the Debian bare GitOps mirror as the cluster source and add object-storage backups when OCI storage is ready.', 'blog_todo_3' => 'Add a real OpenTofu remote state backend with backup, locking, and a documented recovery path.', 'blog_todo_4' => 'Replace mutable latest image references with immutable tags or digest pins for website and demo workloads.', 'blog_todo_5' => 'Generate SBOMs and sign images so the local registry can prove what it is serving.', 'blog_todo_6' => 'Add Renovate or Dependabot-style dependency updates for base images, Helm charts, and GitHub/Gitea Actions.', 'blog_todo_7' => 'Enforce baseline Kubernetes policy with Kyverno or Gatekeeper: non-root, read-only roots, resource requests, and allowed registries.', 'blog_todo_8' => 'Turn the installed observability stack into useful operations views: a few high-signal dashboards, alerts for node health, storage pressure, certificate expiry, and failed app syncs.', - 'blog_todo_9' => 'Schedule backup restore drills for Gitea and OpenEBS volumes, then write the exact restore runbook.', + 'blog_todo_9' => 'Schedule backup restore drills for external Gitea and OpenEBS volumes, then write the exact restore runbook.', 'blog_todo_10' => 'Tighten TLS, SSH, and token rotation around the OCI edge, Gitea, registry, and runner credentials.', 'blog_todo_11' => 'Document the new storage split: local for the Pimox template, nvme_thin_pool for VM workers, OpenEBS for Kubernetes app data, and backup targets for anything that must survive a rebuild.', 'blog_todo_12' => 'Move sensitive app configuration into Sealed Secrets, External Secrets, or another explicit secret-management path.', diff --git a/apps/website/lang/nah.php b/apps/website/lang/nah.php index 8aa5d7b..6a2ef96 100644 --- a/apps/website/lang/nah.php +++ b/apps/website/lang/nah.php @@ -69,7 +69,7 @@ return [ 'blog_q3' => 'Canin nemi CI/CD ipan inin setup?', 'blog_a3' => 'Pipeline achi tepiton. OpenTofu quichihua cluster, platform, apps, ihuan edge. Argo CD quitta Git repo ihuan quichihua sync. Docker Buildx quichihua PHP website image para linux/arm64 ihuan quipush ipan local registry.', 'blog_q4' => 'Tleica private registry ihuan Gitea ipan lab?', - 'blog_a4' => 'Registry amo monequi nicpush nochi experiment ipan public repo. Gitea quimaca lab se Git service. In ome quichihua ce tepiton production platform.', + 'blog_a4' => 'Registry amo monequi nicpush nochi experiment ipan public repo. Gitea nemi fuera Kubernetes ipan Raspberry Pi ihuan quimaca lab se Git service. In ome quichihua ce tepiton production platform.', 'blog_q5' => 'Tlein achi ohui omomachtih?', 'blog_a5' => 'Storage. Kubernetes, Docker, retained volumes, ihuan build cache huel quitemitia root disk. OpenEBS ihuan Docker data omoyecpan ipan external SSD, ic system achi yec nemi.', 'blog_q6' => 'Ihuan axcan website quipia demos ihuan CV occeppa?', diff --git a/bootstrap/apps/variables.tf b/bootstrap/apps/variables.tf index 685c5fe..a8a2ab4 100644 --- a/bootstrap/apps/variables.tf +++ b/bootstrap/apps/variables.tf @@ -34,15 +34,6 @@ variable "applications" { self_heal = true create_namespace = true } - gitea = { - project = "default" - path = "apps/gitea" - namespace = "gitea-system" - target_revision = "main" - prune = true - self_heal = true - create_namespace = true - } website-production = { project = "default" path = "apps/website" diff --git a/bootstrap/cluster/variables.tf b/bootstrap/cluster/variables.tf index 40e8370..6a5eccb 100644 --- a/bootstrap/cluster/variables.tf +++ b/bootstrap/cluster/variables.tf @@ -48,7 +48,6 @@ variable "persistent_volume_dirs" { type = list(string) default = [ "/var/openebs/local/registry", - "/var/openebs/local/gitea", ] } @@ -99,16 +98,16 @@ variable "tailscale_nodeport_access" { node_tailscale_ip = "100.77.80.72" pod_cidr = "10.244.0.0/16" node_port = 30080 - target_port = 80 + target_port = 8080 } } variable "tailscale_nodeport_extra_ports" { type = list(number) - default = [30081, 30300] + default = [30081] } variable "tailscale_nodeport_extra_target_ports" { type = list(number) - default = [3000] + default = [] } diff --git a/bootstrap/edge/variables.tf b/bootstrap/edge/variables.tf index 0d28eff..04adab8 100644 --- a/bootstrap/edge/variables.tf +++ b/bootstrap/edge/variables.tf @@ -55,7 +55,7 @@ variable "demos_backend_port" { variable "gitea_backend_port" { type = number - default = 30300 + default = 3000 } variable "haproxy_stats_user" { diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 359cecb..c6d552c 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -101,15 +101,18 @@ LAB_PIMOX_PIPELINE=true ./lab.sh up ``` Defaults match the observed Pimox template VM shape: OVMF firmware, virtio -networking, virtio-scsi disk, `vmbr0`, `local` template storage, 2 vCPU, and -2 GiB memory. Override `TF_VAR_pimox_template_scsi0`, -`TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or -`TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. +networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with +2 cores, 4 GiB memory, and high-speed CPU affinity `4-5`. Override +`TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, +`TF_VAR_pimox_template_cores`, `TF_VAR_pimox_template_memory`, or +`TF_VAR_pimox_template_cpu_affinity` if the Orange Pi template layout changes. `./lab.sh up` also creates or reuses worker clones after the template exists. It defaults to one worker, VMID `9010`, names like `pimox-worker-01`, deterministic -locally administered MAC addresses, `nvme_thin_pool` clone storage, and -qemu-guest-agent IP discovery. New workers are full clones created with +locally administered MAC addresses, 1 socket with 2 cores, 4 GiB RAM, +Orange Pi 5 high-speed CPU affinity pairs `4-5` and `6-7`, +`nvme_thin_pool` clone storage, and qemu-guest-agent IP discovery. New workers +are full clones created with `qm clone --storage`, so the template can remain on `local` while worker disks land on the NVMe thin pool. The pipeline refuses `LAB_PIMOX_WORKER_STORAGE=local` so only the template VM lives on local storage. Useful overrides: @@ -120,6 +123,7 @@ LAB_PIMOX_WORKER_COUNT=0 ./lab.sh up LAB_PIMOX_WORKER_COUNT=2 ./lab.sh up LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up +LAB_PIMOX_WORKER_CPU_AFFINITIES="4-5 6-7" ./lab.sh up LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up ``` diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index 3684e2f..97a2229 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -129,6 +129,7 @@ resource "null_resource" "pimox_template_vm_create" { name = var.pimox_template_name cores = tostring(var.pimox_template_cores) memory = tostring(var.pimox_template_memory) + cpu_affinity = var.pimox_template_cpu_affinity bridge = var.pimox_template_bridge net0 = local.pimox_template_net0 scsi0 = var.pimox_template_scsi0 @@ -196,6 +197,7 @@ sudo "$qm_cmd" create "$vmid" \ --name "${self.triggers.name}" \ --bios ovmf \ --boot "order=scsi0;net0" \ + --affinity "${self.triggers.cpu_affinity}" \ --cores "${self.triggers.cores}" \ --memory "${self.triggers.memory}" \ --net0 "${self.triggers.net0}" \ diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index 485d7e3..e9efb35 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -196,7 +196,12 @@ variable "pimox_template_cores" { variable "pimox_template_memory" { type = number - default = 2048 + default = 4096 +} + +variable "pimox_template_cpu_affinity" { + type = string + default = "4-5" } variable "pimox_template_bridge" { diff --git a/infra/gitea/README.md b/infra/gitea/README.md new file mode 100644 index 0000000..dad409a --- /dev/null +++ b/infra/gitea/README.md @@ -0,0 +1,26 @@ +# External Gitea + +Gitea is bootstrap infrastructure, not a Kubernetes workload. + +`lab.sh deploy-gitea` copies `docker-compose.yml` to the Raspberry Pi and runs +Gitea as an always-on Docker Compose service. The current default stores data on +the Pi SD card under `/opt/homelab-gitea/data`; move +`LAB_GITEA_INSTALL_DIR` to an SSD mount when the SSD is added. + +Defaults: + +- host: `192.168.100.89` +- user: `jv` +- install dir: `/opt/homelab-gitea` +- HTTP port: `3000` +- SSH port: `32222` +- public root URL: `https://lab2025.duckdns.org/git/` + +Kubernetes consumes Git from the Debian bare GitOps mirror at +`/home/jv/git-server/my-homelab-configs.git`. Gitea is the human-facing Git +service and remains available when the cluster is destroyed. + +Backups are installed on the Debian host by `lab.sh deploy-gitea` and +`lab.sh backup-gitea`. The timer runs `gitea dump` inside the Raspberry Pi +container, copies the archive to Debian, and stores it under +`/home/jv/backups/gitea`. diff --git a/infra/gitea/docker-compose.yml b/infra/gitea/docker-compose.yml new file mode 100644 index 0000000..e5d5176 --- /dev/null +++ b/infra/gitea/docker-compose.yml @@ -0,0 +1,26 @@ +services: + gitea: + image: ${GITEA_IMAGE:-gitea/gitea:1.21.7} + container_name: ${GITEA_CONTAINER_NAME:-homelab-gitea} + restart: unless-stopped + environment: + USER_UID: ${GITEA_UID:-1000} + USER_GID: ${GITEA_GID:-1000} + GITEA__database__DB_TYPE: sqlite3 + GITEA__repository__ENABLE_PUSH_MIRROR: "true" + GITEA__migrations__ALLOW_LOCALNETWORKS: "true" + GITEA__actions__ENABLED: "true" + GITEA__repository__DEFAULT_PRIVATE: public + GITEA__security__INSTALL_LOCK: "true" + GITEA__server__DOMAIN: ${GITEA_DOMAIN:-lab2025.duckdns.org} + GITEA__server__ROOT_URL: ${GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/} + GITEA__server__SERVE_FROM_SUB_PATH: "true" + GITEA__server__SSH_PORT: ${GITEA_SSH_PORT:-32222} + GITEA__server__SSH_LISTEN_PORT: "22" + GITEA__service__DISABLE_REGISTRATION: "true" + GITEA__service__REQUIRE_SIGNIN_VIEW: "false" + ports: + - "${GITEA_HTTP_PORT:-3000}:3000" + - "${GITEA_SSH_PORT:-32222}:22" + volumes: + - ./data:/data diff --git a/lab.sh b/lab.sh index ac0d0bc..e840e43 100755 --- a/lab.sh +++ b/lab.sh @@ -188,6 +188,61 @@ pimox_generated_mac() { $((vmid & 255)) } +cpuset_cpu_count() { + local cpuset="$1" + local count=0 + local part + local start + local end + local -a parts + + IFS=',' read -r -a parts <<<"${cpuset}" + for part in "${parts[@]}"; do + if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then + start="${BASH_REMATCH[1]}" + end="${BASH_REMATCH[2]}" + if ((end < start)); then + return 1 + fi + count=$((count + end - start + 1)) + elif [[ "${part}" =~ ^[0-9]+$ ]]; then + count=$((count + 1)) + else + return 1 + fi + done + + printf '%s\n' "${count}" +} + +pimox_worker_cpu_affinity() { + local index="$1" + local affinities="$2" + local worker_cores="$3" + local affinity + local affinity_index=1 + local cpu_count + + for affinity in ${affinities}; do + if ((affinity_index == index)); then + if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then + echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 + exit 1 + fi + if ((cpu_count != worker_cores)); then + echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2 + exit 1 + fi + printf '%s\n' "${affinity}" + return 0 + fi + affinity_index=$((affinity_index + 1)) + done + + echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2 + exit 1 +} + ensure_pimox_worker_node() { local index="$1" local spec_file="$2" @@ -208,6 +263,7 @@ ensure_pimox_worker_node() { local timeout_seconds="${17}" local qm_bin="${18}" local worker_storage="${19}" + local worker_cpu_affinity="${20}" local padded local vmid local worker_key @@ -228,7 +284,7 @@ ensure_pimox_worker_node() { echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2 exit 1 fi - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${vmid}' --agent enabled=1 + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' --affinity '${worker_cpu_affinity}' if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi" else pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu @@ -250,7 +306,7 @@ if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && fi sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' set '${vmid}' --agent enabled=1 -sudo '${qm_bin}' set '${vmid}' --cores '${worker_cores}' --memory '${worker_memory}' +sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' --affinity '${worker_cpu_affinity}' sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}' sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0' sudo '${qm_bin}' set '${vmid}' --onboot 1 @@ -338,7 +394,8 @@ run_pimox_pipeline() { local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-1}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" - local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-2048}" + local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}" + local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-4-5 6-7}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-nvme_thin_pool}}" local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" @@ -349,6 +406,7 @@ run_pimox_pipeline() { local index local readiness_output local readiness_status + local worker_cpu_affinity if disabled_value "${mode}"; then return 0 @@ -440,6 +498,7 @@ fi" 2>&1)" continue fi + worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" ensure_pimox_worker_node \ "${index}" \ "${spec_file}" \ @@ -459,7 +518,8 @@ fi" 2>&1)" "${ip_prefix}" \ "${timeout_seconds}" \ "${qm_bin}" \ - "${worker_storage}" + "${worker_storage}" \ + "${worker_cpu_affinity}" done write_cluster_worker_var_file "${spec_file}" "${var_file}" @@ -1165,18 +1225,414 @@ wait_for_deployment_ready() { done } -apply_gitea_bootstrap_manifests() { - kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/namespace.yaml" - kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/storage.yaml" - kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/service.yaml" - kubectl --kubeconfig "${KUBECONFIG}" apply -f "${REPO_ROOT}/apps/gitea/deployment.yaml" +deploy_gitea() { + local mode="${LAB_GITEA_DEPLOY:-true}" + local gitea_host="${LAB_GITEA_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local gitea_user="${LAB_GITEA_USER:-${LAB_RASPBERRY_USER:-jv}}" + local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_GITEA_INSTALL_DIR:-/opt/homelab-gitea}" + local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}" + local http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" + local domain="${LAB_GITEA_DOMAIN:-lab2025.duckdns.org}" + local root_url="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}" + local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" + local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml" - wait_for_namespace gitea-system gitea 300 - wait_for_namespaced_resource gitea-system deployment gitea gitea 300 - wait_for_deployment_ready gitea-system gitea gitea 300 + require_debian_server "deploy-gitea" + + if disabled_value "${mode}"; then + install_gitea_backup_timer + return 0 + fi + + if [[ ! -s "${compose_file}" ]]; then + echo "Missing ${compose_file}" >&2 + exit 1 + fi + + echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..." + + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea" + scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml" + + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu +install_dir='${install_dir}' + +install_missing_packages() { + missing_packages='' + for package in \"\$@\"; do + if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then + missing_packages=\"\$missing_packages \$package\" + fi + done + if [ -n \"\$missing_packages\" ]; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends \$missing_packages + fi +} + +install_missing_packages ca-certificates curl iptables + +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sudo sh +fi + +if ! sudo docker compose version >/dev/null 2>&1; then + install_missing_packages docker-compose-plugin +fi + +repair_docker_iptables() { + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...' + sudo systemctl restart docker + sleep 3 + + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo 'Docker NAT chain is still missing after restarting Docker.' >&2 + sudo iptables -t nat -S >&2 || true + sudo systemctl status docker --no-pager -l >&2 || true + exit 1 +} + +repair_docker_iptables + +sudo mkdir -p \"\$install_dir/data\" +sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\" +sudo chown -R 1000:1000 \"\$install_dir/data\" +sudo tee \"\$install_dir/.env\" >/dev/null </dev/null 2>&1; then + openssl rand -hex 32 + return 0 + fi + + python3 - <<'PY' +import secrets + +print(secrets.token_hex(32)) +PY +} + +gitea_api_base_url() { + local gitea_host="$1" + local http_port="$2" + local candidate + local api_base_override="${LAB_GITEA_API_BASE_URL:-}" + + if [[ -n "${api_base_override}" ]]; then + printf '%s\n' "${api_base_override%/}" + return 0 + fi + + for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do + if curl -fsS "${candidate}/version" >/dev/null 2>&1; then + printf '%s\n' "${candidate}" + return 0 + fi + done + + echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2 + exit 1 +} + +gitea_repo_exists() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local status + + status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")" + case "${status}" in + 200) + return 0 + ;; + 404) + return 1 + ;; + 401 | 403) + echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2 + exit 1 + ;; + *) + echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2 + exit 1 + ;; + esac +} + +gitea_branch_exists() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local branch="$6" + local status + + status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")" + case "${status}" in + 200) + return 0 + ;; + 404) + return 1 + ;; + 401 | 403) + echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2 + exit 1 + ;; + *) + echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2 + exit 1 + ;; + esac +} + +create_gitea_repo() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local repo_name="$4" + local default_branch="$5" + local payload + + payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY' +import json +import sys + +repo_name, default_branch = sys.argv[1:3] +print(json.dumps({ + "name": repo_name, + "private": False, + "auto_init": False, + "default_branch": default_branch, + "description": "Homelab infrastructure configuration", +})) +PY +)" + + curl -fsS \ + -u "${auth_user}:${auth_password}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "${payload}" \ + "${api_base}/user/repos" >/dev/null +} + +bootstrap_gitea_repo() { + local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}" + local gitea_host="${LAB_GITEA_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local gitea_user="${LAB_GITEA_USER:-${LAB_RASPBERRY_USER:-jv}}" + local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" + local http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local root_url="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}" + local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}" + local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}" + local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}" + local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}" + local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}" + local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}" + local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}" + local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}" + local api_base + local public_repo_url + local direct_repo_url + local push_url + local askpass + local credentials_dir + local remote_status + local worktree_status + + require_debian_server "bootstrap-gitea-repo" + + if disabled_value "${mode}"; then + return 0 + fi + + ensure_python3 + for value_name in repo_owner repo_name default_branch bootstrap_user; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "${value_name} contains unsupported characters." >&2 + exit 1 + fi + done + if [[ "${bootstrap_email}" == *"'"* ]]; then + echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2 + exit 1 + fi + + if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then + # shellcheck disable=SC1090 + source "${credentials_file}" + bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}" + bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}" + bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}" + fi + + if [[ -z "${bootstrap_password}" ]]; then + bootstrap_password="$(gitea_bootstrap_password)" + credentials_dir="$(dirname "${credentials_file}")" + mkdir -p "${credentials_dir}" + chmod 0700 "${credentials_dir}" + { + printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}" + printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}" + printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}" + } > "${credentials_file}" + chmod 0600 "${credentials_file}" + echo "Generated Gitea bootstrap credentials at ${credentials_file}." + fi + + for value_name in repo_owner repo_name default_branch bootstrap_user; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "${value_name} contains unsupported characters." >&2 + exit 1 + fi + done + for value_name in bootstrap_email bootstrap_password; do + local value="${!value_name}" + if [[ "${value}" == *"'"* ]]; then + echo "${value_name} cannot contain a single quote." >&2 + exit 1 + fi + done + + echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..." + + # shellcheck disable=SC2087 + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" </dev/null 2>&1; then + echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2 + exit 1 +fi + +for attempt in \$(seq 1 60); do + if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 || + curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then + break + fi + if [ "\${attempt}" = "60" ]; then + echo "Timed out waiting for Gitea API inside \${container_name}." >&2 + exit 1 + fi + sleep 2 +done + +if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \ + --username "\${bootstrap_user}" \ + --password "\${bootstrap_password}" \ + --email "\${bootstrap_email}" \ + --admin \ + --must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then + if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then + cat /tmp/homelab-gitea-user-create.log >&2 + exit 1 + fi +fi +EOF + + api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")" + + if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then + echo "Gitea repository ${repo_owner}/${repo_name} already exists." + else + if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then + echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2 + exit 1 + fi + create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}" + echo "Created Gitea repository ${repo_owner}/${repo_name}." + fi + + public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git" + if [[ "${api_base}" == */git/api/v1 ]]; then + direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git" + else + direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git" + fi + push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}" + + git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null + git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null || + git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}" + + if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then + echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged." + else + worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)" + if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then + echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2 + echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2 + exit 1 + fi + + askpass="$(mktemp)" + trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT + cat > "${askpass}" </dev/null 2>&1; then - echo "kubectl is required for Gitea backups." >&2 - exit 1 -fi - -pod="\$(kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" get pods \ - -l "\${GITEA_SELECTOR}" \ - --field-selector=status.phase=Running \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" - -if [[ -z "\${pod}" ]]; then - echo "Skipping Gitea backup: no running Gitea pod found." - exit 0 -fi - timestamp="\$(date -u +%Y%m%dT%H%M%SZ)" tmp_archive="\$(mktemp "/tmp/gitea-\${timestamp}.XXXXXX.zip")" backup_archive="\${GITEA_BACKUP_DIR}/gitea-\${timestamp}.zip" +remote_host_archive="/tmp/gitea-\${timestamp}.zip" + +ssh_gitea() { + ssh -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "\${GITEA_USER}@\${GITEA_HOST}" "\$@" +} cleanup() { rm -f "\${tmp_archive}" - kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- rm -f "\${REMOTE_ARCHIVE}" >/dev/null 2>&1 || true + ssh_gitea "rm -f '\${remote_host_archive}'; sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true } trap cleanup EXIT -kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- rm -f "\${REMOTE_ARCHIVE}" >/dev/null 2>&1 || true -kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- \ - sh -c 'mkdir -p /data/git/repositories && chown git:git /data/git /data/git/repositories' -kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" exec "\${pod}" -c "\${GITEA_CONTAINER}" -- \ - su-exec git gitea dump -c /data/gitea/conf/app.ini --file "\${REMOTE_ARCHIVE}" -kubectl --kubeconfig "\${KUBECONFIG_PATH}" -n "\${GITEA_NAMESPACE}" cp -c "\${GITEA_CONTAINER}" \ - "\${GITEA_NAMESPACE}/\${pod}:\${REMOTE_ARCHIVE}" "\${tmp_archive}" +ssh_gitea "set -eu +sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true +sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories' +sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}' +sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}' +sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}' +sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" + +scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}" sudo mkdir -p "\${GITEA_BACKUP_DIR}" -sudo install -m 0640 -o root -g root "\${tmp_archive}" "\${backup_archive}" +sudo chown jv:jv "\${GITEA_BACKUP_DIR}" +sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}" sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete echo "Created \${backup_archive}" @@ -1240,7 +1684,7 @@ BACKUP_SCRIPT_EOT sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT' [Unit] -Description=Back up in-cluster Gitea to Debian host storage +Description=Back up external Homelab Gitea to Debian host storage After=network-online.target Wants=network-online.target @@ -1266,8 +1710,8 @@ TIMER_EOT #!/usr/bin/env bash set -euo pipefail -GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/var/backups/homelab/gitea}" -GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/var/backups/homelab/gitea-restore-drills}" +GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" +GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}" GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}" if ! command -v python3 >/dev/null 2>&1; then @@ -1380,7 +1824,6 @@ RESTORE_DRILL_TIMER_EOT backup_gitea() { require_debian_server "backup-gitea" - export KUBECONFIG="${KUBECONFIG_PATH}" install_gitea_backup_timer sudo /usr/local/sbin/homelab-gitea-backup.sh } @@ -1542,12 +1985,10 @@ apps() { echo "Deploying homelab applications..." - apply_gitea_bootstrap_manifests run_tofu_stack "bootstrap/apps" refresh_argocd_application container-registry refresh_argocd_application demos-static - refresh_argocd_application gitea refresh_argocd_application website-production wait_for_namespace container-registry container-registry 300 @@ -1634,11 +2075,12 @@ up() { echo "Deploying the homelab infrastructure..." + deploy_gitea + bootstrap_gitea_repo run_pimox_pipeline run_openwrt_pipeline run_tofu_stack "bootstrap/cluster" run_tofu_stack "bootstrap/platform" - install_gitea_backup_timer apps run_tofu_stack "bootstrap/edge" @@ -1793,6 +2235,12 @@ case "${1:-}" in apps) apps ;; + deploy-gitea) + deploy_gitea + ;; + bootstrap-gitea-repo) + bootstrap_gitea_repo + ;; backup-gitea) backup_gitea ;; @@ -1806,7 +2254,8 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|apps|backup-gitea|drill-gitea-restore|install-gitea-runner|nuke}" + echo "Usage: $0 {up|apps|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|nuke}" exit 1 ;; esac + diff --git a/renovate.json b/renovate.json index 0bfdfdb..57f03e5 100644 --- a/renovate.json +++ b/renovate.json @@ -41,6 +41,7 @@ "description": "Do not automerge homelab infrastructure updates.", "matchFileNames": [ "bootstrap/**", + "infra/gitea/**", "lab.sh", ".gitea/workflows/**" ],