From 84855cd9277aa9fcbe45cc7d6b20f010808eb44b Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Sat, 27 Jun 2026 15:42:59 -0600 Subject: [PATCH] Add generated docs config --- .gitea/workflows/homelab-main.yml | 18 +- README.md | 4 +- README.md.tmpl | 834 ++++++++++++++++++++++++++++++ apps/website/blog.php | 14 +- apps/website/homelab-tree.php | 2 +- apps/website/lang/en.php | 6 +- apps/website/lang/nah.php | 6 +- bootstrap/host/README.md | 4 +- bootstrap/provisioning/README.md | 32 +- docs.yml | 1 + infra/arr-stack/README.md | 2 +- infra/arr-stack/docs/bootstrap.md | 2 +- infra/gitea/README.md | 8 +- infra/youtube-backup/README.md | 2 +- renovate.json | 2 +- scripts/render-docs | 108 ++++ 16 files changed, 998 insertions(+), 47 deletions(-) create mode 100644 README.md.tmpl create mode 100644 docs.yml create mode 100755 scripts/render-docs diff --git a/.gitea/workflows/homelab-main.yml b/.gitea/workflows/homelab-main.yml index 3c91dc7..ba91b18 100644 --- a/.gitea/workflows/homelab-main.yml +++ b/.gitea/workflows/homelab-main.yml @@ -21,6 +21,12 @@ jobs: git checkout --force "${{ gitea.sha }}" git config --global --add safe.directory "$PWD" + - name: Check generated docs + run: | + set -euo pipefail + + scripts/render-docs --check + - name: Block automatic deploy for external Gitea changes run: | set -euo pipefail @@ -38,14 +44,14 @@ jobs: printf '%s\n' "${changed_files}" blocked_files="$(printf '%s\n' "${changed_files}" | grep -E '^infra/gitea/' || true)" - if printf '%s\n' "${changed_files}" | grep -qx 'lab.sh' && + if printf '%s\n' "${changed_files}" | grep -qx 'jeannie' && python3 - "${base_ref}" <<'PY' import re import subprocess import sys base_ref = sys.argv[1] - path = "lab.sh" + path = "jeannie" target_functions = { "deploy_gitea", "install_gitea_backup_timer", @@ -109,7 +115,7 @@ jobs: sys.exit(1) PY then - blocked_files="${blocked_files}"$'\n'"lab.sh" + blocked_files="${blocked_files}"$'\n'"jeannie" fi blocked_files="$(printf '%s\n' "${blocked_files}" | sed '/^$/d' | sort -u)" @@ -160,7 +166,7 @@ jobs: if [[ "${action_command}" == "auto" ]]; then if [[ -s "${kubeconfig_probe}" ]]; then set +e - version_report="$("${deploy_dir}/lab.sh" doctor-versions 2>&1)" + version_report="$("${deploy_dir}/jeannie" doctor-versions 2>&1)" version_status=$? set -e printf '%s\n' "${version_report}" @@ -181,8 +187,8 @@ jobs: case "${action_command}" in apps | rebuild-cluster) - echo "Running lab.sh ${action_command}" - "${deploy_dir}/lab.sh" "${action_command}" + echo "Running jeannie ${action_command}" + "${deploy_dir}/jeannie" "${action_command}" ;; *) echo "Unsupported HOMELAB_ACTION_COMMAND: ${action_command}" >&2 diff --git a/README.md b/README.md index 0a40326..1406612 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Homelab Kubernetes Pipeline + + This repo bootstraps a hybrid kubeadm cluster and then hands app delivery to Argo CD. @@ -9,7 +11,7 @@ The lab is intentionally small but production-shaped: - a Debian amd64 host runs the kubeadm control plane and local deployment tools - a Raspberry Pi arm64 node runs selected workloads -- the Raspberry Pi also runs the always-on Gitea Docker service outside +- the Debian host also runs the always-on Gitea Docker service outside Kubernetes - the Debian host keeps a bare GitOps mirror under `/home/jv/git-server/my-homelab-configs.git` diff --git a/README.md.tmpl b/README.md.tmpl new file mode 100644 index 0000000..44051dc --- /dev/null +++ b/README.md.tmpl @@ -0,0 +1,834 @@ +# Homelab Kubernetes Pipeline + + + +This repo bootstraps a hybrid kubeadm cluster and then hands app delivery to +Argo CD. + +## Architecture + +The lab is intentionally small but production-shaped: + +- a Debian amd64 host runs the kubeadm control plane and local deployment tools +- a Raspberry Pi arm64 node runs selected workloads +- the Debian host also runs the always-on Gitea Docker service outside + Kubernetes +- the Debian host keeps a bare GitOps mirror under + `/home/jv/git-server/my-homelab-configs.git` +- a provisioning layer can PXE boot Debian 13 arm64 VMs for Pimox worker + templates +- OpenTofu owns the bootstrap layers for cluster, platform, apps, and edge +- Argo CD continuously reconciles Kubernetes manifests from this repo +- a local registry stores the website and demos images built for the worker + architecture +- the website image keeps Apache as the public server, uses PHP-FPM with + OPcache for dynamic endpoints, and pre-renders read-heavy pages to static + HTML during the image build +- SOPS with age is the committed secret-management path for future encrypted + Kubernetes secrets +- an OCI jump box provides the public edge path back into the homelab over + Tailscale, with nginx caching, gzip, HTTP/2, TLS session reuse, and upstream + keepalive + +Run `./{{ main_script }} up` and `./{{ main_script }} nuke` only from the Debian homelab server. The +script intentionally refuses to run from non-Debian machines so a laptop cannot +accidentally modify the cluster. + +## Flow + +1. `bootstrap/provisioning` + - prepares a Debian server as a PXE and preseed service for arm64 VMs + - serves Debian 13 arm64 netboot assets through TFTP and HTTP + - creates a golden image install path with Kubernetes, containerd, + qemu-guest-agent, cloud-init, and storage client packages ready + - is driven by `./{{ main_script }} up` when Pimox is reachable, without changing + Orange Pi host networking + +2. `bootstrap/cluster` + - creates the kubeadm control plane on the Debian amd64 node + - joins worker nodes such as Raspberry Pi and Pimox Debian arm64 nodes + - configures Calico-compatible pod CIDR + - configures containerd to pull from the in-cluster NodePort registry + - creates retained host directories under `/data/openebs/local` + +3. `bootstrap/platform` + - installs a minimal Calico deployment through the Tigera operator + - installs NodeLocal DNSCache for node-local DNS query caching + - installs MetalLB for LAN `LoadBalancer` services + - installs Traefik as the single Kubernetes ingress entry point + - installs OpenEBS + - creates `openebs-hostpath-retain` + - installs Argo CD + - installs Kyverno with audit-first baseline Pod Security policies and + homelab image signature/SBOM admission controls + - registers the private GitOps repo without storing the SSH private key in + Terraform state + +4. `bootstrap/apps` + - registers Argo CD Applications from the `applications` map + - passes the website image produced by the build step into Argo CD as a + Kustomize image override + - deploys the tuned website runtime: Apache listens on port 8080, PHP-FPM + listens on loopback inside the container, OPcache is enabled, and + pre-rendered HTML handles the read-heavy pages + - default apps are `container-registry`, `website-production`, + `demos-static`, `heimdall`, `n8n`, and `supply-chain-policy` + +5. `bootstrap/edge` + - connects to the OCI jump box + - uploads nginx, HAProxy, Varnish, and Squid configs + - obtains and renews Let's Encrypt certificates for the configured hostname + - runs the edge cache/proxy chain with Docker Compose + - enables gzip, HTTP/2, TLS session reuse, client keepalive, upstream + keepalive, and cache headers for the public website path + +## Prerequisites + +On the Debian host: + +- OpenTofu +- Docker with Buildx +- kubeadm, kubelet, kubectl, and containerd pinned to the target Kubernetes + minor, currently `v1.36` +- SSH access to worker nodes +- SSH access to the OCI edge host +- enough persistent storage for `/data/openebs/local` and `/data/docker` + +After a clean Debian reinstall, bootstrap the host package set and desktop +configuration first: + +```bash +sudo apt-get update +sudo apt-get install -y git ansible sudo +git clone /home/jv/my-homelab-configs +cd /home/jv/my-homelab-configs +ansible-playbook -K -i bootstrap/host/inventory.ini bootstrap/host/playbook.yml +``` + +If you kept the private app-config archive generated from the old Debian host, +pass its generated `ansible-vars.yml` with `-e @/path/to/ansible-vars.yml`. +Details are in `bootstrap/host/README.md`. + +The default kubeconfig path is `/home/jv/.kube/config`. Override it with +`KUBECONFIG_PATH` or `TF_VAR_kubeconfig_path` when needed. + +## Version Discipline + +The lab should converge on one Kubernetes minor across the API server and every +kubelet. The Pimox golden-image default is `v1.36` so rebuilt arm64 VM workers +match the Debian control plane and Raspberry Pi worker instead of staying on an +older template. Avoid treating a three-minor kubelet skew as normal steady +state: it is only a temporary compatibility window and blocks the next control +plane upgrade. + +Check node versions from the Debian host: + +```bash +./{{ main_script }} doctor-versions +``` + +The check prints the API server, kubelet versions, kubelet minors, and containerd +runtimes. It exits nonzero when kubelet minors differ from the API server minor +or when containerd major versions are mixed. The Gitea Actions auto path runs +this check before app deploys; kubelet minor drift switches the job to +`./{{ main_script }} rebuild-cluster` so the Pimox template and worker VMs are replaced +from the pinned image. For Pimox workers, prefer rebuilding from the corrected +golden template and rejoining the worker over ad hoc live package upgrades. + +## Deploying + +From the Debian server: + +```bash +cd ~/my-homelab-configs +./{{ main_script }} up +``` + +The script first deploys external Gitea to the Debian host with Docker Compose +under `/data/homelab-gitea`, backed by the HP laptop NVMe, so Git stays +outside the Kubernetes rebuild blast radius. It then detects the +Pimox host at `192.168.100.80` in auto mode. When SSH, `qm`, and `vmbr0` are +available, it applies `bootstrap/provisioning`, creates or reuses the Debian 13 +arm64 template, creates or reuses one worker VM clone, discovers the guest IP +through qemu-guest-agent, and passes that worker into the cluster layer. It then +applies the remaining OpenTofu stacks, refreshes Argo CD apps, waits for the +local registry, builds the website and demos images when their source changed, +pushes them to the registry, recreates pods only after a new image is built, and +applies the edge stack. + +Set `LAB_PIMOX_PIPELINE=false` to skip Pimox automation. Set +`LAB_PIMOX_WORKER_COUNT=0` to create or refresh only the template. The pipeline +keeps the template on its configured `local` storage, creates new worker VM +clones on `opi5_ssd` by default when that Pimox storage is available, +checks that the Pimox bridge already exists, refuses `local` as worker clone +storage, and refuses to edit Orange Pi host networking. Keep durable data on +the HP Debian laptop's `data-vg` or external backups; treat Orange Pi VM disks +as rebuildable worker capacity. + +`LAB_PIMOX_SKIP_WORKER_INDEXES` defaults to an empty value, so the pipeline owns +worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set +`LAB_PIMOX_SKIP_WORKER_INDEXES=1` if an existing manually created first worker +must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID +`9010` and VMID `9011`. + +OpenWrt firewall VM automation is available as a standalone command because it +attaches to both WAN and LAN bridges. Run `./{{ main_script }} openwrt` after `vmbr1` +already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM +SystemReady EFI image, writes basic WAN/LAN/firewall config into the image, +imports it as VM `9100`, attaches `vmbr0` as WAN and `vmbr1` as LAN, and stores +the VM disk on the selected non-local Pimox storage. It leaves the VM stopped +and not enabled for host boot by default. It does not use the Debian Kubernetes +golden-node template for OpenWrt. + +The website and demos images default to `linux/amd64,linux/arm64` because both +deployments can schedule on either the Debian host or ARM workers. Override with +`WEBSITE_IMAGE_PLATFORMS` or `DEMOS_IMAGE_PLATFORMS` only when intentionally +restricting node placement. + +Build metadata is written under `.lab/` so repeat runs can skip the website +or demos image build when the source hash, platform, image reference, and +registry manifest still match. + +The website build is intentionally performed on the Debian homelab path and the +Gitea Actions runner, not on a laptop. The current image uses +`php:8.5-fpm-alpine`, installs Apache, routes PHP through PHP-FPM, enables +OPcache for immutable image deploys, and runs +`apps/website/render-static-pages.sh` during the image build. That render step +produces static HTML for `/`, `/cv.php`, `/blog.php`, `/demos.php`, and +`/homelab-tree.php` in the static languages. Runtime PHP remains for write and +translation endpoints such as `save_idea.php`, `visitor_ideas.php`, +`translate.php`, and `save_lang.php`. + +Set `LAB_GITEA_DEPLOY=false` to skip the external Gitea deployment step when the +service is already managed manually. The default Gitea target is +`jv@192.168.100.73`, install directory `/data/homelab-gitea`, HTTP port `3000`, +and SSH port `32222`. + +## Validation + +Useful checks after a rebuild: + +```bash +export KUBECONFIG=/home/jv/.kube/config + +kubectl get nodes +kubectl -n argocd get applications +kubectl -n container-registry get pods +kubectl -n website-production get pods -o wide +kubectl -n demos-static get pods -o wide +kubectl -n heimdall get pods -o wide + +ssh jv@192.168.100.73 'cd /data/homelab-gitea && sudo docker compose ps' + +docker info --format '{{.DockerRootDir}}' +df -h / /data /data/openebs/local /data/docker +``` + +The website should be reached through the configured public hostname, not the raw +OCI IP address, because the Let's Encrypt certificate is issued for the +hostname. + +Run a read-only health snapshot from the Debian server with: + +```bash +./{{ main_script }} status +``` + +It reports host memory/disk, systemd services, Docker Compose stacks, +Kubernetes health when the API is reachable, Pimox worker VM status, Tailscale, +and key local/public HTTP endpoints. + +## Adding Nodes + +For Pimox on Orange Pi 5 Plus, `./{{ main_script }} up` can create the Debian 13 arm64 +template and worker VM clones automatically when the Orange Pi storage path is +healthy. Defaults are intentionally tied to the observed host: Pimox SSH host +`192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two +4 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, +and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe +that was previously attached to the Orange Pi now belongs to the HP Debian +laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox +datastore is `opi5_ssd` and is used for rebuildable VM root disks. Details and +override variables are in +`bootstrap/provisioning/README.md`. + +Worker indexes are stable. Index `1` maps to VMID `9010`, node name +`pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`, and +so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while +allowing higher indexes to be automated. + +Run a full cluster rebuild from the Debian server with: + +```bash +./{{ main_script }} rebuild-cluster +``` + +That path preserves external Gitea, rebuilds the Pimox template +with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and +4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is +disabled by default because the Bullseye-pinned Pimox `qm` does not support it. +The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` +does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. + +To pause the Kubernetes cluster without deleting kubeadm files, OpenTofu state, +PV data, or VM disks, run: + +```bash +./{{ main_script }} stop-cluster +``` + +This stops `kubelet`, stops Kubernetes CRI containers with `crictl` when +available, stops any workers listed in `WORKER_SSH_TARGETS`, and gracefully +shuts down automated Pimox worker VMs. It intentionally leaves the host +`containerd`/Docker stack running so external Docker Compose services such as +Gitea stay online. It does not run `kubeadm reset`, delete CNI files, or remove +state. Use `LAB_CLUSTER_STOP_CONTAINERD=true` only when you intentionally want +to stop the host container runtime too. Use `LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS` +to change the Pimox shutdown timeout, `LAB_CLUSTER_STOP_FORCE=false` to avoid a +hard `qm stop` after timeout, and the usual `LAB_PIMOX_WORKER_COUNT`, +`LAB_PIMOX_WORKER_BASE_VMID`, and `LAB_PIMOX_SKIP_WORKER_INDEXES` variables to +select VM workers. Resume the runtime with: + +```bash +./{{ main_script }} start-cluster +``` + +To exclude the Raspberry Pi from the Kubernetes cluster, set +`LAB_INCLUDE_RASPBERRY_WORKER=false`. To manage workers manually instead, add +entries to +`bootstrap/cluster/variables.tf` or a `.tfvars` file: + +```hcl +worker_nodes = { + raspberrypi = { + host = "192.168.100.89" + user = "jv" + node_name = "raspberry" + ssh_key_path = "/home/jv/.ssh/id_ed25519" + } +} +``` + +Stateful apps currently pin retained local PVs to the `debian` node. Move or +duplicate those PV manifests when you want storage on another node. + +## Workload Placement + +`bootstrap/cluster` labels nodes with homelab placement metadata: + +- `node-role.kubernetes.io/worker=worker` on every worker so `kubectl get nodes` + shows `worker` instead of `` in the ROLES column +- `homelab.dev/node-role=control-plane`, `homelab.dev/storage=local`, and + `homelab.dev/workload-class=control-plane` on the Debian control plane +- `homelab.dev/node-role=edge-app`, `homelab.dev/storage=local`, and + `homelab.dev/workload-class=edge` on the Raspberry Pi worker +- `homelab.dev/node-role=app`, `homelab.dev/storage=ssd`, and + `homelab.dev/workload-class=platform` on automated Pimox worker clones when + those workers are enabled + +Override `control_plane_node_labels`, `worker_node_labels`, +`LAB_RASPBERRY_NODE_LABELS_JSON`, or `LAB_PIMOX_WORKER_NODE_LABELS_JSON` when +the physical layout changes. The current website, demos, and registry manifests +are not moved automatically because the public NodePort path and retained +OpenEBS hostpath PVs are node-local. Move workloads only after their storage and +edge path are ready on the target node. Gitea is outside Kubernetes and is moved +by changing the Raspberry Pi Docker install target instead. + +The stateless platform controllers are pinned to Pimox worker nodes through +`homelab.dev/workload-class=platform` and include hostname topology spread plus +preferred pod anti-affinity so future Argo CD, Kyverno, Prometheus operator, and +kube-state-metrics scheduling does not collapse onto the first worker that joins. +PVC-backed monitoring StatefulSets are intentionally treated separately because +their retained OpenEBS hostpath volumes are node-local. Run +`./{{ main_script }} move-prometheus-stack-workers` from the Debian host to label existing +worker nodes, destroy only the existing `prometheus-stack` Helm release, delete +its retained PVC/PV objects, and recreate the stack on the worker selector when +you intentionally accept losing that monitoring data. A planned monitoring data +migration should be handled as a separate maintenance task with backup, +delete/recreate or storage migration steps, and post-restore checks. + +The older NodePort path is now reserved for special cases such as the local +registry. `bootstrap/cluster` still contains `homelab-tailscale-nodeport` +support, but app traffic should normally enter through Traefik's MetalLB +address instead of per-app NodePorts. The cluster stack advertises the LAN +subnet from the configured Tailscale worker so the OCI edge can route to the +Traefik LoadBalancer address: + +```hcl +metallb = { + enabled = true + repository = "https://metallb.github.io/metallb" + version = "0.16.0" + namespace = "metallb-system" + address_pool = ["192.168.100.240-192.168.100.240"] + l2_advertisement_enabled = true + pool_name = "homelab-lan" +} + +traefik = { + enabled = true + repository = "https://helm.traefik.io/traefik" + chart_version = "40.2.0" + namespace = "traefik" + load_balancer_ip = "192.168.100.240" + ingress_class = "traefik" +} + +tailscale_subnet_routes = { + enabled = true + worker_key = "raspberrypi" + routes = ["192.168.100.0/24"] +} +``` + +DuckDNS resolves `*.lab2025.duckdns.org` to the OCI edge, so public requests for +the service hostnames land on the same edge host. For direct LAN testing, point +LAN DNS, `/etc/hosts`, or a Tailscale DNS override for app hostnames at +Traefik's address: + +```text +192.168.100.240 lab2025.duckdns.org +192.168.100.240 demos.lab2025.duckdns.org +192.168.100.240 heimdall.lab2025.duckdns.org +192.168.100.240 n8n.lab2025.duckdns.org +192.168.100.240 prowlarr.lab2025.duckdns.org +192.168.100.240 sonarr.lab2025.duckdns.org +192.168.100.240 radarr.lab2025.duckdns.org +192.168.100.240 qbittorrent.lab2025.duckdns.org +192.168.100.240 kapowarr.lab2025.duckdns.org +192.168.100.240 suwayomi.lab2025.duckdns.org +192.168.100.240 maintainerr.lab2025.duckdns.org +192.168.100.240 grafana.lab2025.duckdns.org +192.168.100.240 argocd.lab2025.duckdns.org +``` + +The edge stack uses Traefik as its backend by default and validates +`http://192.168.100.240:80/` before updating the edge containers. If Tailscale +subnet-route approval is not automatic in the tailnet policy, the edge deploy +will fail clearly instead of silently keeping a broken public path. + +For `./{{ main_script }} nuke`, set `WORKER_SSH_TARGETS` to a space-separated list of +remote SSH targets when more worker nodes exist. Set it to an empty string for a +single-node rebuild. + +## Adding Platform Tools + +Add Helm releases through `bootstrap/platform`'s `extra_helm_releases` map. + +## Policy Guardrails + +`bootstrap/platform` installs Kyverno and the upstream baseline Pod Security +policies in `Audit` mode. This gives the lab policy reports for unsafe workload +settings without blocking existing pods during the first rollout. After reports +are clean, individual policies can be promoted to `Enforce` in +`bootstrap/platform/main.tf`. + +`apps/supply-chain-policy` adds a separate Kyverno `ImageValidatingPolicy` for +the images built by this repo and pushed to the local registry. The policy +targets `192.168.100.73:30500/php-website:*` and +`192.168.100.73:30500/demos-static:*`, mutates admitted pods to image digests, +and audits whether the image has both: + +- a valid Cosign signature from the homelab signing key +- a signed SPDX SBOM attestation + +The private Cosign key, generated password, and downloaded Cosign binary live in +`${XDG_DATA_HOME:-$HOME/.local/share}/homelab/` by default so the signing +identity survives temporary checkout cleanup. Repo-local image state files still +live under `.lab/`, which is ignored by Git. `./{{ main_script }} apps` creates or reuses +the persistent Cosign key, publishes the public key into the Kyverno namespace as +`homelab-cosign-public-key`, builds images with BuildKit SBOM attestations, signs +the pushed images, attaches a signed SPDX SBOM predicate, and verifies both +artifacts before refreshing the workload Applications. The policy starts in +`Audit` mode so an existing live deployment can recover while the current images +are signed; after `./{{ main_script }} apps` verifies both images, change `validationActions` in +`apps/supply-chain-policy/local-registry-image-policy.yaml` from `Audit` to +`Deny` to make it admission-enforcing. Set `COSIGN_PASSWORD` if you want to +manage the key password externally; otherwise the Debian runner creates +`cosign/cosign.password` under the homelab state directory for non-interactive +runs. Set `COSIGN_KEY_PREFIX`, `COSIGN_PASSWORD_FILE`, or `COSIGN_BIN` to override +those paths. + +## DNS Cache + +`bootstrap/platform` installs NodeLocal DNSCache in `kube-system` with +`registry.k8s.io/dns/k8s-dns-node-cache`. The default listens on +`169.254.20.10` and the kube-dns service IP `10.96.0.10`, which keeps the +rollout compatible with the current kube-proxy iptables path without rewriting +kubelet DNS settings across the nodes. Override `nodelocal_dns` if the service +CIDR or upstream DNS servers change. + +## MetalLB + +MetalLB is present in `bootstrap/platform` but disabled by default. Enable it +only after reserving a LAN IP range outside DHCP and outside any future OpenWrt +LAN pool: + +```bash +export TF_VAR_metallb='{ + enabled = true + repository = "https://metallb.github.io/metallb" + version = "0.16.0" + namespace = "metallb-system" + address_pool = ["192.168.100.240-192.168.100.250"] + l2_advertisement_enabled = true + pool_name = "homelab-lan" +}' +``` + +Traefik uses MetalLB for a LAN `LoadBalancer` address. App services such as the +website, demos, and Heimdall should be `ClusterIP` services behind Kubernetes +Ingress objects. The local registry remains a `NodePort` because the cluster +nodes use it as a pull endpoint. Gitea is not a Kubernetes service; it runs on +the Raspberry Pi Docker host. + +## Secrets + +Use SOPS with age for secrets that need to live in Git. Start from +`.sops.yaml.example`, replace the age recipient with the public key generated on +the Debian host, and commit the resulting `.sops.yaml`. Keep the private age key +outside the repo. Operational notes are in `docs/secrets.md`. + +## Edge Services + +The OCI jump box runs the public edge path: + +```text +nginx -> HAProxy -> Varnish/Squid -> Traefik MetalLB IP +``` + +The `bootstrap/edge` stack renders configs from `bootstrap/edge/templates` and +deploys them to `/opt/homelab-edge` on the OCI host. Defaults are in +`bootstrap/edge/variables.tf`; override them through `TF_VAR_*` or a `.tfvars` +file when the public host, SSH key, server name, backend Tailscale IP, or +Traefik backend address changes. + +The `/git/` route is intentionally different from the Kubernetes app routes: it +proxies to Gitea on the Raspberry Pi at the configured `gitea_backend_port` +instead of Traefik. This keeps public read-only source browsing available even +when the cluster has been destroyed. + +For the main website, nginx terminates TLS, serves cached HTML and static +assets, compresses text responses with gzip, advertises HTTP/2, reuses TLS +sessions, and keeps upstream connections open to the Traefik backend. The +application side marks pre-rendered HTML as +`Cache-Control: public, max-age=300, s-maxage=3600`; the edge keeps CSS and JS +under the stronger immutable asset policy. In local measurements, cached +single-resource requests are dominated by the remote TCP/TLS path, while a +reused HTTP/2 asset batch completes in roughly 60 ms after the connection is +established. + +Use the configured `server_name` in the browser, for example +`https://lab2025.duckdns.org`. A raw OCI IP address will still show a browser +certificate warning because the trusted certificate is issued for the hostname. + +The edge stack uses HTTP-01 validation and requests one certificate covering +`server_name` plus `additional_server_names`. DuckDNS resolves sub-subdomains +under `lab2025.duckdns.org` to the same edge IP, so inbound TCP 80 and 443 must +be open before `./{{ main_script }} up` runs. Set `TF_VAR_letsencrypt_email` to receive +expiry notices, or leave it empty to register without an email. Set +`TF_VAR_enable_letsencrypt=false` to keep using the temporary local certificate. + +## Adding Apps + +Add Kubernetes manifests under `apps/` and register them in +`bootstrap/apps`'s `applications` map. Argo CD will own sync, pruning, and +self-healing for the app. + +The `heimdall` app is intentionally waited on at the end of `./{{ main_script }} apps`. +It runs the LinuxServer.io Heimdall dashboard, persists `/config` on +OpenEBS, and seeds tiles for the website, demo apps, Gitea, Grafana, +Prometheus, Alertmanager, Argo CD, the local registry, and Heimdall itself. +Its manifest only selects Linux nodes; the bound OpenEBS local PV supplies the +node affinity. Do not also pin it to a workload class unless the retained PV is +rebuilt on a node with that label. +Because Heimdall does not support reverse-proxy subfolder hosting cleanly, it +is exposed through the dedicated hostname `heimdall.lab2025.duckdns.org` rather +than a `/heimdall/` path. + +The `n8n` app runs in the `n8n` namespace with retained OpenEBS storage and is +exposed at `https://n8n.lab2025.duckdns.org`. It receives +`OLLAMA_BASE_URL=http://192.168.100.73:11434` so workflows can call the existing +Debian-host Ollama API for translation prewarming and batch jobs. + +The ARR/media stack is intentionally not deployed by the Kubernetes app +pipeline. It is better suited to a Docker Compose stack on the Debian host with +persistent volumes under `/data`, where large media paths, qBittorrent writes, +and service config directories can survive cluster rebuilds without OpenEBS +node-affinity or PVC migration concerns. The legacy `apps/arr-stack` manifests +are retained only as migration reference. The active host-level Compose +definition lives under `infra/arr-stack` and stores persistent data under +`/data/arr`. + +## Storage + +OpenEBS provides the platform storage provisioner. Stateful Kubernetes apps use +retained local PV paths such as `/data/openebs/local/registry`; these paths are +intentionally outside kubeadm reset paths so data can survive cluster +destroy/create cycles. Those critical volumes are declared explicitly as +retained local PVs so a rebuilt cluster binds back to the same host paths +instead of creating fresh directories. + +For the current lab, the HP Debian laptop's NVMe is managed by LVM volume group +`data-vg` and mounted under `/data`. OpenEBS retained hostpath data lives under +`/data/openebs/local`, and Docker data should live under `/data/docker` where +the host is configured that way. This keeps retained PVs, container layers, +Buildx state, and image caches from filling `/`. + +## Gitea + +Gitea is external bootstrap infrastructure. It runs on the Debian host as an +always-on Docker Compose service from `infra/gitea/docker-compose.yml`, not as a +Kubernetes workload. This keeps Git available when the Kubernetes cluster is +destroyed and rebuilt. + +The default data path is `/data/homelab-gitea/data` on the HP laptop NVMe. The +Docker root is also expected to live on the `/data` storage path rather than the +root filesystem. + +Public source browsing stays available through +`https://lab2025.duckdns.org/git/`. Registration is disabled and anonymous users +can view public repositories, so the blog can link to code read-only while +writes still require an authenticated Gitea account. + +The Debian bare repo remains the GitOps mirror: + +```text +/home/jv/git-server/my-homelab-configs.git +``` + +Argo CD consumes that Debian mirror through the default `gitops_repo_url`. +Gitea Actions pushes the `main` commit into the mirror before running the +selected deploy command. + +The platform bootstrap registers the Argo CD repository secret and the SSH host +key for the Debian GitOps mirror. If Argo CD reports +`knownhosts: key is unknown` after the Debian host was rebuilt or its SSH host +key changed, refresh `argocd-ssh-known-hosts-cm` in the `argocd` namespace, +restart `argocd-repo-server`, and hard-refresh the affected Application. + +Deploy or refresh the external Gitea container from the Debian host with: + +```bash +./{{ main_script }} deploy-gitea +``` + +## Gitea Backups + +`./{{ main_script }} up` installs a Debian-host systemd timer named +`homelab-gitea-backup.timer`. The timer runs daily, SSHes to the configured +Gitea host, executes `gitea dump` inside the Gitea Docker container, copies the +dump back to Debian, and stores it under `/home/jv/backups/gitea`. The default +retention is 30 days. + +The same install step also creates `homelab-gitea-restore-drill.timer`. The +monthly drill is non-destructive: it verifies the latest backup ZIP, extracts it +to a temporary directory, records a report under +`/home/jv/backups/gitea-restore-drills`, and removes the temporary extract. It +does not write into the live Gitea data directory. + +Run a manual backup from the Debian server with: + +```bash +./{{ main_script }} backup-gitea +``` + +Run the restore drill manually with: + +```bash +./{{ main_script }} drill-gitea-restore +``` + +Useful checks: + +```bash +systemctl list-timers homelab-gitea-backup.timer +systemctl list-timers homelab-gitea-restore-drill.timer +sudo systemctl start homelab-gitea-backup.service +ls -lh /home/jv/backups/gitea +ls -lh /home/jv/backups/gitea-restore-drills +``` + +## Gitea Actions + +This repo includes a Gitea Actions workflow at +`.gitea/workflows/homelab-main.yml`. It runs only on pushes to `main` and targets +a repository-scoped Debian host runner with the label `homelab-debian`. + +The workflow only blocks automatic deploy for external Gitea service +changes: files under `infra/gitea/`, or edits inside the `deploy_gitea`, +`install_gitea_backup_timer`, `backup_gitea`, or `drill_gitea_restore` +functions in `{{ main_script }}`. Other changes use `HOMELAB_ACTION_COMMAND=auto` by +default: Actions runs `./{{ main_script }} doctor-versions` when the Debian runner already +has a cluster kubeconfig. If node versions are aligned it runs `./{{ main_script }} apps`; +if kubelet minor drift is detected it runs `./{{ main_script }} rebuild-cluster`; if no +kubeconfig exists it also runs `./{{ main_script }} rebuild-cluster`. + +Set `HOMELAB_ACTION_COMMAND=apps` or `HOMELAB_ACTION_COMMAND=rebuild-cluster` +on the runner to force one path. + +`./{{ main_script }} bootstrap-gitea-repo` also registers the Debian host SSH public key +with the Gitea repository and switches the Debian working copy's `gitea` remote +to `ssh://git@192.168.100.73:32222/jv/my-homelab-configs.git`. The default key +is `/home/jv/.ssh/id_ed25519.pub`; set `LAB_GITEA_REPO_SSH_KEY_PATH` to use a +different Debian-host key, or `LAB_GITEA_REPO_SSH_BOOTSTRAP=false` to leave SSH +access unchanged. The Actions deploy job uses the checked-out Actions workspace +as the source commit, updates the first available persistent checkout from +`HOMELAB_DEPLOY_DIR`, `/home/jv/my-homelab-configs`, or +`/home/jv/repos/my-homelab-configs`, and otherwise deploys directly from the +Actions workspace. It does not need SSH read access back to Gitea. + +Enable Actions for the repository in Gitea, then create a repository-level runner +token from: + +```text +https://lab2025.duckdns.org/git/jv/my-homelab-configs/settings/actions/runners +``` + +Register and start the Debian runner from the Debian server: + +```bash +cd ~/my-homelab-configs +GITEA_RUNNER_REGISTRATION_TOKEN='' ./{{ main_script }} install-gitea-runner +``` + +The runner is installed as `homelab-gitea-runner.service`, runs as user `jv`, and +uses a host label instead of a Docker job container because deployment needs the +Debian host's Docker, OpenTofu, kubeconfig, SSH keys, and local state. + +The deployment job is non-interactive. User `jv` must be able to run `sudo -n +true` on the Debian host for deployment commands that require sudo. + +Useful checks: + +```bash +systemctl status homelab-gitea-runner.service +journalctl -u homelab-gitea-runner.service -n 100 --no-pager +``` + +## Renovate + +`renovate.json` defines dependency update rules for Dockerfiles, OpenTofu +providers, Helm chart versions, and the pinned tools used by the Gitea Actions +workflow. Renovate should open reviewable update branches or PRs only; it must +not auto-merge infrastructure changes. Keep app-only dependency updates on the +normal Gitea Actions path, and run `./{{ main_script }} up` manually on the Debian server +for platform or provisioning updates. + +## Destructive Rebuilds + +`./{{ main_script }} nuke` resets kubeadm, containerd runtime state, CNI files, Calico +links, iptables rules, and local OpenTofu state. It does not delete retained data +under `/data/openebs/local`. + +For multi-node labs, set `WORKER_SSH_TARGETS` to a space-separated list of SSH +targets. It defaults to an empty string so worker nodes are not cleaned unless +you explicitly include them. + +## Website App + +The website is a PHP app under `apps/website`. It includes a home page, CV page, +blog page, and demos page, plus a lightweight translation flow backed by Redis, +n8n, and Ollama. +Static language files live in `apps/website/lang`; `en.php` and `nah.php` are +curated source files, with the Nahuatl home page intentionally biased toward as +many Nahuatl words as possible while keeping technical terms understandable. +Unsupported browser languages use the same-origin `/translate.php` endpoint, +which calls Ollama server-side through `OLLAMA_HOST` and `OLLAMA_MODEL`; the +browser never calls the private Ollama IP directly. Redis caches per-string +translation results in the `website-production` namespace, while n8n is +available for translation prewarming and batch workflow jobs. The default model +is the custom `website-translator` Ollama model defined in +`apps/website/ollama/Modelfile`. Generated runtime language JSON is saved +through `save_lang.php` on the website PVC, and `translate.php` emits structured +logs for cache hits, misses, Ollama latency, JSON parse failures, and timeouts. + +Create or refresh the Ollama model on the Debian server before deploying a +website image that points at it: + +```bash +./{{ main_script }} website-translation-model +``` + +Ollama must also listen on the Debian host LAN address so Kubernetes pods on +other nodes can reach `OLLAMA_HOST=http://192.168.100.73:11434`: + +```bash +./{{ main_script }} website-ollama-listen +``` + +The CV page has two client-side presentation modes: + +- `Elegant`: dark, minimal, terminal-inspired styling with a square profile + image and light green console text. +- `Fancy`: centered circular profile image, cursive orbit text, and a + cursor-following portrait rotation effect. + +The Demos page is a catalog in the PHP website. The actual demo applications are +served from a separate `demos-static` artifact under `apps/demos-static` and are +published through the `demos-static` Argo CD application. Public traffic reaches +them through the edge path at `/demo-apps/`. + +`./{{ main_script }} up` builds and pushes two independent images: + +- a content-hash `php-website` tag generated by `{{ main_script }}` and passed to Argo CD + as a Kustomize image override +- `demos-static:latest` from `apps/demos-static` +- Cosign signatures and signed SPDX SBOM attestations for both pushed images + +The website manifest keeps the stable base image name `php-website:bootstrap`. +During bootstrap, `{{ main_script }}` hashes `apps/website`, builds +`/php-website:src-`, exports that exact reference through +`TF_VAR_website_image_ref`, and the Argo CD Application applies it through +Kustomize. This keeps the GitOps source generic while the deployed image remains +immutable. The Kyverno supply-chain policy mutates admitted pods to the verified +image digest, so the workload runs the same digest that was signed and attested. + +After `./{{ main_script }} apps`, the live deployment image should be a content-hash tag, +for example `192.168.100.73:30500/php-website:src-...`. If it still shows +`php-website:latest`, Argo CD has not rendered the current Application source. +Check the `website-production` Application source, sync status, and repository +access before restarting pods. + +The first demo, `The Client-Side Media Cruncher (Wasm + TS)`, currently performs +private, browser-only image compression and conversion using native Canvas APIs. +Heavier video conversion, such as MP4 to WebM, should use a Rust core compiled +to WebAssembly with a TypeScript UI so the codec work stays fast and still +avoids backend uploads. + +The demos are designed to be local-first so the current cluster can serve them +from any Linux app node without turning either pod into an application server. +The website pod serves the portfolio shell and the `demos-static` pod serves +static demo bundles; CPU-heavy work runs in the visitor's browser. Because the +deployments can run on either Debian or ARM workers, avoid bundling large ML +models, server-side WebSocket probes, or backend video transcoders into either +image. If those demos become production-grade, lazy load model assets in the +browser or move backend workers to a larger node, such as VMs on the Orange Pi 5 +Plus. + +Current demo inventory: + +- Client-side media cruncher: image conversion/compression with Canvas; future + Rust/Wasm codec path for video. +- Internet quality visualizer: live Canvas graph for latency, jitter, and + stability using same-origin browser probes; a dedicated WebSocket echo endpoint + would be the production version. +- Local log and JSON toolbelt: JSON formatting, JWT decoding, URL parsing, and + local text-log filtering. +- Architecture simulator: click-driven load, crash, and auto-scale simulation. +- Offline traveler converter: PWA shell with timezone, currency, and GB/GiB + conversions. +- Privacy-first redactor: local image redaction prototype; future + onnxruntime-web plus quantized YOLO or face model path. +- Local sentiment sandbox: lightweight local sentiment, keyword, and summary + prototype; future Transformers.js/ONNX path. +- Model drift simulator: visual MLOps playground for spikes, corrupted inputs, + and retraining. + +The Kubernetes deployment uses `apps/website/web-app.yaml` as a Kustomize base. +Keep `TF_VAR_registry_endpoint` aligned with the local registry endpoint used by +the app image build and with the image globs in +`apps/supply-chain-policy/local-registry-image-policy.yaml`. + +Keep the `.terraform.lock.hcl` files committed. They pin provider selections and +make bootstrap behavior reproducible across nodes and rebuilds. diff --git a/apps/website/blog.php b/apps/website/blog.php index 6bf29bb..5d448c5 100644 --- a/apps/website/blog.php +++ b/apps/website/blog.php @@ -88,7 +88,7 @@ $treeHref = 'homelab-tree.php?lang=' . urlencode($lang); $giteaSourceBase = 'https://lab2025.duckdns.org/git/jv/my-homelab-configs/src/branch/main/'; $stackSourceLinks = [ 'blog_stack_1' => [ - ['label' => 'lab.sh', 'path' => 'lab.sh'], + ['label' => 'jeannie', 'path' => 'jeannie'], ['label' => 'cluster/main.tf', 'path' => 'bootstrap/cluster/main.tf'], ['label' => 'provisioning/main.tf', 'path' => 'bootstrap/provisioning/main.tf'], ], @@ -97,7 +97,7 @@ $stackSourceLinks = [ ['label' => 'provisioning/main.tf', 'path' => 'bootstrap/provisioning/main.tf'], ], 'blog_stack_3' => [ - ['label' => 'lab.sh', 'path' => 'lab.sh'], + ['label' => 'jeannie', 'path' => 'jeannie'], ['label' => 'cluster', 'path' => 'bootstrap/cluster/main.tf'], ['label' => 'platform', 'path' => 'bootstrap/platform/main.tf'], ['label' => 'apps', 'path' => 'bootstrap/apps/main.tf'], @@ -144,7 +144,7 @@ $stackSourceLinks = [ ], 'blog_stack_12' => [ ['label' => 'provisioning/main.tf', 'path' => 'bootstrap/provisioning/main.tf'], - ['label' => 'lab.sh Pimox pipeline', 'path' => 'lab.sh'], + ['label' => 'jeannie Pimox pipeline', 'path' => 'jeannie'], ['label' => 'provisioning README', 'path' => 'bootstrap/provisioning/README.md'], ], 'blog_stack_13' => [ @@ -153,11 +153,11 @@ $stackSourceLinks = [ ['label' => 'prepare-template', 'path' => 'bootstrap/provisioning/templates/prepare-template.sh.tftpl'], ], 'blog_stack_14' => [ - ['label' => 'lab.sh worker storage guardrail', 'path' => 'lab.sh'], + ['label' => 'jeannie worker storage guardrail', 'path' => 'jeannie'], ['label' => 'README Pimox defaults', 'path' => 'README.md'], ], 'blog_stack_15' => [ - ['label' => 'lab.sh OpenWrt pipeline', 'path' => 'lab.sh'], + ['label' => 'jeannie OpenWrt pipeline', 'path' => 'jeannie'], ['label' => 'README OpenWrt notes', 'path' => 'README.md'], ], 'blog_stack_16' => [ @@ -172,7 +172,7 @@ $stackSourceLinks = [ ], 'blog_stack_18' => [ ['label' => 'supply-chain policy', 'path' => 'apps/supply-chain-policy/local-registry-image-policy.yaml'], - ['label' => 'Cosign build flow', 'path' => 'lab.sh'], + ['label' => 'Cosign build flow', 'path' => 'jeannie'], ['label' => 'apps variables', 'path' => 'bootstrap/apps/variables.tf'], ], 'blog_stack_19' => [ @@ -368,7 +368,7 @@ function renderStackSourceLinks(string $stackKey, array $sourceLinks, string $so - OpenTofu + lab.sh + OpenTofu + jeannie manual infra apply path apps command for CI deploys diff --git a/apps/website/homelab-tree.php b/apps/website/homelab-tree.php index 105d8f8..dbbb766 100644 --- a/apps/website/homelab-tree.php +++ b/apps/website/homelab-tree.php @@ -115,7 +115,7 @@ $blogHref = 'blog.php?lang=' . urlencode($lang); OpenTofu - lab.sh apply + jeannie apply diff --git a/apps/website/lang/en.php b/apps/website/lang/en.php index 8f62b64..bfd065d 100644 --- a/apps/website/lang/en.php +++ b/apps/website/lang/en.php @@ -153,7 +153,7 @@ return [ 'blog_stack_19' => 'The translation path now uses Redis for per-string cache hits, a custom Ollama website-translator model on the Debian server for cache misses, n8n for prewarming and batch workflow automation, and structured logs for cache, latency, JSON parse, and timeout signals.', 'blog_arch_kicker' => 'Architecture map', 'blog_arch_title' => 'The homelab, end to end', - 'blog_arch_intro' => 'The current delivery path starts with a push to Gitea, runs local validation, builds arm64 images, syncs the validated commit into the GitOps mirror, and lets Argo CD reconcile from the app workers. The infrastructure path stays manual through lab.sh, including the PXE/Pimox template builder, NVMe-backed worker clones, Kyverno policy placement, Redis-backed website translation, n8n workflow automation, and the opt-in OpenWrt firewall VM, while the OCI edge routes public traffic back through the private path.', + 'blog_arch_intro' => 'The current delivery path starts with a push to Gitea, runs local validation, builds arm64 images, syncs the validated commit into the GitOps mirror, and lets Argo CD reconcile from the app workers. The infrastructure path stays manual through jeannie, including the PXE/Pimox template builder, NVMe-backed worker clones, Kyverno policy placement, Redis-backed website translation, n8n workflow automation, and the opt-in OpenWrt firewall VM, while the OCI edge routes public traffic back through the private path.', 'blog_arch_caption' => 'The diagram is intentionally operational: it shows the app delivery loop, image flow, provisioning path, worker-placement boundary, monitoring layer, Redis/n8n/Ollama translation path, OpenWrt firewall option, and public traffic path without hiding the practical bits that make a small lab behave like a platform.', 'blog_arch_fun_link' => 'Open the Christmas-tree version', 'blog_activity_kicker' => 'Recent activity log', @@ -163,7 +163,7 @@ return [ 'blog_activity_2' => 'Installed and validated a Debian-hosted Gitea Actions runner so pushes to main can build, scan, and deploy without depending on a laptop session.', 'blog_activity_3' => 'Added a custom checkout flow for the /git/ subpath and kept a persistent Debian checkout for the deployment scripts.', 'blog_activity_4' => 'Added Gitleaks secret scanning and Trivy scanning for the app and infrastructure tree.', - 'blog_activity_5' => 'Changed deployment so the validated commit is pushed into the local GitOps mirror before lab.sh runs, preventing Argo CD from reconciling an older tree.', + 'blog_activity_5' => 'Changed deployment so the validated commit is pushed into the local GitOps mirror before jeannie runs, preventing Argo CD from reconciling an older tree.', 'blog_activity_6' => 'Hardened the website, demos-static, and registry workloads with non-root containers, read-only root filesystems, resource limits, and explicit writable volumes.', 'blog_activity_7' => 'Split the demos into a dedicated demos-static image and Argo CD application so the PHP website stays small and boring.', 'blog_activity_8' => 'Changed Gitea backups to dump from the Raspberry Pi Docker container and store archives on the Debian host.', @@ -174,7 +174,7 @@ return [ 'blog_activity_13' => 'Installed the monitoring stack and moved platform add-ons such as Argo CD, Kyverno, and prometheus-stack work toward app-worker placement instead of treating the control plane as spare capacity.', 'blog_activity_14' => 'Reworked the website image to keep Apache as the web server while moving PHP execution to PHP-FPM, enabling OPcache, and validating the runtime through Gitea Actions instead of local image builds.', 'blog_activity_15' => 'Pre-rendered the read-heavy website pages into static HTML, moved visitor ideas behind a small JSON endpoint, added cache headers, enabled HTTP/2 and connection reuse at the OCI edge, and measured reused HTTP/2 asset loads around 60ms.', - 'blog_activity_16' => 'Added image signing and SBOM validation: lab.sh now builds with SBOM metadata, signs php-website and demos-static with Cosign, attaches signed SPDX SBOM attestations, and Kyverno audits unsigned local-registry pods before enforcement is promoted.', + 'blog_activity_16' => 'Added image signing and SBOM validation: jeannie now builds with SBOM metadata, signs php-website and demos-static with Cosign, attaches signed SPDX SBOM attestations, and Kyverno audits unsigned local-registry pods before enforcement is promoted.', 'blog_todo_kicker' => 'Improvement backlog', 'blog_todo_title' => 'Todo list for the next homelab pass', 'blog_todo_intro' => 'These are improvement proposals, not chores for the sake of chores. Each item either reduces rebuild risk, tightens supply-chain hygiene, or makes the platform easier to operate when something fails.', diff --git a/apps/website/lang/nah.php b/apps/website/lang/nah.php index c58e245..3e56903 100644 --- a/apps/website/lang/nah.php +++ b/apps/website/lang/nah.php @@ -153,7 +153,7 @@ return [ 'blog_stack_19' => 'Translation path axcan uses Redis para per-string cache hits, custom Ollama website-translator model ipan Debian server para cache misses, n8n para prewarming ihuan batch workflows, ihuan structured logs para cache, latency, JSON parse, ihuan timeout signals.', 'blog_arch_kicker' => 'Architecture mapa', 'blog_arch_title' => 'Homelab, end to end', - 'blog_arch_intro' => 'Current delivery path starts ika push to Gitea, runs local validation, builds arm64 images, syncs validated commit into GitOps mirror, ihuan lets Argo CD reconcile from app workers. Infrastructure path stays manual through lab.sh, including PXE/Pimox template builder, NVMe-backed worker clones, Kyverno policy placement, Redis-backed website translation, n8n workflow automation, ihuan opt-in OpenWrt firewall VM, while OCI edge routes public traffic back through private path.', + 'blog_arch_intro' => 'Current delivery path starts ika push to Gitea, runs local validation, builds arm64 images, syncs validated commit into GitOps mirror, ihuan lets Argo CD reconcile from app workers. Infrastructure path stays manual through jeannie, including PXE/Pimox template builder, NVMe-backed worker clones, Kyverno policy placement, Redis-backed website translation, n8n workflow automation, ihuan opt-in OpenWrt firewall VM, while OCI edge routes public traffic back through private path.', 'blog_arch_caption' => 'Diagram intentionally operational: quinextia app delivery loop, image flow, provisioning path, worker-placement boundary, monitoring layer, Redis/n8n/Ollama translation path, OpenWrt firewall option, ihuan public traffic path ahmo hiding practical bits in quichihua small lab behave like platform.', 'blog_arch_fun_link' => 'Xiquitta Christmas-tree version', 'blog_activity_kicker' => 'Yancuic activity log', @@ -163,7 +163,7 @@ return [ 'blog_activity_2' => 'Installed ihuan validated Debian-hosted Gitea Actions runner para pushes to main can build, scan, ihuan deploy without laptop session.', 'blog_activity_3' => 'Added custom checkout flow para /git/ subpath ihuan kept persistent Debian checkout para deployment scripts.', 'blog_activity_4' => 'Added Gitleaks secret scanning ihuan Trivy scanning para app ihuan infrastructure tree.', - 'blog_activity_5' => 'Changed deployment so validated commit is pushed into local GitOps mirror before lab.sh runs, preventing Argo CD from reconciling older tree.', + 'blog_activity_5' => 'Changed deployment so validated commit is pushed into local GitOps mirror before jeannie runs, preventing Argo CD from reconciling older tree.', 'blog_activity_6' => 'Hardened website, demos-static, ihuan registry workloads ika non-root containers, read-only root filesystems, resource limits, ihuan explicit writable volumes.', 'blog_activity_7' => 'Split demos into dedicated demos-static image ihuan Argo CD application so PHP website stays small and boring.', 'blog_activity_8' => 'Changed Gitea backups to dump from Raspberry Pi Docker container ihuan store archives on Debian host.', @@ -174,7 +174,7 @@ return [ 'blog_activity_13' => 'Installed monitoring stack ihuan moved platform add-ons such as Argo CD, Kyverno, ihuan prometheus-stack toward app-worker placement instead of treating control plane as spare capacity.', 'blog_activity_14' => 'Reworked website image para Apache ma mocahua web server while PHP runs through PHP-FPM, OPcache enabled, ihuan runtime validated through Gitea Actions instead of local image builds.', 'blog_activity_15' => 'Pre-rendered read-heavy website pages into static HTML, moved visitor ideas behind small JSON endpoint, added cache headers, enabled HTTP/2 ihuan connection reuse at OCI edge, ihuan measured reused HTTP/2 asset loads around 60ms.', - 'blog_activity_16' => 'Added image signing ihuan SBOM validation: lab.sh axcan builds ika SBOM metadata, signs php-website ihuan demos-static ika Cosign, attaches signed SPDX SBOM attestations, ihuan Kyverno audits unsigned local-registry pods before enforcement is promoted.', + 'blog_activity_16' => 'Added image signing ihuan SBOM validation: jeannie axcan builds ika SBOM metadata, signs php-website ihuan demos-static ika Cosign, attaches signed SPDX SBOM attestations, ihuan Kyverno audits unsigned local-registry pods before enforcement is promoted.', 'blog_todo_kicker' => 'Improvement tlatecpanaliztli', 'blog_todo_title' => 'Todo list para next homelab pass', 'blog_todo_intro' => 'Inin cateh improvement proposals, amo chores zan pampa chores. Each item either reduces rebuild risk, tightens supply-chain hygiene, o makes platform easier to operate quema tlein fails.', diff --git a/bootstrap/host/README.md b/bootstrap/host/README.md index ec69265..cf96e2d 100644 --- a/bootstrap/host/README.md +++ b/bootstrap/host/README.md @@ -12,7 +12,7 @@ sudo usermod -aG sudo jv git clone /home/jv/my-homelab-configs cd /home/jv/my-homelab-configs ansible-playbook -K -i bootstrap/host/inventory.ini bootstrap/host/playbook.yml -./lab.sh up +./jeannie up ``` To also restore captured application behavior/configuration, keep the generated @@ -34,7 +34,7 @@ The playbook: - adds `jv` to expected local groups - enables core services such as Docker, containerd, kubelet, ssh, cron, and Bluetooth when installed -- creates homelab persistence directories used by `lab.sh up` +- creates homelab persistence directories used by `jeannie up` - optionally extracts the captured user config archive into `/home/jv` Secrets and active sessions are intentionally not stored in this repo. Keep diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 93b5e9c..18cb0d6 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -2,7 +2,7 @@ This layer prepares a Debian server to PXE boot a Debian 13 arm64 VM and install a reusable worker-node golden image for Pimox on Orange Pi 5 Plus. -`./lab.sh up` drives this layer in auto mode when the Pimox host is reachable at +`./jeannie up` drives this layer in auto mode when the Pimox host is reachable at `192.168.100.80`, `qm` is installed, and the existing bridge `vmbr0` is present. The automation creates VM definitions only; it validates the bridge and refuses to edit Orange Pi host networking. @@ -26,7 +26,7 @@ The normal path is: ```bash cd ~/my-homelab-configs -./lab.sh up +./jeannie up ``` For manual provisioning-only testing, find the LAN interface on the Debian @@ -88,7 +88,7 @@ Run the cluster layer from the Debian homelab server after the cloned VM is reac ## Pimox Automation -`./lab.sh up` runs Pimox provisioning by default and sets +`./jeannie up` runs Pimox provisioning by default and sets `TF_VAR_pimox_template_builder_enabled=true` when Pimox is reachable. The layer SSHes into the Pimox host, create the template-build VM with `qm`, boot it from PXE, wait for the installed VM over SSH, run `/usr/local/sbin/homelab-prepare-template.sh`, @@ -100,7 +100,7 @@ template-build VM. If you still want to force a known address, set `TF_VAR_pimox_template_build_host`. ```bash -./lab.sh up +./jeannie up ``` Defaults match the observed Pimox template VM shape: OVMF firmware, virtio @@ -109,7 +109,7 @@ networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. -`./lab.sh up` also creates or reuses worker clones after the template exists. It +`./jeannie up` also creates or reuses worker clones after the template exists. It defaults to one worker, VMID `9010`, name `pimox-worker-01`, deterministic locally administered MAC address, 1 socket with 2 cores, 4 GiB RAM, `opi5_ssd` clone storage, and qemu-guest-agent IP @@ -132,16 +132,16 @@ defaults to empty, so the pipeline owns index `1` unless you set Useful overrides: ```bash -./lab.sh rebuild-cluster -LAB_PIMOX_PIPELINE=false ./lab.sh up -LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true ./lab.sh up -LAB_PIMOX_WORKER_COUNT=0 ./lab.sh up -LAB_PIMOX_WORKER_COUNT=2 ./lab.sh up -LAB_PIMOX_SKIP_WORKER_INDEXES=1 LAB_PIMOX_WORKER_COUNT=2 ./lab.sh up -LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up -LAB_PIMOX_WORKER_STORAGE= ./lab.sh up -LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./lab.sh up -LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up +./jeannie rebuild-cluster +LAB_PIMOX_PIPELINE=false ./jeannie up +LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true ./jeannie up +LAB_PIMOX_WORKER_COUNT=0 ./jeannie up +LAB_PIMOX_WORKER_COUNT=2 ./jeannie up +LAB_PIMOX_SKIP_WORKER_INDEXES=1 LAB_PIMOX_WORKER_COUNT=2 ./jeannie up +LAB_PIMOX_WORKER_BASE_VMID=9020 ./jeannie up +LAB_PIMOX_WORKER_STORAGE= ./jeannie up +LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up +LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./jeannie up ``` ## OpenWrt firewall VM @@ -155,7 +155,7 @@ after `vmbr1` exists on the Pimox host and the second NIC/LAN side is safe to use: ```bash -./lab.sh openwrt +./jeannie openwrt ``` Defaults: diff --git a/docs.yml b/docs.yml new file mode 100644 index 0000000..a1fb761 --- /dev/null +++ b/docs.yml @@ -0,0 +1 @@ +main_script: jeannie diff --git a/infra/arr-stack/README.md b/infra/arr-stack/README.md index 2306b29..b665aff 100644 --- a/infra/arr-stack/README.md +++ b/infra/arr-stack/README.md @@ -1,7 +1,7 @@ # External ARR Stack This Docker Compose stack runs the media-management apps outside Kubernetes on -the Debian host. It is not called by `lab.sh` and is not part of the homelab +the Debian host. It is not called by `jeannie` and is not part of the homelab OpenTofu/Argo pipeline. Services: diff --git a/infra/arr-stack/docs/bootstrap.md b/infra/arr-stack/docs/bootstrap.md index d651ee9..1a7f8fb 100644 --- a/infra/arr-stack/docs/bootstrap.md +++ b/infra/arr-stack/docs/bootstrap.md @@ -1,6 +1,6 @@ # ARR Stack Bootstrap Notes -This stack is intentionally separate from `lab.sh`, OpenTofu, Argo CD, and the +This stack is intentionally separate from `jeannie`, OpenTofu, Argo CD, and the Kubernetes app pipeline. ## Storage Layout diff --git a/infra/gitea/README.md b/infra/gitea/README.md index 4bb1683..329e8af 100644 --- a/infra/gitea/README.md +++ b/infra/gitea/README.md @@ -2,7 +2,7 @@ Gitea is bootstrap infrastructure, not a Kubernetes workload. -`lab.sh deploy-gitea` copies `docker-compose.yml` to the Debian host and runs +`jeannie deploy-gitea` copies `docker-compose.yml` to the Debian host and runs Gitea as an always-on Docker Compose service. The current default stores data on the HP laptop NVMe under `/data/homelab-gitea/data`; Docker root is also expected to live on the `/data` storage path. @@ -20,7 +20,7 @@ Kubernetes consumes Git from the Debian bare GitOps mirror at `/home/jv/git-server/my-homelab-configs.git`. Gitea is the human-facing Git service and remains available when the cluster is destroyed. -`./lab.sh bootstrap-gitea-repo` creates or validates the public Gitea repository, +`./jeannie bootstrap-gitea-repo` creates or validates the public Gitea repository, adds the Debian host deploy key when needed, and points the Debian checkout's `gitea` remote at: @@ -39,7 +39,7 @@ The platform bootstrap registers that repo secret and updates `argocd-ssh-known-hosts-cm`. If Argo CD reports `knownhosts: key is unknown`, refresh the Debian host key in that ConfigMap and restart `argocd-repo-server`. -Backups are installed on the Debian host by `lab.sh deploy-gitea` and -`lab.sh backup-gitea`. The timer runs `gitea dump` inside the Gitea container, +Backups are installed on the Debian host by `jeannie deploy-gitea` and +`jeannie backup-gitea`. The timer runs `gitea dump` inside the Gitea container, copies the archive to Debian, and stores it under `/home/jv/backups/gitea`. diff --git a/infra/youtube-backup/README.md b/infra/youtube-backup/README.md index 141fc55..9131a93 100644 --- a/infra/youtube-backup/README.md +++ b/infra/youtube-backup/README.md @@ -1,7 +1,7 @@ # YouTube Backup Standalone helper for backing up your own uploaded YouTube videos with -`yt-dlp`. It is intentionally outside `lab.sh`, OpenTofu, Argo CD, and the +`yt-dlp`. It is intentionally outside `jeannie`, OpenTofu, Argo CD, and the homelab deployment pipeline. The backup root defaults to `/data/youtube-backup`. diff --git a/renovate.json b/renovate.json index 57f03e5..478b361 100644 --- a/renovate.json +++ b/renovate.json @@ -42,7 +42,7 @@ "matchFileNames": [ "bootstrap/**", "infra/gitea/**", - "lab.sh", + "jeannie", ".gitea/workflows/**" ], "automerge": false diff --git a/scripts/render-docs b/scripts/render-docs new file mode 100755 index 0000000..6bdf8f1 --- /dev/null +++ b/scripts/render-docs @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +CONFIG_FILE="${REPO_ROOT}/docs.yml" +README_TEMPLATE="${REPO_ROOT}/README.md.tmpl" +README_OUTPUT="${REPO_ROOT}/README.md" + +usage() { + cat <<'USAGE' +Usage: scripts/render-docs [--check] + +Render generated Markdown docs from repo-local templates. + +Options: + --check fail if generated docs differ from committed output +USAGE +} + +read_config_value() { + local key="$1" + + awk -F ':' -v key="${key}" ' + $1 == key { + value = $0 + sub("^[^:]*:[[:space:]]*", "", value) + gsub(/^[[:space:]"'"'"']+|[[:space:]"'"'"']+$/, "", value) + print value + found = 1 + exit + } + END { + if (!found) { + exit 1 + } + } + ' "${CONFIG_FILE}" +} + +render_readme() { + local main_script="$1" + + if grep -Eo '{{[[:space:]]*[A-Za-z_][A-Za-z0-9_]*[[:space:]]*}}' "${README_TEMPLATE}" | + grep -Evq '{{[[:space:]]*main_script[[:space:]]*}}'; then + printf 'Unsupported template variable in %s\n' "${README_TEMPLATE}" >&2 + return 1 + fi + + sed "s/{{[[:space:]]*main_script[[:space:]]*}}/${main_script}/g" "${README_TEMPLATE}" +} + +main() { + local mode="render" + local main_script + local rendered + + case "${1:-}" in + "") + ;; + --check) + mode="check" + ;; + -h | --help) + usage + return 0 + ;; + *) + usage >&2 + return 2 + ;; + esac + + if [[ ! -s "${CONFIG_FILE}" ]]; then + printf 'Missing docs config: %s\n' "${CONFIG_FILE}" >&2 + return 1 + fi + if [[ ! -s "${README_TEMPLATE}" ]]; then + printf 'Missing README template: %s\n' "${README_TEMPLATE}" >&2 + return 1 + fi + + main_script="$(read_config_value main_script)" + if [[ -z "${main_script}" ]]; then + printf 'docs.yml main_script must not be empty\n' >&2 + return 1 + fi + + rendered="$(mktemp)" + render_readme "${main_script}" >"${rendered}" + + if [[ "${mode}" == "check" ]]; then + if ! cmp -s "${rendered}" "${README_OUTPUT}"; then + printf 'Generated README.md is stale. Run scripts/render-docs.\n' >&2 + diff -u "${README_OUTPUT}" "${rendered}" || true + rm -f "${rendered}" + return 1 + fi + printf 'docs are current\n' + rm -f "${rendered}" + return 0 + fi + + mv "${rendered}" "${README_OUTPUT}" + printf 'Rendered %s from %s\n' "${README_OUTPUT}" "${README_TEMPLATE}" +} + +main "$@"