From 534c87132e0fa21af15cbc10dddd578c2868e869 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Sat, 27 Jun 2026 16:18:17 -0600 Subject: [PATCH] Add RPi DNS services stack --- README.md | 40 ++++- README.md.tmpl | 40 ++++- infra/rpi-services/README.md | 41 +++++ infra/rpi-services/adlists.txt | 7 + infra/rpi-services/bootstrap.sh | 234 ++++++++++++++++++++++++++ infra/rpi-services/docker-compose.yml | 58 +++++++ jeannie | 77 ++++++++- 7 files changed, 484 insertions(+), 13 deletions(-) create mode 100644 infra/rpi-services/README.md create mode 100644 infra/rpi-services/adlists.txt create mode 100755 infra/rpi-services/bootstrap.sh create mode 100644 infra/rpi-services/docker-compose.yml diff --git a/README.md b/README.md index 1406612..a214874 100644 --- a/README.md +++ b/README.md @@ -235,8 +235,36 @@ Run a read-only health snapshot from the Debian server with: ``` It reports host memory/disk, systemd services, Docker Compose stacks, -Kubernetes health when the API is reachable, Pimox worker VM status, Tailscale, -and key local/public HTTP endpoints. +Kubernetes health when the API is reachable, Pimox worker VM status, RPi +services, Tailscale, and key local/public HTTP endpoints. + +## RPi Services + +The Raspberry Pi runs lightweight always-on services outside Kubernetes from +`infra/rpi-services`: + +- Pi-hole on DNS port `53` and web port `8081` +- Unbound as Pi-hole's first upstream resolver +- Uptime Kuma on port `3001` + +Deploy or refresh them from the Debian server with: + +```bash +./jeannie rpi-services +``` + +`./jeannie up` also runs this step unless +`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under +`/nvme-storage/docker` only when that path is an active writable mount point. If +the NVMe is missing or resets, it rewrites Docker's `data-root` to +`/var/lib/docker` and restarts Docker so DNS can still come up from the +SD/root filesystem. + +Pi-hole uses Unbound first and public fallback upstreams after that, so DNS can +continue resolving if the Unbound container stops. Managed adlists live in +`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without +deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, +the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. ## Adding Nodes @@ -480,7 +508,7 @@ Traefik uses MetalLB for a LAN `LoadBalancer` address. App services such as the website, demos, and Heimdall should be `ClusterIP` services behind Kubernetes Ingress objects. The local registry remains a `NodePort` because the cluster nodes use it as a pull endpoint. Gitea is not a Kubernetes service; it runs on -the Raspberry Pi Docker host. +the Debian Docker host. ## Secrets @@ -504,9 +532,9 @@ file when the public host, SSH key, server name, backend Tailscale IP, or Traefik backend address changes. The `/git/` route is intentionally different from the Kubernetes app routes: it -proxies to Gitea on the Raspberry Pi at the configured `gitea_backend_port` -instead of Traefik. This keeps public read-only source browsing available even -when the cluster has been destroyed. +proxies to Gitea on the Debian host through Tailscale instead of Traefik. This +keeps public read-only source browsing available even when the cluster has been +destroyed. For the main website, nginx terminates TLS, serves cached HTML and static assets, compresses text responses with gzip, advertises HTTP/2, reuses TLS diff --git a/README.md.tmpl b/README.md.tmpl index 44051dc..cfb2a74 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -235,8 +235,36 @@ Run a read-only health snapshot from the Debian server with: ``` It reports host memory/disk, systemd services, Docker Compose stacks, -Kubernetes health when the API is reachable, Pimox worker VM status, Tailscale, -and key local/public HTTP endpoints. +Kubernetes health when the API is reachable, Pimox worker VM status, RPi +services, Tailscale, and key local/public HTTP endpoints. + +## RPi Services + +The Raspberry Pi runs lightweight always-on services outside Kubernetes from +`infra/rpi-services`: + +- Pi-hole on DNS port `53` and web port `8081` +- Unbound as Pi-hole's first upstream resolver +- Uptime Kuma on port `3001` + +Deploy or refresh them from the Debian server with: + +```bash +./{{ main_script }} rpi-services +``` + +`./{{ main_script }} up` also runs this step unless +`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under +`/nvme-storage/docker` only when that path is an active writable mount point. If +the NVMe is missing or resets, it rewrites Docker's `data-root` to +`/var/lib/docker` and restarts Docker so DNS can still come up from the +SD/root filesystem. + +Pi-hole uses Unbound first and public fallback upstreams after that, so DNS can +continue resolving if the Unbound container stops. Managed adlists live in +`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without +deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, +the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. ## Adding Nodes @@ -480,7 +508,7 @@ Traefik uses MetalLB for a LAN `LoadBalancer` address. App services such as the website, demos, and Heimdall should be `ClusterIP` services behind Kubernetes Ingress objects. The local registry remains a `NodePort` because the cluster nodes use it as a pull endpoint. Gitea is not a Kubernetes service; it runs on -the Raspberry Pi Docker host. +the Debian Docker host. ## Secrets @@ -504,9 +532,9 @@ file when the public host, SSH key, server name, backend Tailscale IP, or Traefik backend address changes. The `/git/` route is intentionally different from the Kubernetes app routes: it -proxies to Gitea on the Raspberry Pi at the configured `gitea_backend_port` -instead of Traefik. This keeps public read-only source browsing available even -when the cluster has been destroyed. +proxies to Gitea on the Debian host through Tailscale instead of Traefik. This +keeps public read-only source browsing available even when the cluster has been +destroyed. For the main website, nginx terminates TLS, serves cached HTML and static assets, compresses text responses with gzip, advertises HTTP/2, reuses TLS diff --git a/infra/rpi-services/README.md b/infra/rpi-services/README.md new file mode 100644 index 0000000..7bd85a0 --- /dev/null +++ b/infra/rpi-services/README.md @@ -0,0 +1,41 @@ +# RPi Services + +This stack runs lightweight always-on services on the Raspberry Pi outside the +Kubernetes cluster: + +- Pi-hole on DNS port `53` and web port `8081` +- Unbound as Pi-hole's first upstream resolver +- Uptime Kuma on port `3001` + +Pi-hole also keeps public fallback upstreams in its config, so DNS can continue +to resolve through `1.1.1.1` and `9.9.9.9` if the Unbound container stops. + +The bootstrap script prefers Docker data under `/nvme-storage/docker` only when +that path is an active mount point and writable. If the NVMe is missing or +reset, it rewrites Docker's `data-root` to `/var/lib/docker` and restarts +Docker, allowing the services to come up from the SD/root filesystem instead. + +Service config lives under `/opt/homelab-rpi-services/data` by default. Backup +storage is intentionally not placed on the NVMe until the disk has proven +stable. + +Deploy from the Debian homelab server: + +```bash +./jeannie rpi-services +``` + +Useful overrides: + +```bash +LAB_RPI_HOST=192.168.100.89 ./jeannie rpi-services +PIHOLE_WEBPASSWORD='' ./jeannie rpi-services +PIHOLE_WEB_PORT=8081 UPTIME_KUMA_PORT=3001 ./jeannie rpi-services +LAB_RPI_STOP_LEGACY_PIHOLE=false ./jeannie rpi-services +``` + +If `PIHOLE_WEBPASSWORD` is not provided, the bootstrap generates one and keeps +it in `/opt/homelab-rpi-services/.env` for future runs. + +Managed Pi-hole adlists are stored in `adlists.txt`. The bootstrap inserts them +without deleting existing manual Pi-hole lists. diff --git a/infra/rpi-services/adlists.txt b/infra/rpi-services/adlists.txt new file mode 100644 index 0000000..a4ffdbe --- /dev/null +++ b/infra/rpi-services/adlists.txt @@ -0,0 +1,7 @@ +# Managed Pi-hole adlists for the RPi4 DNS stack. +# Existing manual Pi-hole lists are not removed by the bootstrap script. +https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts +https://big.oisd.nl/ +https://nsfw.oisd.nl/ +https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/pro.txt +https://raw.githubusercontent.com/hagezi/dns-blocklists/main/adblock/tif.txt diff --git a/infra/rpi-services/bootstrap.sh b/infra/rpi-services/bootstrap.sh new file mode 100755 index 0000000..1e84f3f --- /dev/null +++ b/infra/rpi-services/bootstrap.sh @@ -0,0 +1,234 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INSTALL_DIR="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" +DATA_DIR="${LAB_RPI_SERVICES_DATA_DIR:-${INSTALL_DIR}/data}" +DOCKER_NVME_ROOT="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" +DOCKER_FALLBACK_ROOT="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" +STOP_LEGACY_PIHOLE="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" +PIHOLE_WEBPASSWORD="${PIHOLE_WEBPASSWORD:-}" + +install_missing_packages() { + local missing_packages=() + local package + + for package in "$@"; do + if ! dpkg-query -W -f='${Status}' "${package}" 2>/dev/null | grep -q 'install ok installed'; then + missing_packages+=("${package}") + fi + done + + if ((${#missing_packages[@]} > 0)); then + sudo apt-get update + sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi +} + +docker_root_target() { + if mountpoint -q "${DOCKER_NVME_ROOT}"; then + sudo mkdir -p "${DOCKER_NVME_ROOT}" + if sudo test -w "${DOCKER_NVME_ROOT}"; then + printf '%s\n' "${DOCKER_NVME_ROOT}" + return 0 + fi + fi + + printf '%s\n' "${DOCKER_FALLBACK_ROOT}" +} + +configure_docker_root() { + local target_root="$1" + local current_root="" + local configured_root="" + local daemon_file="/etc/docker/daemon.json" + + sudo mkdir -p "${target_root}" /etc/docker + + if sudo systemctl is-active --quiet docker; then + current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" + fi + if [[ -s "${daemon_file}" ]]; then + configured_root="$(python3 - "${daemon_file}" <<'PY' +import json +import sys + +try: + with open(sys.argv[1], encoding="utf-8") as handle: + document = json.load(handle) +except Exception: + document = {} +print(document.get("data-root", "")) +PY +)" + fi + + if [[ "${current_root}" == "${target_root}" && "${configured_root}" == "${target_root}" ]]; then + return 0 + fi + + echo "Configuring Docker data-root: ${target_root}" + sudo python3 - "${daemon_file}" "${target_root}" <<'PY' +import json +import os +import sys + +path, target_root = sys.argv[1:3] +document = {} +if os.path.exists(path) and os.path.getsize(path) > 0: + with open(path, encoding="utf-8") as handle: + document = json.load(handle) +document["data-root"] = target_root +with open(path, "w", encoding="utf-8") as handle: + json.dump(document, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + sudo systemctl restart docker +} + +stop_legacy_pihole() { + local container + + if [[ "${STOP_LEGACY_PIHOLE}" =~ ^(0|false|no|off|disabled)$ ]]; then + return 0 + fi + + for container in pihole pi-hole; do + if sudo docker ps -a --format '{{.Names}}' | grep -qx "${container}"; then + echo "Stopping legacy Pi-hole container ${container}; it is left on disk for manual rollback." + sudo docker stop "${container}" >/dev/null 2>&1 || true + fi + done +} + +resolve_pihole_webpassword() { + local env_file="${INSTALL_DIR}/.env" + local existing_password="" + + if [[ -n "${PIHOLE_WEBPASSWORD}" ]]; then + printf '%s\n' "${PIHOLE_WEBPASSWORD}" + return 0 + fi + + if [[ -s "${env_file}" ]]; then + existing_password="$(grep -E '^PIHOLE_WEBPASSWORD=' "${env_file}" | tail -n 1 | cut -d= -f2- || true)" + if [[ -n "${existing_password}" ]]; then + printf '%s\n' "${existing_password}" + return 0 + fi + fi + + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 18 + return 0 + fi + + python3 - <<'PY' +import secrets + +print(secrets.token_hex(18)) +PY +} + +write_env_file() { + PIHOLE_WEBPASSWORD="$(resolve_pihole_webpassword)" + sudo tee "${INSTALL_DIR}/.env" >/dev/null </dev/null 2>&1; do + if ((elapsed >= 120)); then + echo "Timed out waiting for Pi-hole gravity database." >&2 + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + sql_file="$(mktemp)" + python3 - "${adlists_file}" >"${sql_file}" <<'PY' +import sys + +adlists_file = sys.argv[1] + +with open(adlists_file, encoding="utf-8") as handle: + addresses = [ + line.strip() + for line in handle + if line.strip() and not line.lstrip().startswith("#") + ] + +if not addresses: + raise SystemExit(0) + +def sql_string(value): + return "'" + value.replace("'", "''") + "'" + +for address in addresses: + quoted = sql_string(address) + print(f"insert or ignore into adlist (address, enabled, comment) values ({quoted}, 1, 'homelab managed');") + print(f"update adlist set enabled = 1, comment = coalesce(nullif(comment, ''), 'homelab managed') where address = {quoted};") +PY + # shellcheck disable=SC2024 + sudo docker exec -i "${pihole_container}" pihole-FTL sqlite3 /etc/pihole/gravity.db <"${sql_file}" + rm -f "${sql_file}" + sudo docker exec "${pihole_container}" pihole -g +} + +install_missing_packages ca-certificates curl iptables python3 rsync + +if ! command -v docker >/dev/null 2>&1; then + curl -fsSL https://get.docker.com | sudo sh +fi +sudo systemctl enable docker >/dev/null + +configure_docker_root "$(docker_root_target)" + +if ! sudo docker compose version >/dev/null 2>&1; then + install_missing_packages docker-compose-plugin +fi + +sudo mkdir -p "${INSTALL_DIR}" "${DATA_DIR}/pihole/etc-pihole" "${DATA_DIR}/pihole/etc-dnsmasq.d" "${DATA_DIR}/uptime-kuma" +sudo cp "${SCRIPT_DIR}/docker-compose.yml" "${INSTALL_DIR}/docker-compose.yml" +sudo cp "${SCRIPT_DIR}/adlists.txt" "${INSTALL_DIR}/adlists.txt" +write_env_file + +stop_legacy_pihole + +cd "${INSTALL_DIR}" +sudo docker compose pull +sudo docker compose up -d --remove-orphans +sync_pihole_adlists +sudo docker compose ps diff --git a/infra/rpi-services/docker-compose.yml b/infra/rpi-services/docker-compose.yml new file mode 100644 index 0000000..8471611 --- /dev/null +++ b/infra/rpi-services/docker-compose.yml @@ -0,0 +1,58 @@ +services: + unbound: + image: ${UNBOUND_IMAGE:-mvance/unbound-rpi:latest} + container_name: ${UNBOUND_CONTAINER_NAME:-homelab-unbound} + restart: unless-stopped + networks: + dns: + aliases: + - unbound + + pihole: + image: ${PIHOLE_IMAGE:-pihole/pihole:latest} + container_name: ${PIHOLE_CONTAINER_NAME:-homelab-pihole} + restart: unless-stopped + depends_on: + - unbound + ports: + - "${PIHOLE_DNS_BIND_IP:-0.0.0.0}:53:53/tcp" + - "${PIHOLE_DNS_BIND_IP:-0.0.0.0}:53:53/udp" + - "${PIHOLE_WEB_BIND_IP:-0.0.0.0}:${PIHOLE_WEB_PORT:-8081}:80/tcp" + environment: + TZ: ${TZ:-America/Mexico_City} + WEBPASSWORD: ${PIHOLE_WEBPASSWORD:-homelab-change-me} + DNSMASQ_LISTENING: all + PIHOLE_DNS_: ${PIHOLE_UPSTREAMS:-unbound#53;1.1.1.1;9.9.9.9} + FTLCONF_dns_upstreams: ${PIHOLE_UPSTREAMS:-unbound#53;1.1.1.1;9.9.9.9} + REV_SERVER: ${PIHOLE_REV_SERVER:-false} + REV_SERVER_TARGET: ${PIHOLE_REV_SERVER_TARGET:-} + REV_SERVER_DOMAIN: ${PIHOLE_REV_SERVER_DOMAIN:-} + REV_SERVER_CIDR: ${PIHOLE_REV_SERVER_CIDR:-} + volumes: + - ${PIHOLE_DATA_DIR:-/opt/homelab-rpi-services/data/pihole/etc-pihole}:/etc/pihole + - ${PIHOLE_DNSMASQ_DIR:-/opt/homelab-rpi-services/data/pihole/etc-dnsmasq.d}:/etc/dnsmasq.d + cap_add: + - NET_ADMIN + dns: + - 127.0.0.1 + - 1.1.1.1 + networks: + - dns + + uptime-kuma: + image: ${UPTIME_KUMA_IMAGE:-louislam/uptime-kuma:1} + container_name: ${UPTIME_KUMA_CONTAINER_NAME:-homelab-uptime-kuma} + restart: unless-stopped + ports: + - "${UPTIME_KUMA_BIND_IP:-0.0.0.0}:${UPTIME_KUMA_PORT:-3001}:3001/tcp" + volumes: + - ${UPTIME_KUMA_DATA_DIR:-/opt/homelab-rpi-services/data/uptime-kuma}:/app/data + dns: + - 1.1.1.1 + - 9.9.9.9 + networks: + - dns + +networks: + dns: + name: ${RPI_SERVICES_NETWORK:-homelab-rpi-services} diff --git a/jeannie b/jeannie index 35ac8ca..30686ce 100755 --- a/jeannie +++ b/jeannie @@ -2225,6 +2225,59 @@ sudo docker compose ps install_gitea_backup_timer } +deploy_rpi_services() { + local mode="${LAB_RPI_SERVICES_DEPLOY:-true}" + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" + local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}" + local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" + local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}" + local source_dir="${REPO_ROOT}/infra/rpi-services" + local value_name + local value + + require_debian_server "rpi-services" + + if disabled_value "${mode}"; then + return 0 + fi + + if [[ ! -d "${source_dir}" ]]; then + echo "Missing ${source_dir}" >&2 + exit 1 + fi + + for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole pihole_webpassword; do + value="${!value_name}" + if [[ "${value}" == *"'"* ]]; then + echo "${value_name} cannot contain a single quote." >&2 + exit 1 + fi + done + + echo "Deploying RPi services on ${rpi_user}@${rpi_host}..." + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services" + scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "${source_dir}/docker-compose.yml" \ + "${source_dir}/adlists.txt" \ + "${source_dir}/bootstrap.sh" \ + "${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " +LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \ +LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \ +LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \ +LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \ +LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \ +PIHOLE_WEBPASSWORD='${pihole_webpassword}' \ +bash /tmp/homelab-rpi-services/bootstrap.sh" +} + gitea_bootstrap_password() { if command -v openssl >/dev/null 2>&1; then openssl rand -hex 32 @@ -3209,6 +3262,7 @@ up() { deploy_gitea bootstrap_gitea_repo + deploy_rpi_services run_pimox_pipeline run_openwrt_pipeline if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then @@ -3561,6 +3615,23 @@ status_pimox_workers() { done } +status_rpi_services() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" + + status_section "RPi Services" + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " +if [ -d '${install_dir}' ]; then + cd '${install_dir}' + sudo docker info --format 'DockerRootDir={{.DockerRootDir}}' 2>/dev/null || true + sudo docker compose ps || true +else + echo 'RPi services install directory is missing: ${install_dir}' +fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" +} + status_report() { require_debian_server "status" @@ -3585,6 +3656,7 @@ status_report() { status_kubernetes status_pimox_workers + status_rpi_services status_section "Tailscale" if command -v tailscale >/dev/null 2>&1; then @@ -3773,6 +3845,9 @@ case "${1:-}" in deploy-gitea) deploy_gitea ;; + rpi-services) + deploy_rpi_services + ;; bootstrap-gitea-repo) bootstrap_gitea_repo ;; @@ -3798,7 +3873,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" exit 1 ;; esac