diff --git a/README.md b/README.md index e9658ac..840e77e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Homelab Kubernetes Pipeline - + This repo bootstraps a hybrid kubeadm cluster and then hands app delivery to Argo CD. @@ -37,6 +37,11 @@ Tailscale IPs, public hostnames, ports, storage paths, and service ownership there before copying values into scripts, OpenTofu variables, runbooks, or service docs. Secrets and tokens do not belong in that file. +`./jeannie` reads `homelab.yml` at startup and exports matching OpenTofu +variables for the edge, provisioning, cluster, platform, and apps stacks. This +keeps values such as the Debian LAN IP, Debian Tailscale IP, RPi LAN IP, OCI IP, +Traefik MetalLB IP, and domain name from drifting across scripts and templates. + Run `./jeannie up` and `./jeannie nuke` only from the Debian homelab server. The script intentionally refuses to run from non-Debian machines so a laptop cannot accidentally modify the cluster. @@ -211,6 +216,16 @@ service is already managed manually. The default Gitea target is `jv@192.168.100.73`, install directory `/data/homelab-gitea`, HTTP port `3000`, and SSH port `32222`. +Before continuing into Pimox, cluster, apps, or edge changes, `./jeannie up` +runs `./jeannie preflight`. The preflight verifies Gitea, Debian Docker +root, Debian Tailscale IP, RPi Docker root fallback state, RPi Tailscale IP, +Pimox worker storage, and OCI edge SSH. Run it directly when changing inventory +or host storage/networking: + +```bash +./jeannie preflight +``` + ## Validation Useful checks after a rebuild: @@ -278,10 +293,15 @@ the NVMe is missing or resets, it rewrites Docker's `data-root` to SD/root filesystem. Pi-hole uses Unbound first and public fallback upstreams after that, so DNS can -continue resolving if the Unbound container stops. Managed adlists live in -`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without -deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, -the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. +continue resolving if the Unbound container stops. Repo-managed Pi-hole state +lives in `infra/rpi-services/adlists.txt`, +`infra/rpi-services/local-dns-records.txt`, +`infra/rpi-services/cname-records.txt`, and +`infra/rpi-services/static-dhcp-hosts.txt`. The bootstrap inserts adlists without +deleting manually added Pi-hole lists, then renders the DNS/CNAME/DHCP files into +a managed dnsmasq config inside the Pi-hole container. If `PIHOLE_WEBPASSWORD` +is not provided, the bootstrap generates one and keeps it in +`/opt/homelab-rpi-services/.env`. Uptime Kuma monitor seeds live in `infra/rpi-services/uptime-kuma-monitors.json` and are applied after the first Kuma user exists. diff --git a/README.md.tmpl b/README.md.tmpl index d05b37a..4afc533 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -1,6 +1,6 @@ # Homelab Kubernetes Pipeline - + This repo bootstraps a hybrid kubeadm cluster and then hands app delivery to Argo CD. @@ -37,6 +37,11 @@ Tailscale IPs, public hostnames, ports, storage paths, and service ownership there before copying values into scripts, OpenTofu variables, runbooks, or service docs. Secrets and tokens do not belong in that file. +`./{{ main_script }}` reads `homelab.yml` at startup and exports matching OpenTofu +variables for the edge, provisioning, cluster, platform, and apps stacks. This +keeps values such as the Debian LAN IP, Debian Tailscale IP, RPi LAN IP, OCI IP, +Traefik MetalLB IP, and domain name from drifting across scripts and templates. + Run `./{{ main_script }} up` and `./{{ main_script }} nuke` only from the Debian homelab server. The script intentionally refuses to run from non-Debian machines so a laptop cannot accidentally modify the cluster. @@ -211,6 +216,16 @@ service is already managed manually. The default Gitea target is `jv@192.168.100.73`, install directory `/data/homelab-gitea`, HTTP port `3000`, and SSH port `32222`. +Before continuing into Pimox, cluster, apps, or edge changes, `./{{ main_script }} up` +runs `./{{ main_script }} preflight`. The preflight verifies Gitea, Debian Docker +root, Debian Tailscale IP, RPi Docker root fallback state, RPi Tailscale IP, +Pimox worker storage, and OCI edge SSH. Run it directly when changing inventory +or host storage/networking: + +```bash +./{{ main_script }} preflight +``` + ## Validation Useful checks after a rebuild: @@ -278,10 +293,15 @@ the NVMe is missing or resets, it rewrites Docker's `data-root` to SD/root filesystem. Pi-hole uses Unbound first and public fallback upstreams after that, so DNS can -continue resolving if the Unbound container stops. Managed adlists live in -`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without -deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, -the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. +continue resolving if the Unbound container stops. Repo-managed Pi-hole state +lives in `infra/rpi-services/adlists.txt`, +`infra/rpi-services/local-dns-records.txt`, +`infra/rpi-services/cname-records.txt`, and +`infra/rpi-services/static-dhcp-hosts.txt`. The bootstrap inserts adlists without +deleting manually added Pi-hole lists, then renders the DNS/CNAME/DHCP files into +a managed dnsmasq config inside the Pi-hole container. If `PIHOLE_WEBPASSWORD` +is not provided, the bootstrap generates one and keeps it in +`/opt/homelab-rpi-services/.env`. Uptime Kuma monitor seeds live in `infra/rpi-services/uptime-kuma-monitors.json` and are applied after the first Kuma user exists. diff --git a/docs.yml b/docs.yml deleted file mode 100644 index a1fb761..0000000 --- a/docs.yml +++ /dev/null @@ -1 +0,0 @@ -main_script: jeannie diff --git a/homelab.yml b/homelab.yml index 670b416..1e9eee5 100644 --- a/homelab.yml +++ b/homelab.yml @@ -90,6 +90,10 @@ services: pihole_web_port: 8081 uptime_kuma_port: 3001 unbound_container_port: 53 + adlists_file: infra/rpi-services/adlists.txt + local_dns_records_file: infra/rpi-services/local-dns-records.txt + cname_records_file: infra/rpi-services/cname-records.txt + static_dhcp_hosts_file: infra/rpi-services/static-dhcp-hosts.txt uptime_kuma_seed_file: infra/rpi-services/uptime-kuma-monitors.json arr: host: debian diff --git a/infra/rpi-services/README.md b/infra/rpi-services/README.md index 90e3912..56df643 100644 --- a/infra/rpi-services/README.md +++ b/infra/rpi-services/README.md @@ -39,8 +39,17 @@ LAB_RPI_STOP_LEGACY_PIHOLE=false ./jeannie rpi-services If `PIHOLE_WEBPASSWORD` is not provided, the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env` for future runs. -Managed Pi-hole adlists are stored in `adlists.txt`. The bootstrap inserts them -without deleting existing manual Pi-hole lists. +Managed Pi-hole state is stored in: + +- `adlists.txt` +- `local-dns-records.txt` +- `cname-records.txt` +- `static-dhcp-hosts.txt` + +The bootstrap inserts adlists without deleting existing manual Pi-hole lists. It +renders local DNS, CNAME, and optional static DHCP entries into +`/etc/dnsmasq.d/02-homelab-managed-dns.conf` inside the Pi-hole container, then +reloads DNS. Managed Uptime Kuma monitors are stored in `uptime-kuma-monitors.json`. The bootstrap seeds them after the Uptime Kuma database exists. If Uptime Kuma has diff --git a/infra/rpi-services/bootstrap.sh b/infra/rpi-services/bootstrap.sh index c16d00f..d120a71 100755 --- a/infra/rpi-services/bootstrap.sh +++ b/infra/rpi-services/bootstrap.sh @@ -207,6 +207,57 @@ PY sudo docker exec "${pihole_container}" pihole -g } +sync_pihole_managed_dns() { + local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" + local local_records_file="${INSTALL_DIR}/local-dns-records.txt" + local cname_records_file="${INSTALL_DIR}/cname-records.txt" + local static_dhcp_hosts_file="${INSTALL_DIR}/static-dhcp-hosts.txt" + local dnsmasq_file + + dnsmasq_file="$(mktemp)" + python3 - "${local_records_file}" "${cname_records_file}" "${static_dhcp_hosts_file}" >"${dnsmasq_file}" <<'PY' +import sys + +local_records_file, cname_records_file, static_dhcp_hosts_file = sys.argv[1:4] + +print("# Generated by homelab rpi-services bootstrap. Edit repo files, not this file.") + +def useful_lines(path): + try: + with open(path, encoding="utf-8") as handle: + for raw_line in handle: + line = raw_line.strip() + if line and not line.startswith("#"): + yield line + except FileNotFoundError: + return + +for line in useful_lines(local_records_file): + fields = line.split() + if len(fields) != 2: + raise SystemExit(f"Invalid local DNS record: {line}") + hostname, address = fields + print(f"address=/{hostname}/{address}") + +for line in useful_lines(cname_records_file): + fields = line.split() + if len(fields) != 2: + raise SystemExit(f"Invalid CNAME record: {line}") + alias, target = fields + print(f"cname={alias},{target}") + +for line in useful_lines(static_dhcp_hosts_file): + if not line.startswith("dhcp-host="): + raise SystemExit(f"Invalid static DHCP entry: {line}") + print(line) +PY + sudo docker cp "${dnsmasq_file}" "${pihole_container}:/etc/dnsmasq.d/02-homelab-managed-dns.conf" + rm -f "${dnsmasq_file}" + if ! sudo docker exec "${pihole_container}" pihole restartdns reload; then + sudo docker restart "${pihole_container}" >/dev/null + fi +} + seed_uptime_kuma_monitors() { local uptime_kuma_container="${UPTIME_KUMA_CONTAINER_NAME:-homelab-uptime-kuma}" local monitor_file="${INSTALL_DIR}/uptime-kuma-monitors.json" @@ -252,6 +303,9 @@ fi sudo mkdir -p "${INSTALL_DIR}" "${DATA_DIR}/pihole/etc-pihole" "${DATA_DIR}/pihole/etc-dnsmasq.d" "${DATA_DIR}/uptime-kuma" sudo cp "${SCRIPT_DIR}/docker-compose.yml" "${INSTALL_DIR}/docker-compose.yml" sudo cp "${SCRIPT_DIR}/adlists.txt" "${INSTALL_DIR}/adlists.txt" +sudo cp "${SCRIPT_DIR}/local-dns-records.txt" "${INSTALL_DIR}/local-dns-records.txt" +sudo cp "${SCRIPT_DIR}/cname-records.txt" "${INSTALL_DIR}/cname-records.txt" +sudo cp "${SCRIPT_DIR}/static-dhcp-hosts.txt" "${INSTALL_DIR}/static-dhcp-hosts.txt" sudo cp "${SCRIPT_DIR}/uptime-kuma-monitors.json" "${INSTALL_DIR}/uptime-kuma-monitors.json" sudo cp "${SCRIPT_DIR}/seed-uptime-kuma.js" "${INSTALL_DIR}/seed-uptime-kuma.js" write_env_file @@ -262,5 +316,6 @@ cd "${INSTALL_DIR}" sudo docker compose pull sudo docker compose up -d --remove-orphans sync_pihole_adlists +sync_pihole_managed_dns seed_uptime_kuma_monitors sudo docker compose ps diff --git a/infra/rpi-services/cname-records.txt b/infra/rpi-services/cname-records.txt new file mode 100644 index 0000000..4c032ce --- /dev/null +++ b/infra/rpi-services/cname-records.txt @@ -0,0 +1,5 @@ +# Repo-managed Pi-hole CNAME records. +# Format: alias target +git.homelab.local gitea.homelab.local +pihole.homelab.local rpi4.homelab.local +uptime.homelab.local rpi4.homelab.local diff --git a/infra/rpi-services/local-dns-records.txt b/infra/rpi-services/local-dns-records.txt new file mode 100644 index 0000000..b0e0a2e --- /dev/null +++ b/infra/rpi-services/local-dns-records.txt @@ -0,0 +1,7 @@ +# Repo-managed Pi-hole local DNS records. +# Format: hostname ip-address +debian.homelab.local 192.168.100.73 +gitea.homelab.local 192.168.100.73 +rpi4.homelab.local 192.168.100.89 +opi5-pimox.homelab.local 192.168.100.80 +traefik.homelab.local 192.168.100.240 diff --git a/infra/rpi-services/static-dhcp-hosts.txt b/infra/rpi-services/static-dhcp-hosts.txt new file mode 100644 index 0000000..49b4955 --- /dev/null +++ b/infra/rpi-services/static-dhcp-hosts.txt @@ -0,0 +1,3 @@ +# Optional repo-managed dnsmasq DHCP host entries. +# Keep this file non-secret. Use one dnsmasq dhcp-host entry per line, for example: +# dhcp-host=AA:BB:CC:DD:EE:FF,192.168.100.10,my-host,infinite diff --git a/jeannie b/jeannie index 43c8754..e3823ee 100755 --- a/jeannie +++ b/jeannie @@ -3,7 +3,6 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILDX_CONFIG="/tmp/buildx-config.toml" -KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-/home/jv/.kube/config}}" HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}" COSIGN_KEY_PREFIX="${COSIGN_KEY_PREFIX:-${HOMELAB_STATE_DIR}/cosign/cosign}" COSIGN_KEY_PATH="${COSIGN_KEY_PATH:-${COSIGN_KEY_PREFIX}.key}" @@ -16,6 +15,146 @@ HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Doc trap 'rm -f "${BUILDX_CONFIG}"' EXIT +load_homelab_inventory_defaults() { + local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" + local rendered_defaults + + if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then + return 0 + fi + + rendered_defaults="$(python3 - "${inventory_file}" <<'PY' +import re +import shlex +import sys + +inventory_file = sys.argv[1] +mapping = { + "domain.base": "LAB_DOMAIN", + "domain.public_url": "LAB_PUBLIC_URL", + "domain.gitea_url": "LAB_GITEA_ROOT_URL", + "network.lan_cidr": "LAB_LAN_CIDR", + "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX", + "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP", + "hosts.debian.user": "LAB_DEBIAN_USER", + "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP", + "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP", + "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT", + "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH", + "hosts.rpi4.user": "LAB_RPI_USER", + "hosts.rpi4.lan_ip": "LAB_RPI_HOST", + "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", + "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", + "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", + "hosts.opi5_pimox.user": "LAB_PIMOX_USER", + "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST", + "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE", + "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE", + "hosts.oci_edge.user": "LAB_EDGE_USER", + "hosts.oci_edge.public_ip": "LAB_EDGE_HOST", + "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR", + "services.gitea.http_port": "LAB_GITEA_HTTP_PORT", + "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", + "services.gitea.root_url": "LAB_GITEA_ROOT_URL", + "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", + "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", + "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT", + "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT", + "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", + "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", +} + +stack = [] +values = {} +pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") + +with open(inventory_file, encoding="utf-8") as handle: + for raw_line in handle: + if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): + continue + match = pattern.match(raw_line.rstrip("\n")) + if not match: + continue + indent = len(match.group(1)) + key = match.group(2) + value = (match.group(3) or "").strip() + while stack and stack[-1][0] >= indent: + stack.pop() + path = ".".join([item[1] for item in stack] + [key]) + if value == "": + stack.append((indent, key)) + continue + if " #" in value: + value = value.split(" #", 1)[0].strip() + value = value.strip("\"'") + values[path] = value + +for path, env_name in mapping.items(): + value = values.get(path) + if value: + print(f": ${{{env_name}:={shlex.quote(value)}}}") + +debian_ip = values.get("hosts.debian.lan_ip") +gitea_http = values.get("services.gitea.http_port") +if debian_ip: + print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}") +if debian_ip and gitea_http: + print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}") +gitea_ts = values.get("hosts.debian.tailscale_ip") +if gitea_ts: + print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}") +PY +)" + + if [[ -n "${rendered_defaults}" ]]; then + eval "${rendered_defaults}" + fi +} + +load_homelab_inventory_defaults + +export_if_unset() { + local name="$1" + local value="$2" + + if [[ -z "${value}" ]]; then + return 0 + fi + if [[ -z "${!name:-}" ]]; then + printf -v "${name}" '%s' "${value}" + fi + # shellcheck disable=SC2163 + export "${name?}" +} + +export_homelab_inventory_tf_vars() { + export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}" + export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}" + export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}" + export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}" + export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}" + export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}" + export_if_unset TF_VAR_pimox_template_build_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}" + export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}" + export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}" + export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}" + export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}" + export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" + export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" + export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" + export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" +} + +export_homelab_inventory_tf_vars + +KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}" + require_debian_server() { local command_name="$1" local os_id="" @@ -876,6 +1015,171 @@ prepare_cluster_worker_var_file() { export LAB_CLUSTER_VAR_FILE="${var_file}" } +preflight_check() { + local description="$1" + local output + shift + + printf '%s %s... ' '-->' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + return 0 + fi + + printf 'failed\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed 's/^/ /' >&2 + fi + return 1 +} + +preflight_warn() { + local description="$1" + local output + shift + + printf '%s %s... ' '-->' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + return 0 + fi + + printf 'warning\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed 's/^/ /' >&2 + fi + return 0 +} + +check_gitea_reachable() { + local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" + + curl -fsS --max-time 10 "${gitea_url}" >/dev/null +} + +check_debian_docker_root() { + local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/data/docker}" + local actual_root + + if ! command -v docker >/dev/null 2>&1; then + echo "docker command was not found on Debian host" >&2 + return 1 + fi + actual_root="$(sudo docker info --format '{{.DockerRootDir}}')" + if [[ "${actual_root}" != "${expected_root}" ]]; then + echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2 + return 1 + fi +} + +check_debian_tailscale_ip() { + local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}" + + if [[ -z "${expected_ip}" ]]; then + return 0 + fi + if ! command -v tailscale >/dev/null 2>&1; then + echo "tailscale command was not found on Debian host" >&2 + return 1 + fi + if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then + echo "Debian Tailscale IP does not include ${expected_ip}" >&2 + tailscale ip -4 >&2 || true + return 1 + fi +} + +check_edge_ssh() { + local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true" +} + +check_rpi_docker_root_state() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu +actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" +if [ \"\$actual_root\" = '${nvme_root}' ]; then + mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; } + exit 0 +fi +if [ \"\$actual_root\" = '${fallback_root}' ]; then + echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2 + exit 0 +fi +echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2 +exit 1" +} + +check_rpi_tailscale_ip() { + local expected_ip="${LAB_RPI_TAILSCALE_IP:-}" + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + if [[ -z "${expected_ip}" ]]; then + return 0 + fi + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'" +} + +check_pimox_storage() { + local mode="${LAB_PIMOX_PIPELINE:-true}" + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" + + if disabled_value "${mode}"; then + return 0 + fi + + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" +if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z \"\$pvesm_cmd\" ]; then + echo 'pvesm was not found' >&2 + exit 1 +fi +sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'" +} + +homelab_preflight() { + local failures=0 + + require_debian_server "preflight" + + if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then + echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}." + return 0 + fi + + echo "Running homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..." + + preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1)) + preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/data/docker}" check_debian_docker_root || failures=$((failures + 1)) + preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1)) + preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1)) + preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip + preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) + preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:-132.145.170.74}" check_edge_ssh || failures=$((failures + 1)) + + if ((failures > 0)); then + echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2 + exit 1 + fi + + echo "Preflight checks passed." +} + run_pimox_pipeline() { local mode="${LAB_PIMOX_PIPELINE:-true}" local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" @@ -2126,15 +2430,15 @@ wait_for_deployment_ready() { deploy_gitea() { local mode="${LAB_GITEA_DEPLOY:-true}" - local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" - local gitea_user="${LAB_GITEA_USER:-jv}" + local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}" + local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}" local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}" local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}" local http_port="${LAB_GITEA_HTTP_PORT:-3000}" local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" - local domain="${LAB_GITEA_DOMAIN:-lab2025.duckdns.org}" - local root_url="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}" + local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:-lab2025.duckdns.org}}" + local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml" @@ -2265,6 +2569,9 @@ deploy_rpi_services() { scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ "${source_dir}/docker-compose.yml" \ "${source_dir}/adlists.txt" \ + "${source_dir}/local-dns-records.txt" \ + "${source_dir}/cname-records.txt" \ + "${source_dir}/static-dhcp-hosts.txt" \ "${source_dir}/uptime-kuma-monitors.json" \ "${source_dir}/seed-uptime-kuma.js" \ "${source_dir}/bootstrap.sh" \ @@ -3265,6 +3572,7 @@ up() { deploy_gitea bootstrap_gitea_repo deploy_rpi_services + homelab_preflight run_pimox_pipeline run_openwrt_pipeline if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then @@ -3291,6 +3599,7 @@ rebuild_cluster() { echo "Rebuilding the Kubernetes cluster without touching external Gitea..." + homelab_preflight nuke run_pimox_pipeline run_openwrt_pipeline @@ -3753,10 +4062,10 @@ status_report() { fi status_section "HTTP" - status_http "website public" "https://lab2025.duckdns.org/" - status_http "gitea public" "https://lab2025.duckdns.org/git/" - status_http "gitea local" "http://127.0.0.1:3000/" - status_http "traefik lb" "http://192.168.100.240/" + status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" + status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" + status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" + status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" status_http "arr radarr" "http://127.0.0.1:7878/" status_http "arr sonarr" "http://127.0.0.1:8989/" status_http "arr prowlarr" "http://127.0.0.1:9696/" @@ -3772,8 +4081,8 @@ doctor_edge() { require_debian_server "doctor-edge" status_section "Doctor Edge" - status_http "website public" "https://lab2025.duckdns.org/" - status_http "gitea public" "https://lab2025.duckdns.org/git/" + status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" + status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" printf '\n-- edge backend reachability\n' if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " @@ -3810,7 +4119,7 @@ doctor_gitea() { status_section "Doctor Gitea" status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/" - status_http "gitea public" "https://lab2025.duckdns.org/git/" + status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" status_compose_stack "gitea" "/data/homelab-gitea" status_run "gitea git remote" git ls-remote gitea main @@ -3856,7 +4165,7 @@ doctor_cluster() { status_systemd_units kubelet containerd docker status_kubernetes status_pimox_workers - status_http "traefik lb" "http://192.168.100.240/" + status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" printf '\n-- local CRI containers\n' if command -v crictl >/dev/null 2>&1; then @@ -4078,6 +4387,9 @@ case "${1:-}" in doctor-cluster) doctor_cluster ;; + preflight) + homelab_preflight + ;; openwrt) openwrt ;; @@ -4085,7 +4397,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/render-docs b/scripts/render-docs index 6bdf8f1..5d638bb 100755 --- a/scripts/render-docs +++ b/scripts/render-docs @@ -3,7 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -CONFIG_FILE="${REPO_ROOT}/docs.yml" +INVENTORY_FILE="${REPO_ROOT}/homelab.yml" README_TEMPLATE="${REPO_ROOT}/README.md.tmpl" README_OUTPUT="${REPO_ROOT}/README.md" @@ -18,24 +18,43 @@ Options: USAGE } -read_config_value() { - local key="$1" +read_inventory_value() { + local path="$1" - awk -F ':' -v key="${key}" ' - $1 == key { - value = $0 - sub("^[^:]*:[[:space:]]*", "", value) - gsub(/^[[:space:]"'"'"']+|[[:space:]"'"'"']+$/, "", value) - print value - found = 1 - exit - } - END { - if (!found) { - exit 1 - } - } - ' "${CONFIG_FILE}" + python3 - "${INVENTORY_FILE}" "${path}" <<'PY' +import re +import sys + +inventory_file, requested_path = sys.argv[1:3] +stack = [] +values = {} +pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") + +with open(inventory_file, encoding="utf-8") as handle: + for raw_line in handle: + if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): + continue + match = pattern.match(raw_line.rstrip("\n")) + if not match: + continue + indent = len(match.group(1)) + key = match.group(2) + value = (match.group(3) or "").strip() + while stack and stack[-1][0] >= indent: + stack.pop() + current_path = ".".join([item[1] for item in stack] + [key]) + if value == "": + stack.append((indent, key)) + continue + if " #" in value: + value = value.split(" #", 1)[0].strip() + values[current_path] = value.strip("\"'") + +value = values.get(requested_path) +if not value: + raise SystemExit(1) +print(value) +PY } render_readme() { @@ -71,8 +90,8 @@ main() { ;; esac - if [[ ! -s "${CONFIG_FILE}" ]]; then - printf 'Missing docs config: %s\n' "${CONFIG_FILE}" >&2 + if [[ ! -s "${INVENTORY_FILE}" ]]; then + printf 'Missing homelab inventory: %s\n' "${INVENTORY_FILE}" >&2 return 1 fi if [[ ! -s "${README_TEMPLATE}" ]]; then @@ -80,9 +99,9 @@ main() { return 1 fi - main_script="$(read_config_value main_script)" + main_script="$(read_inventory_value metadata.main_script)" if [[ -z "${main_script}" ]]; then - printf 'docs.yml main_script must not be empty\n' >&2 + printf 'homelab.yml metadata.main_script must not be empty\n' >&2 return 1 fi