diff --git a/README.md b/README.md index 99c8cf8..a323eb4 100644 --- a/README.md +++ b/README.md @@ -685,8 +685,8 @@ Deploy or refresh them from the Debian server with: ``` `./jeannie up` also runs this step unless -`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under -`/nvme-storage/docker` only when that path is an active writable mount point. If +`LAB_RPI_SERVICES_DEPLOY=false` is set. Docker data for the Raspberry Pi is +expected under `/nvme-storage/docker` when that path is an active writable mount point. If the NVMe is missing or resets, it rewrites Docker's `data-root` to `/var/lib/docker` and restarts Docker so DNS can still come up from the SD/root filesystem. @@ -706,7 +706,8 @@ Uptime Kuma monitor seeds live in Kuma user exists. `./jeannie status` and `./jeannie doctor-rpi` test Pi-hole DNS, direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and -whether Docker is currently using the NVMe path or `/var/lib/docker`. +whether Docker is currently using `/nvme-storage/docker` or has fallen back to +`/var/lib/docker`. ## Adding Nodes @@ -744,7 +745,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when you explicitly want Pimox to generate the MAC. Worker creation fails faster than template creation: static guest-network -configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, +configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`. Static guest-network validation skips the extra LAN ICMP probe by default diff --git a/README.md.tmpl b/README.md.tmpl index 720f8b9..3c1f61c 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -685,8 +685,8 @@ Deploy or refresh them from the Debian server with: ``` `./{{ main_script }} up` also runs this step unless -`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under -`/nvme-storage/docker` only when that path is an active writable mount point. If +`LAB_RPI_SERVICES_DEPLOY=false` is set. Docker data for the Raspberry Pi is +expected under `/nvme-storage/docker` when that path is an active writable mount point. If the NVMe is missing or resets, it rewrites Docker's `data-root` to `/var/lib/docker` and restarts Docker so DNS can still come up from the SD/root filesystem. @@ -706,7 +706,8 @@ Uptime Kuma monitor seeds live in Kuma user exists. `./{{ main_script }} status` and `./{{ main_script }} doctor-rpi` test Pi-hole DNS, direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and -whether Docker is currently using the NVMe path or `/var/lib/docker`. +whether Docker is currently using `/nvme-storage/docker` or has fallen back to +`/var/lib/docker`. ## Adding Nodes @@ -744,7 +745,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when you explicitly want Pimox to generate the MAC. Worker creation fails faster than template creation: static guest-network -configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, +configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`. Static guest-network validation skips the extra LAN ICMP probe by default diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 7ec8d26..c196983 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -160,7 +160,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when Pimox-generated MAC addresses are preferred. Worker clone failures are intentionally faster than template build failures: -`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, +`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`, `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600` by default. The broader `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation. diff --git a/homelab.yml b/homelab.yml index 02fc908..6bb5dbc 100644 --- a/homelab.yml +++ b/homelab.yml @@ -47,6 +47,7 @@ hosts: user: jv lan_ip: 192.168.100.89 tailscale_ip: 100.77.80.72 + docker_root: /nvme-storage/docker docker_nvme_root: /nvme-storage/docker docker_fallback_root: /var/lib/docker opi5_pimox: diff --git a/infra/rpi-services/README.md b/infra/rpi-services/README.md index e5d9ad7..a38a960 100644 --- a/infra/rpi-services/README.md +++ b/infra/rpi-services/README.md @@ -16,10 +16,11 @@ For Pi-hole v6, the stack sets `FTLCONF_dns_interface=ALL` and `PIHOLE_DNS_LISTENING_MODE` so LAN clients such as the Debian control host are not rejected as non-local networks. -The bootstrap script prefers Docker data under `/nvme-storage/docker` only when +Docker data for the Raspberry Pi is expected under `/nvme-storage/docker` when that path is an active mount point and writable. If the NVMe is missing or -reset, it rewrites Docker's `data-root` to `/var/lib/docker` and restarts -Docker, allowing the services to come up from the SD/root filesystem instead. +reset, the bootstrap rewrites Docker's `data-root` to `/var/lib/docker` and +restarts Docker, allowing the services to come up from the SD/root filesystem +instead. Service config lives under `/opt/homelab-rpi-services/data` by default. Backup storage is intentionally not placed on the NVMe until the disk has proven diff --git a/infra/rpi-services/bootstrap.sh b/infra/rpi-services/bootstrap.sh index 39f711e..f7f1b2d 100755 --- a/infra/rpi-services/bootstrap.sh +++ b/infra/rpi-services/bootstrap.sh @@ -4,7 +4,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INSTALL_DIR="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" DATA_DIR="${LAB_RPI_SERVICES_DATA_DIR:-${INSTALL_DIR}/data}" -DOCKER_NVME_ROOT="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" +DOCKER_NVME_ROOT="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" DOCKER_FALLBACK_ROOT="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" STOP_LEGACY_PIHOLE="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" PIHOLE_WEBPASSWORD="${PIHOLE_WEBPASSWORD:-}" diff --git a/jeannie b/jeannie index 46fe009..3897d4f 100755 --- a/jeannie +++ b/jeannie @@ -52,6 +52,7 @@ mapping = { "hosts.rpi4.user": "LAB_RPI_USER", "hosts.rpi4.lan_ip": "LAB_RPI_HOST", "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", + "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT", "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", "hosts.opi5_pimox.user": "LAB_PIMOX_USER", @@ -1090,6 +1091,28 @@ journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true ' || true" >&2 || true } +pimox_worker_guest_agent_recovery_hint() { + local host="$1" + local user="$2" + local vmid="$3" + local qm_bin="$4" + + cat >&2 <&2 fi + pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}" return 1 } @@ -2097,7 +2121,7 @@ check_rpi_docker_root_state() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu @@ -3661,7 +3685,7 @@ deploy_rpi_services() { local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}" - local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}" @@ -3706,6 +3730,7 @@ deploy_rpi_services() { ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \ LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \ +LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \ LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \ LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \ LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \ @@ -6147,7 +6172,7 @@ status_rpi_services() { local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" - local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}" @@ -6215,7 +6240,7 @@ if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class' fi elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then - printf '%-34s ok - fallback root active\n' 'Docker root class' + printf '%-34s warn - fallback root active; expected ${docker_nvme_root}\n' 'Docker root class' else printf '%-34s warn - unexpected docker root\n' 'Docker root class' fi @@ -6527,7 +6552,7 @@ doctor_rpi() { cat <<'EOF' Next steps: -- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check. +- Docker root should be /nvme-storage/docker; if it is /var/lib/docker, the NVMe mount is unavailable or failed the writable check. - If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks. - If Pi-hole DNS fails, clients using the RPi as DNS will be affected. - Reapply the stack with ./jeannie rpi-services. @@ -6537,7 +6562,7 @@ EOF Doctor command: doctor-rpi RPi LAN host: ${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}} RPi Tailscale IP: ${LAB_RPI_TAILSCALE_IP:-unknown} -Docker NVMe root: ${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker} +Docker root: ${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}} Docker fallback root: ${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker} Services: Pi-hole, Unbound, Uptime Kuma EOF diff --git a/scripts/capacity-report b/scripts/capacity-report index 3f92d3d..3a340ab 100755 --- a/scripts/capacity-report +++ b/scripts/capacity-report @@ -158,14 +158,15 @@ rpi_capacity() { section "RPi4" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" ' set +e +docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)" echo "-- memory" free -h echo echo "-- disk" -df -h / /nvme-storage /var/lib/docker 2>/dev/null || df -h +df -h / /nvme-storage "${docker_root:-/nvme-storage/docker}" 2>/dev/null || df -h echo echo "-- docker root" -sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true +printf "%s\n" "${docker_root:-unknown}" echo echo "-- docker disk usage" sudo docker system df 2>/dev/null || true @@ -345,6 +346,8 @@ compact_rpi_capacity() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}" + local expected_docker_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" + local fallback_docker_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" local output output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" ' @@ -352,9 +355,11 @@ set +e docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)" root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")" nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")" +docker_root_used="$(df -P "${docker_root:-/nvme-storage/docker}" 2>/dev/null | awk "NR == 2 { print \$5 }")" echo "docker_root=${docker_root:-unknown}" echo "root_used=${root_used:-unknown}" echo "nvme_used=${nvme_used:-unknown}" +echo "docker_root_used=${docker_root_used:-unknown}" ' 2>/dev/null || true)" if [[ -z "${output}" ]]; then @@ -365,10 +370,18 @@ echo "nvme_used=${nvme_used:-unknown}" local docker_root local root_used local nvme_used + local docker_root_used docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')" root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')" nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')" - emit ok "RPi4" "Docker root" "${docker_root:-unknown}" + docker_root_used="$(printf '%s\n' "${output}" | awk -F= '/docker_root_used=/ { print $2 }')" + if [[ "${docker_root}" == "${expected_docker_root}" ]]; then + emit ok "RPi4" "Docker root" "${docker_root}" "used=${docker_root_used:-unknown}" + elif [[ "${docker_root}" == "${fallback_docker_root}" ]]; then + emit warn "RPi4" "Docker root" "${docker_root}" "expected ${expected_docker_root}" "./jeannie doctor-rpi" + else + emit warn "RPi4" "Docker root" "${docker_root:-unknown}" "expected ${expected_docker_root}" "./jeannie doctor-rpi" + fi emit ok "RPi4" "Root disk" "${root_used:-unknown}" if [[ "${nvme_used:-unknown}" = "unknown" ]]; then emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi" diff --git a/scripts/validate-homelab-inventory b/scripts/validate-homelab-inventory index 54763cf..a8322fd 100755 --- a/scripts/validate-homelab-inventory +++ b/scripts/validate-homelab-inventory @@ -64,6 +64,7 @@ else: "hosts.debian.docker_root", "hosts.rpi4.lan_ip", "hosts.rpi4.tailscale_ip", + "hosts.rpi4.docker_root", "hosts.opi5_pimox.lan_ip", "hosts.opi5_pimox.worker_storage", "hosts.oci_edge.public_ip", @@ -155,6 +156,7 @@ required_paths = %w[ hosts.debian.docker_root hosts.rpi4.lan_ip hosts.rpi4.tailscale_ip + hosts.rpi4.docker_root hosts.opi5_pimox.lan_ip hosts.opi5_pimox.worker_storage hosts.oci_edge.public_ip