Update docker root dir in rpi

README.md

@@ -685,8 +685,8 @@ Deploy or refresh them from the Debian server with:
 ```
 
 `./jeannie up` also runs this step unless
-`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under
+`LAB_RPI_SERVICES_DEPLOY=false` is set. Docker data for the Raspberry Pi is
-`/nvme-storage/docker` only when that path is an active writable mount point. If
+expected under `/nvme-storage/docker` when that path is an active writable mount point. If
 the NVMe is missing or resets, it rewrites Docker's `data-root` to
 `/var/lib/docker` and restarts Docker so DNS can still come up from the
 SD/root filesystem.
@@ -706,7 +706,8 @@ Uptime Kuma monitor seeds live in
 Kuma user exists.
 `./jeannie status` and `./jeannie doctor-rpi` test Pi-hole DNS,
 direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
-whether Docker is currently using the NVMe path or `/var/lib/docker`.
+whether Docker is currently using `/nvme-storage/docker` or has fallen back to
+`/var/lib/docker`.
 
 ## Adding Nodes
 
@@ -744,7 +745,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used
 by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto`
 only when you explicitly want Pimox to generate the MAC.
 Worker creation fails faster than template creation: static guest-network
-configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
+configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`,
 worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
 and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`.
 Static guest-network validation skips the extra LAN ICMP probe by default

README.md.tmpl

@@ -685,8 +685,8 @@ Deploy or refresh them from the Debian server with:
 ```
 
 `./{{ main_script }} up` also runs this step unless
-`LAB_RPI_SERVICES_DEPLOY=false` is set. The bootstrap prefers Docker data under
+`LAB_RPI_SERVICES_DEPLOY=false` is set. Docker data for the Raspberry Pi is
-`/nvme-storage/docker` only when that path is an active writable mount point. If
+expected under `/nvme-storage/docker` when that path is an active writable mount point. If
 the NVMe is missing or resets, it rewrites Docker's `data-root` to
 `/var/lib/docker` and restarts Docker so DNS can still come up from the
 SD/root filesystem.
@@ -706,7 +706,8 @@ Uptime Kuma monitor seeds live in
 Kuma user exists.
 `./{{ main_script }} status` and `./{{ main_script }} doctor-rpi` test Pi-hole DNS,
 direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
-whether Docker is currently using the NVMe path or `/var/lib/docker`.
+whether Docker is currently using `/nvme-storage/docker` or has fallen back to
+`/var/lib/docker`.
 
 ## Adding Nodes
 
@@ -744,7 +745,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used
 by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto`
 only when you explicitly want Pimox to generate the MAC.
 Worker creation fails faster than template creation: static guest-network
-configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
+configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`,
 worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
 and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`.
 Static guest-network validation skips the extra LAN ICMP probe by default

bootstrap/provisioning/README.md

@@ -160,7 +160,7 @@ Worker NIC MAC addresses default to the VMID-derived deterministic format used
 by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto`
 only when Pimox-generated MAC addresses are preferred.
 Worker clone failures are intentionally faster than template build failures:
-`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
+`LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=300`,
 `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
 `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600` by default. The broader
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation.

homelab.yml

@@ -47,6 +47,7 @@ hosts:
     user: jv
     lan_ip: 192.168.100.89
     tailscale_ip: 100.77.80.72
+    docker_root: /nvme-storage/docker
     docker_nvme_root: /nvme-storage/docker
     docker_fallback_root: /var/lib/docker
   opi5_pimox:

infra/rpi-services/README.md

@@ -16,10 +16,11 @@ For Pi-hole v6, the stack sets `FTLCONF_dns_interface=ALL` and
 `PIHOLE_DNS_LISTENING_MODE` so LAN clients such as the Debian control host are
 not rejected as non-local networks.
 
-The bootstrap script prefers Docker data under `/nvme-storage/docker` only when
+Docker data for the Raspberry Pi is expected under `/nvme-storage/docker` when
 that path is an active mount point and writable. If the NVMe is missing or
-reset, it rewrites Docker's `data-root` to `/var/lib/docker` and restarts
+reset, the bootstrap rewrites Docker's `data-root` to `/var/lib/docker` and
-Docker, allowing the services to come up from the SD/root filesystem instead.
+restarts Docker, allowing the services to come up from the SD/root filesystem
+instead.
 
 Service config lives under `/opt/homelab-rpi-services/data` by default. Backup
 storage is intentionally not placed on the NVMe until the disk has proven

infra/rpi-services/bootstrap.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 INSTALL_DIR="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
 DATA_DIR="${LAB_RPI_SERVICES_DATA_DIR:-${INSTALL_DIR}/data}"
-DOCKER_NVME_ROOT="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
+DOCKER_NVME_ROOT="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
 DOCKER_FALLBACK_ROOT="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
 STOP_LEGACY_PIHOLE="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}"
 PIHOLE_WEBPASSWORD="${PIHOLE_WEBPASSWORD:-}"

jeannie

@@ -52,6 +52,7 @@ mapping = {
     "hosts.rpi4.user": "LAB_RPI_USER",
     "hosts.rpi4.lan_ip": "LAB_RPI_HOST",
     "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP",
+    "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT",
     "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT",
     "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT",
     "hosts.opi5_pimox.user": "LAB_PIMOX_USER",
@@ -1090,6 +1091,28 @@ journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
 ' || true" >&2 || true
 }
 
+pimox_worker_guest_agent_recovery_hint() {
+    local host="$1"
+    local user="$2"
+    local vmid="$3"
+    local qm_bin="$4"
+
+    cat >&2 <<EOF
+
+QEMU guest agent did not become available for Pimox VM ${vmid}.
+Fast checks:
+  ssh ${user}@${host} 'sudo ${qm_bin} status ${vmid}; sudo ${qm_bin} config ${vmid}; sudo ${qm_bin} guest cmd ${vmid} network-get-interfaces'
+
+Recovery options:
+  1. If the clone is still booting slowly, rerun the Pimox stage with a longer agent wait:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=600 ./jeannie up
+  2. If the same VM still reports "QEMU guest agent is not running", recreate the worker clone:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
+  3. If a recreated worker still has no guest agent, rebuild the template and worker:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
+EOF
+}
+
 pimox_guest_exec_exitcode() {
     local guest_exec_json="$1"
 
@@ -1380,7 +1403,7 @@ configure_pimox_worker_guest_network() {
     local gateway="$7"
     local dns_servers="$8"
     local interface_name="$9"
-    local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-120}"
+    local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}"
     local deadline
     local script
     local encoded_script
@@ -1517,6 +1540,7 @@ EOF
     if [[ -n "${network_output:-}" ]]; then
         printf '%s\n' "${network_output}" | sed 's/^/    /' >&2
     fi
+    pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}"
     return 1
 }
 
@@ -2097,7 +2121,7 @@ check_rpi_docker_root_state() {
     local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
     local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
     local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
-    local nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
+    local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
     local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
 
     ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
@@ -3661,7 +3685,7 @@ deploy_rpi_services() {
     local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
     local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
     local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}"
-    local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
+    local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
     local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
     local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}"
     local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}"
@@ -3706,6 +3730,7 @@ deploy_rpi_services() {
     ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
 LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \
 LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \
+LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \
 LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \
 LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \
 LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \
@@ -6147,7 +6172,7 @@ status_rpi_services() {
     local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
     local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
     local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
-    local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
+    local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
     local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
     local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}"
     local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}"
@@ -6215,7 +6240,7 @@ if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then
     printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class'
   fi
 elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then
-  printf '%-34s ok - fallback root active\n' 'Docker root class'
+  printf '%-34s warn - fallback root active; expected ${docker_nvme_root}\n' 'Docker root class'
 else
   printf '%-34s warn - unexpected docker root\n' 'Docker root class'
 fi
@@ -6527,7 +6552,7 @@ doctor_rpi() {
     cat <<'EOF'
 
 Next steps:
-- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check.
+- Docker root should be /nvme-storage/docker; if it is /var/lib/docker, the NVMe mount is unavailable or failed the writable check.
 - If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks.
 - If Pi-hole DNS fails, clients using the RPi as DNS will be affected.
 - Reapply the stack with ./jeannie rpi-services.
@@ -6537,7 +6562,7 @@ EOF
 Doctor command: doctor-rpi
 RPi LAN host: ${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}
 RPi Tailscale IP: ${LAB_RPI_TAILSCALE_IP:-unknown}
-Docker NVMe root: ${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}
+Docker root: ${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}
 Docker fallback root: ${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}
 Services: Pi-hole, Unbound, Uptime Kuma
 EOF

scripts/capacity-report

@@ -158,14 +158,15 @@ rpi_capacity() {
     section "RPi4"
     ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" '
 set +e
+docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)"
 echo "-- memory"
 free -h
 echo
 echo "-- disk"
-df -h / /nvme-storage /var/lib/docker 2>/dev/null || df -h
+df -h / /nvme-storage "${docker_root:-/nvme-storage/docker}" 2>/dev/null || df -h
 echo
 echo "-- docker root"
-sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true
+printf "%s\n" "${docker_root:-unknown}"
 echo
 echo "-- docker disk usage"
 sudo docker system df 2>/dev/null || true
@@ -345,6 +346,8 @@ compact_rpi_capacity() {
     local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
     local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
     local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}"
+    local expected_docker_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
+    local fallback_docker_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
     local output
 
     output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" '
@@ -352,9 +355,11 @@ set +e
 docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)"
 root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")"
 nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")"
+docker_root_used="$(df -P "${docker_root:-/nvme-storage/docker}" 2>/dev/null | awk "NR == 2 { print \$5 }")"
 echo "docker_root=${docker_root:-unknown}"
 echo "root_used=${root_used:-unknown}"
 echo "nvme_used=${nvme_used:-unknown}"
+echo "docker_root_used=${docker_root_used:-unknown}"
 ' 2>/dev/null || true)"
 
     if [[ -z "${output}" ]]; then
@@ -365,10 +370,18 @@ echo "nvme_used=${nvme_used:-unknown}"
     local docker_root
     local root_used
     local nvme_used
+    local docker_root_used
     docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')"
     root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')"
     nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')"
-    emit ok "RPi4" "Docker root" "${docker_root:-unknown}"
+    docker_root_used="$(printf '%s\n' "${output}" | awk -F= '/docker_root_used=/ { print $2 }')"
+    if [[ "${docker_root}" == "${expected_docker_root}" ]]; then
+        emit ok "RPi4" "Docker root" "${docker_root}" "used=${docker_root_used:-unknown}"
+    elif [[ "${docker_root}" == "${fallback_docker_root}" ]]; then
+        emit warn "RPi4" "Docker root" "${docker_root}" "expected ${expected_docker_root}" "./jeannie doctor-rpi"
+    else
+        emit warn "RPi4" "Docker root" "${docker_root:-unknown}" "expected ${expected_docker_root}" "./jeannie doctor-rpi"
+    fi
     emit ok "RPi4" "Root disk" "${root_used:-unknown}"
     if [[ "${nvme_used:-unknown}" = "unknown" ]]; then
         emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi"

scripts/validate-homelab-inventory

@@ -64,6 +64,7 @@ else:
         "hosts.debian.docker_root",
         "hosts.rpi4.lan_ip",
         "hosts.rpi4.tailscale_ip",
+        "hosts.rpi4.docker_root",
         "hosts.opi5_pimox.lan_ip",
         "hosts.opi5_pimox.worker_storage",
         "hosts.oci_edge.public_ip",
@@ -155,6 +156,7 @@ required_paths = %w[
   hosts.debian.docker_root
   hosts.rpi4.lan_ip
   hosts.rpi4.tailscale_ip
+  hosts.rpi4.docker_root
   hosts.opi5_pimox.lan_ip
   hosts.opi5_pimox.worker_storage
   hosts.oci_edge.public_ip