Fail Pimox worker network issues faster

README.md

@@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network
 configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
 worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
 and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`.
+After static network assignment, the guest must ping the gateway and
+`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set
+`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After
+qemu-guest-agent reports an IP, Debian must be able to ping the worker before
+SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when
+worker ICMP is intentionally blocked.
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
 
 Long-running `./jeannie up` steps keep compact output by default, but

README.md.tmpl

@@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network
 configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
 worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
 and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`.
+After static network assignment, the guest must ping the gateway and
+`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set
+`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After
+qemu-guest-agent reports an IP, Debian must be able to ping the worker before
+SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when
+worker ICMP is intentionally blocked.
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
 
 Long-running `./{{ main_script }} up` steps keep compact output by default, but

bootstrap/provisioning/README.md

@@ -155,6 +155,12 @@ Worker clone failures are intentionally faster than template build failures:
 `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
 `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180` by default. The broader
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation.
+The static network step also checks guest LAN reachability by pinging the
+gateway and `${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`. The Debian
+host then pings the reported worker IP before continuing SSH retries. Disable
+those checks only for ICMP-blocked networks with
+`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` or
+`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false`.
 
 Worker indexes are stable: index `1` maps to VMID `9010`,
 `pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`,

docs/jeannie.1.md

@@ -539,6 +539,14 @@ falls back to the first Lynis warning, then the first suggestion.
 : Desired Pimox worker count when not supplied by generated cluster topology
 state.
 
+`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
+: Require the worker guest to ping the LAN gateway and probe host after static
+IP assignment. Defaults to `true`.
+
+`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE`
+: Require the Debian host to ping the worker IP before waiting for SSH. Defaults
+to `true`.
+
 `LAB_SKIP_PREFLIGHT=true`
 : Skip preflight checks. Use only for targeted troubleshooting.
 

jeannie

@@ -1069,6 +1069,9 @@ wait_for_pimox_guest_ssh() {
     local last_guest_ip=""
     local last_known_hosts_ip=""
     local last_ssh_output=""
+    local last_host_probe_output=""
+    local host_probe_failures=0
+    local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-true}"
     local next_log
     local ssh_deadline=0
     local ssh_output
@@ -1082,6 +1085,10 @@ wait_for_pimox_guest_ssh() {
         echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
         return 1
     fi
+    if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
+        echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
+        return 1
+    fi
     mkdir -p "$(dirname "${known_hosts_file}")"
     touch "${known_hosts_file}"
     chmod 0600 "${known_hosts_file}"
@@ -1101,6 +1108,26 @@ wait_for_pimox_guest_ssh() {
                 ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true
                 last_known_hosts_ip="${guest_ip}"
             fi
+            if truthy "${require_host_probe}"; then
+                if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then
+                    host_probe_failures=0
+                else
+                    host_probe_failures=$((host_probe_failures + 1))
+                    last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}"
+                    if ((host_probe_failures >= 3)); then
+                        echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2
+                        echo "Latest host probe: ${last_host_probe_output}" >&2
+                        echo "Fast checks:" >&2
+                        echo "  ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2
+                        echo "  arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2
+                        echo "  Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2
+                        pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
+                        return 1
+                    fi
+                    sleep 10
+                    continue
+                fi
+            fi
             if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then
                 printf '%s\n' "${guest_ip}"
                 return 0
@@ -1128,6 +1155,9 @@ wait_for_pimox_guest_ssh() {
         if [[ -n "${last_ssh_output}" ]]; then
             echo "Last SSH failure: ${last_ssh_output}" >&2
         fi
+        if [[ -n "${last_host_probe_output}" ]]; then
+            echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2
+        fi
     else
         echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2
     fi
@@ -1185,6 +1215,9 @@ configure_pimox_worker_guest_network() {
     local encoded_script
     local network_output
     local worker_address
+    local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}"
+    local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-true}"
+    local lan_probe_block=""
 
     if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
         echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2
@@ -1210,6 +1243,29 @@ configure_pimox_worker_guest_network() {
         echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2
         return 1
     fi
+    if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
+        echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2
+        return 1
+    fi
+    if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then
+        echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2
+        return 1
+    fi
+
+    if truthy "${require_lan_probe}"; then
+        lan_probe_block="$(cat <<EOF
+echo "guest-check: ping gateway ${gateway}"
+ping -c 2 -W 2 '${gateway}'
+EOF
+)"
+        if [[ -n "${lan_probe_host}" ]]; then
+            lan_probe_block+="
+echo \"guest-check: ping Debian/probe host ${lan_probe_host}\"
+ping -c 2 -W 2 '${lan_probe_host}'"
+        fi
+    else
+        lan_probe_block="echo 'guest-check: LAN probe disabled by LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false'"
+    fi
 
     script="$(cat <<EOF
 set -eu
@@ -1238,6 +1294,12 @@ sudo systemctl restart ssh 2>/dev/null || true
 ip -br addr show ${interface_name}
 ip route
 ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
+${lan_probe_block}
+echo 'guest-check: ssh service'
+sudo systemctl is-active ssh
+if command -v ss >/dev/null 2>&1; then
+  ss -ltn | grep -Eq '(^|[[:space:]])0\.0\.0\.0:22[[:space:]]|(^|[[:space:]])\[::\]:22[[:space:]]|(^|[[:space:]])\*:22[[:space:]]'
+fi
 EOF
 )"
     encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"