From 5a14e2d409542980eee8d98399da0727809d5f91 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Wed, 1 Jul 2026 10:49:48 -0600 Subject: [PATCH] Fail Pimox worker network issues faster --- README.md | 6 ++++ README.md.tmpl | 6 ++++ bootstrap/provisioning/README.md | 6 ++++ docs/jeannie.1.md | 8 +++++ jeannie | 62 ++++++++++++++++++++++++++++++++ 5 files changed, 88 insertions(+) diff --git a/README.md b/README.md index 3276908..e3f27f2 100644 --- a/README.md +++ b/README.md @@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`. +After static network assignment, the guest must ping the gateway and +`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After +qemu-guest-agent reports an IP, Debian must be able to ping the worker before +SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when +worker ICMP is intentionally blocked. `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout. Long-running `./jeannie up` steps keep compact output by default, but diff --git a/README.md.tmpl b/README.md.tmpl index a2bf82d..fa3a538 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`. +After static network assignment, the guest must ping the gateway and +`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After +qemu-guest-agent reports an IP, Debian must be able to ping the worker before +SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when +worker ICMP is intentionally blocked. `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout. Long-running `./{{ main_script }} up` steps keep compact output by default, but diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index ea0a1f7..a7a93fd 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -155,6 +155,12 @@ Worker clone failures are intentionally faster than template build failures: `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180` by default. The broader `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation. +The static network step also checks guest LAN reachability by pinging the +gateway and `${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`. The Debian +host then pings the reported worker IP before continuing SSH retries. Disable +those checks only for ICMP-blocked networks with +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` or +`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false`. Worker indexes are stable: index `1` maps to VMID `9010`, `pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`, diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index a6dcd3f..feddec8 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -539,6 +539,14 @@ falls back to the first Lynis warning, then the first suggestion. : Desired Pimox worker count when not supplied by generated cluster topology state. +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE` +: Require the worker guest to ping the LAN gateway and probe host after static +IP assignment. Defaults to `true`. + +`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE` +: Require the Debian host to ping the worker IP before waiting for SSH. Defaults +to `true`. + `LAB_SKIP_PREFLIGHT=true` : Skip preflight checks. Use only for targeted troubleshooting. diff --git a/jeannie b/jeannie index d1d1204..08e94ae 100755 --- a/jeannie +++ b/jeannie @@ -1069,6 +1069,9 @@ wait_for_pimox_guest_ssh() { local last_guest_ip="" local last_known_hosts_ip="" local last_ssh_output="" + local last_host_probe_output="" + local host_probe_failures=0 + local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-true}" local next_log local ssh_deadline=0 local ssh_output @@ -1082,6 +1085,10 @@ wait_for_pimox_guest_ssh() { echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2 return 1 fi + if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then + echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2 + return 1 + fi mkdir -p "$(dirname "${known_hosts_file}")" touch "${known_hosts_file}" chmod 0600 "${known_hosts_file}" @@ -1101,6 +1108,26 @@ wait_for_pimox_guest_ssh() { ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true last_known_hosts_ip="${guest_ip}" fi + if truthy "${require_host_probe}"; then + if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then + host_probe_failures=0 + else + host_probe_failures=$((host_probe_failures + 1)) + last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}" + if ((host_probe_failures >= 3)); then + echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2 + echo "Latest host probe: ${last_host_probe_output}" >&2 + echo "Fast checks:" >&2 + echo " ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2 + echo " arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2 + echo " Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2 + pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" + return 1 + fi + sleep 10 + continue + fi + fi if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then printf '%s\n' "${guest_ip}" return 0 @@ -1128,6 +1155,9 @@ wait_for_pimox_guest_ssh() { if [[ -n "${last_ssh_output}" ]]; then echo "Last SSH failure: ${last_ssh_output}" >&2 fi + if [[ -n "${last_host_probe_output}" ]]; then + echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2 + fi else echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2 fi @@ -1185,6 +1215,9 @@ configure_pimox_worker_guest_network() { local encoded_script local network_output local worker_address + local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}" + local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-true}" + local lan_probe_block="" if ! validate_ipv4_cidr_or_host "${worker_ip}"; then echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2 @@ -1210,6 +1243,29 @@ configure_pimox_worker_guest_network() { echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2 return 1 fi + if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2 + return 1 + fi + if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then + echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2 + return 1 + fi + + if truthy "${require_lan_probe}"; then + lan_probe_block="$(cat <