Fail Pimox worker network issues faster
This commit is contained in:
juvdiaz
parent
1b97cc1404
commit
5a14e2d409
|
|
@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network
|
|||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||
and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`.
|
||||
After static network assignment, the guest must ping the gateway and
|
||||
`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set
|
||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After
|
||||
qemu-guest-agent reports an IP, Debian must be able to ping the worker before
|
||||
SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when
|
||||
worker ICMP is intentionally blocked.
|
||||
`LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
|
||||
|
||||
Long-running `./jeannie up` steps keep compact output by default, but
|
||||
|
|
|
|||
|
|
@ -739,6 +739,12 @@ Worker creation fails faster than template creation: static guest-network
|
|||
configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`,
|
||||
worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`,
|
||||
and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180`.
|
||||
After static network assignment, the guest must ping the gateway and
|
||||
`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set
|
||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After
|
||||
qemu-guest-agent reports an IP, Debian must be able to ping the worker before
|
||||
SSH retries continue; set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false` only when
|
||||
worker ICMP is intentionally blocked.
|
||||
`LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
|
||||
|
||||
Long-running `./{{ main_script }} up` steps keep compact output by default, but
|
||||
|
|
|
|||
|
|
@ -155,6 +155,12 @@ Worker clone failures are intentionally faster than template build failures:
|
|||
`LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
|
||||
`LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=180` by default. The broader
|
||||
`LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation.
|
||||
The static network step also checks guest LAN reachability by pinging the
|
||||
gateway and `${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`. The Debian
|
||||
host then pings the reported worker IP before continuing SSH retries. Disable
|
||||
those checks only for ICMP-blocked networks with
|
||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` or
|
||||
`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=false`.
|
||||
|
||||
Worker indexes are stable: index `1` maps to VMID `9010`,
|
||||
`pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`,
|
||||
|
|
|
|||
|
|
@ -539,6 +539,14 @@ falls back to the first Lynis warning, then the first suggestion.
|
|||
: Desired Pimox worker count when not supplied by generated cluster topology
|
||||
state.
|
||||
|
||||
`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE`
|
||||
: Require the worker guest to ping the LAN gateway and probe host after static
|
||||
IP assignment. Defaults to `true`.
|
||||
|
||||
`LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE`
|
||||
: Require the Debian host to ping the worker IP before waiting for SSH. Defaults
|
||||
to `true`.
|
||||
|
||||
`LAB_SKIP_PREFLIGHT=true`
|
||||
: Skip preflight checks. Use only for targeted troubleshooting.
|
||||
|
||||
|
|
|
|||
62
jeannie
62
jeannie
|
|
@ -1069,6 +1069,9 @@ wait_for_pimox_guest_ssh() {
|
|||
local last_guest_ip=""
|
||||
local last_known_hosts_ip=""
|
||||
local last_ssh_output=""
|
||||
local last_host_probe_output=""
|
||||
local host_probe_failures=0
|
||||
local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-true}"
|
||||
local next_log
|
||||
local ssh_deadline=0
|
||||
local ssh_output
|
||||
|
|
@ -1082,6 +1085,10 @@ wait_for_pimox_guest_ssh() {
|
|||
echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
|
||||
echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
|
||||
return 1
|
||||
fi
|
||||
mkdir -p "$(dirname "${known_hosts_file}")"
|
||||
touch "${known_hosts_file}"
|
||||
chmod 0600 "${known_hosts_file}"
|
||||
|
|
@ -1101,6 +1108,26 @@ wait_for_pimox_guest_ssh() {
|
|||
ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true
|
||||
last_known_hosts_ip="${guest_ip}"
|
||||
fi
|
||||
if truthy "${require_host_probe}"; then
|
||||
if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then
|
||||
host_probe_failures=0
|
||||
else
|
||||
host_probe_failures=$((host_probe_failures + 1))
|
||||
last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}"
|
||||
if ((host_probe_failures >= 3)); then
|
||||
echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2
|
||||
echo "Latest host probe: ${last_host_probe_output}" >&2
|
||||
echo "Fast checks:" >&2
|
||||
echo " ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2
|
||||
echo " arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2
|
||||
echo " Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2
|
||||
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
|
||||
return 1
|
||||
fi
|
||||
sleep 10
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then
|
||||
printf '%s\n' "${guest_ip}"
|
||||
return 0
|
||||
|
|
@ -1128,6 +1155,9 @@ wait_for_pimox_guest_ssh() {
|
|||
if [[ -n "${last_ssh_output}" ]]; then
|
||||
echo "Last SSH failure: ${last_ssh_output}" >&2
|
||||
fi
|
||||
if [[ -n "${last_host_probe_output}" ]]; then
|
||||
echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2
|
||||
fi
|
||||
else
|
||||
echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2
|
||||
fi
|
||||
|
|
@ -1185,6 +1215,9 @@ configure_pimox_worker_guest_network() {
|
|||
local encoded_script
|
||||
local network_output
|
||||
local worker_address
|
||||
local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}"
|
||||
local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-true}"
|
||||
local lan_probe_block=""
|
||||
|
||||
if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
|
||||
echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2
|
||||
|
|
@ -1210,6 +1243,29 @@ configure_pimox_worker_guest_network() {
|
|||
echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
|
||||
echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then
|
||||
echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if truthy "${require_lan_probe}"; then
|
||||
lan_probe_block="$(cat <<EOF
|
||||
echo "guest-check: ping gateway ${gateway}"
|
||||
ping -c 2 -W 2 '${gateway}'
|
||||
EOF
|
||||
)"
|
||||
if [[ -n "${lan_probe_host}" ]]; then
|
||||
lan_probe_block+="
|
||||
echo \"guest-check: ping Debian/probe host ${lan_probe_host}\"
|
||||
ping -c 2 -W 2 '${lan_probe_host}'"
|
||||
fi
|
||||
else
|
||||
lan_probe_block="echo 'guest-check: LAN probe disabled by LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false'"
|
||||
fi
|
||||
|
||||
script="$(cat <<EOF
|
||||
set -eu
|
||||
|
|
@ -1238,6 +1294,12 @@ sudo systemctl restart ssh 2>/dev/null || true
|
|||
ip -br addr show ${interface_name}
|
||||
ip route
|
||||
ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
|
||||
${lan_probe_block}
|
||||
echo 'guest-check: ssh service'
|
||||
sudo systemctl is-active ssh
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn | grep -Eq '(^|[[:space:]])0\.0\.0\.0:22[[:space:]]|(^|[[:space:]])\[::\]:22[[:space:]]|(^|[[:space:]])\*:22[[:space:]]'
|
||||
fi
|
||||
EOF
|
||||
)"
|
||||
encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
|
||||
|
|
|
|||
Loading…
Reference in New Issue