From 86ac132629f78b6d9cb689b1cec560485eff9013 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Wed, 1 Jul 2026 18:41:29 -0600 Subject: [PATCH] Fix Pimox worker static networking --- README.md | 21 +++++++---- README.md.tmpl | 21 +++++++---- bootstrap/provisioning/README.md | 23 ++++++++---- docs/jeannie.1.md | 16 +++++++-- jeannie | 60 ++++++++++++++++++++++++++++++-- 5 files changed, 115 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 517242f..a6238eb 100644 --- a/README.md +++ b/README.md @@ -727,13 +727,19 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while allowing higher indexes to be automated. -Worker networking uses DHCP by default to preserve the known-good Pimox bridge -path. Set `LAB_PIMOX_WORKER_NETWORK_MODE=static` only when DHCP reservations are -not available and you explicitly want Jeannie to configure guest static IPs with -qemu-guest-agent. Static mode uses +Worker networking defaults to static because the ISP modem DHCP server does not +answer Pimox worker VM DHCP requests behind `vmbr0`. Set +`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` only when LAN DHCP is known to answer VM +traffic. Static mode uses `LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, `LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. +For static workers, Jeannie also adds targeted Pimox-host forwarding exceptions +for each worker IP in `DOCKER-USER` when that chain exists, or `FORWARD` +otherwise. Disable this with `LAB_PIMOX_WORKER_CONFIGURE_FORWARDING=false` only +when the Pimox host firewall already permits bridged worker IP traffic. Before +adding rules, Jeannie saves `iptables-save` output on the Pimox host under +`${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}`. Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when you explicitly want Pimox to generate the MAC. @@ -741,9 +747,10 @@ Worker creation fails faster than template creation: static guest-network configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`. -After static network assignment, the guest must ping the gateway and -`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set -`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After +Static guest-network validation skips the extra LAN ICMP probe by default +because the ISP modem may not answer worker VM pings. Set +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=true` only when the gateway/probe host +should be required during qemu-guest-agent network setup. After qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by default and lets SSH be the reachability check. Set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate. diff --git a/README.md.tmpl b/README.md.tmpl index ecb84fb..4e6095a 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -727,13 +727,19 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while allowing higher indexes to be automated. -Worker networking uses DHCP by default to preserve the known-good Pimox bridge -path. Set `LAB_PIMOX_WORKER_NETWORK_MODE=static` only when DHCP reservations are -not available and you explicitly want Jeannie to configure guest static IPs with -qemu-guest-agent. Static mode uses +Worker networking defaults to static because the ISP modem DHCP server does not +answer Pimox worker VM DHCP requests behind `vmbr0`. Set +`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` only when LAN DHCP is known to answer VM +traffic. Static mode uses `LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, `LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. +For static workers, Jeannie also adds targeted Pimox-host forwarding exceptions +for each worker IP in `DOCKER-USER` when that chain exists, or `FORWARD` +otherwise. Disable this with `LAB_PIMOX_WORKER_CONFIGURE_FORWARDING=false` only +when the Pimox host firewall already permits bridged worker IP traffic. Before +adding rules, Jeannie saves `iptables-save` output on the Pimox host under +`${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}`. Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when you explicitly want Pimox to generate the MAC. @@ -741,9 +747,10 @@ Worker creation fails faster than template creation: static guest-network configuration waits up to `LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=120`, worker IP/SSH readiness waits up to `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and SSH after an IP appears waits up to `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600`. -After static network assignment, the guest must ping the gateway and -`${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`; set -`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false` only when ICMP is blocked. After +Static guest-network validation skips the extra LAN ICMP probe by default +because the ISP modem may not answer worker VM pings. Set +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=true` only when the gateway/probe host +should be required during qemu-guest-agent network setup. After qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by default and lets SSH be the reachability check. Set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate. diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 802f038..7ec8d26 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -142,12 +142,20 @@ destroy and recreate existing worker VMs from the current template. The pipeline refuses `LAB_PIMOX_WORKER_STORAGE=local` so only the template VM lives on local storage. -Worker networking defaults to DHCP to preserve the known-good Pimox bridge path: -`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp`. Static mode is still available when DHCP -reservations are not available: set `LAB_PIMOX_WORKER_NETWORK_MODE=static`, +Worker networking defaults to static because the ISP modem DHCP server does not +answer Pimox worker VM DHCP requests behind `vmbr0`: +`LAB_PIMOX_WORKER_NETWORK_MODE=static`. DHCP mode remains available when LAN +DHCP is known to answer VM traffic: set `LAB_PIMOX_WORKER_NETWORK_MODE=dhcp`. +Static mode uses `LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, `LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`, and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. +For static workers, Jeannie also adds targeted Pimox-host forwarding exceptions +for each worker IP in `DOCKER-USER` when that chain exists, or `FORWARD` +otherwise. Disable this with `LAB_PIMOX_WORKER_CONFIGURE_FORWARDING=false` only +when the Pimox host firewall already permits bridged worker IP traffic. Before +adding rules, Jeannie saves `iptables-save` output on the Pimox host under +`${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}`. Worker NIC MAC addresses default to the VMID-derived deterministic format used by the original working Pimox worker path. Set `LAB_PIMOX_WORKER_MAC_MODE=auto` only when Pimox-generated MAC addresses are preferred. @@ -156,11 +164,12 @@ Worker clone failures are intentionally faster than template build failures: `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600` by default. The broader `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation. -The static network step also checks guest LAN reachability by pinging the -gateway and `${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-LAB_DEBIAN_LAN_IP}`. After +Static guest-network validation skips the extra LAN ICMP probe by default +because the ISP modem may not answer worker VM pings. Set +`LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=true` only when the gateway/probe host +should be required during qemu-guest-agent network setup. After qemu-guest-agent reports an IP, Debian skips the extra host-side ICMP probe by -default and lets SSH be the reachability check. Disable the static LAN probe -with `LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false`; enable the host-side ICMP gate +default and lets SSH be the reachability check. Enable the host-side ICMP gate with `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true`. Worker indexes are stable: index `1` maps to VMID `9010`, diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 5a73b60..7760eec 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -544,8 +544,9 @@ once. Defaults to `4`; set to `1` for sequential troubleshooting. state. `LAB_PIMOX_WORKER_NETWORK_MODE` -: Worker guest networking mode. Defaults to `dhcp`. Set to `static` only when -Jeannie should configure guest static IPs through qemu-guest-agent. +: Worker guest networking mode. Defaults to `static` for the current lab, +because the ISP modem DHCP server does not answer Pimox worker VM DHCP requests +behind `vmbr0`. Set to `dhcp` only when LAN DHCP is known to answer VM traffic. `LAB_PIMOX_WORKER_MAC_MODE` : Worker NIC MAC mode. Defaults to `deterministic`, deriving MAC addresses from @@ -554,7 +555,16 @@ Pimox generate the MAC. `LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE` : Require the worker guest to ping the LAN gateway and probe host after static -IP assignment. Defaults to `true`. +IP assignment. Defaults to `false`; SSH is the default reachability check. + +`LAB_PIMOX_WORKER_CONFIGURE_FORWARDING` +: Add targeted Pimox-host forwarding exceptions for static worker IPs in +`DOCKER-USER` when that chain exists, or `FORWARD` otherwise. Defaults to +`true`. + +`LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR` +: Directory on the Pimox host where Jeannie stores `iptables-save` snapshots +before adding worker forwarding exceptions. Defaults to `~/iptables-backups`. `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE` : Require the Debian host to ping the worker IP before waiting for SSH. Defaults diff --git a/jeannie b/jeannie index e98830f..889da88 100755 --- a/jeannie +++ b/jeannie @@ -1272,7 +1272,7 @@ configure_pimox_worker_guest_network() { local network_exitcode local worker_address local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}" - local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-true}" + local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-false}" local lan_probe_block="" if ! validate_ipv4_cidr_or_host "${worker_ip}"; then @@ -1383,6 +1383,57 @@ EOF return 1 } +configure_pimox_worker_forwarding() { + local pimox_host="$1" + local pimox_user="$2" + local pimox_key="$3" + local worker_ip="$4" + local enabled="${LAB_PIMOX_WORKER_CONFIGURE_FORWARDING:-true}" + local backup_dir="${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}" + local worker_address + + if disabled_value "${enabled}"; then + return 0 + fi + if ! truthy "${enabled}"; then + echo "LAB_PIMOX_WORKER_CONFIGURE_FORWARDING must be true or false." >&2 + return 1 + fi + if ! validate_ipv4_cidr_or_host "${worker_ip}"; then + echo "Invalid Pimox worker forwarding IP '${worker_ip}'." >&2 + return 1 + fi + worker_address="${worker_ip%%/*}" + + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! command -v iptables >/dev/null 2>&1; then + echo 'iptables not found on Pimox host; skipping worker forwarding rules.' + exit 0 +fi +backup_dir='${backup_dir}' +case \"\$backup_dir\" in + '~'|'~/') backup_dir=\"\$HOME\" ;; + '~/'*) backup_dir=\"\$HOME/\${backup_dir#~/}\" ;; +esac +mkdir -p \"\$backup_dir\" +backup_file=\"\$backup_dir/iptables-before-jeannie-workers-\$(date +%Y%m%d-%H%M%S).rules\" +if command -v iptables-save >/dev/null 2>&1; then + sudo iptables-save >\"\$backup_file\" + echo \"Saved Pimox iptables backup: \$backup_file\" +fi +chain=FORWARD +if sudo iptables -S DOCKER-USER >/dev/null 2>&1; then + chain=DOCKER-USER +fi +if ! sudo iptables -C \"\$chain\" -s '${worker_address}/32' -j ACCEPT 2>/dev/null; then + sudo iptables -I \"\$chain\" 1 -s '${worker_address}/32' -j ACCEPT +fi +if ! sudo iptables -C \"\$chain\" -d '${worker_address}/32' -j ACCEPT 2>/dev/null; then + sudo iptables -I \"\$chain\" 1 -d '${worker_address}/32' -j ACCEPT +fi +echo \"Pimox worker forwarding rules active in \$chain for ${worker_address}\"" +} + cpuset_cpu_count() { local cpuset="$1" local count=0 @@ -1460,7 +1511,7 @@ ensure_pimox_worker_node() { local worker_storage="${19}" local worker_replace_existing="${20}" local worker_cpu_affinity="${21}" - local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-dhcp}" + local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" @@ -1573,6 +1624,11 @@ sudo '${qm_bin}' start '${vmid}'" exit 1 fi echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..." + configure_pimox_worker_forwarding \ + "${pimox_host}" \ + "${pimox_user}" \ + "${pimox_key}" \ + "${static_ip}" if ! configure_pimox_worker_guest_network \ "${pimox_host}" \ "${pimox_user}" \