Configure Pimox worker static networking

README.md

@@ -727,6 +727,15 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name
 so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while
 allowing higher indexes to be automated.
 
+Worker networking is deterministic by default. After cloning or reusing a VM,
+the pipeline uses qemu-guest-agent to write a static `/etc/network/interfaces`
+entry before waiting for SSH, so a missing DHCP lease cannot stall the build.
+Defaults are `LAB_PIMOX_WORKER_NETWORK_MODE=static`,
+`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
+`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
+`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Set
+`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` to restore the old DHCP-only behavior.
+
 Long-running `./jeannie up` steps keep compact output by default, but
 the active step line refreshes with elapsed time and the latest log line. Set
 `JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or

README.md.tmpl

@@ -727,6 +727,15 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name
 so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while
 allowing higher indexes to be automated.
 
+Worker networking is deterministic by default. After cloning or reusing a VM,
+the pipeline uses qemu-guest-agent to write a static `/etc/network/interfaces`
+entry before waiting for SSH, so a missing DHCP lease cannot stall the build.
+Defaults are `LAB_PIMOX_WORKER_NETWORK_MODE=static`,
+`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
+`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and
+`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Set
+`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` to restore the old DHCP-only behavior.
+
 Long-running `./{{ main_script }} up` steps keep compact output by default, but
 the active step line refreshes with elapsed time and the latest log line. Set
 `JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or

bootstrap/provisioning/README.md

@@ -142,6 +142,15 @@ destroy and recreate existing worker VMs from the current template. The pipeline
 refuses `LAB_PIMOX_WORKER_STORAGE=local` so only the template VM lives on local
 storage.
 
+Worker networking defaults to deterministic static LAN addresses configured
+through qemu-guest-agent after clone/start and before SSH wait:
+`LAB_PIMOX_WORKER_NETWORK_MODE=static`,
+`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`,
+`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`,
+and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Use
+`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` only when DHCP reservations are known to be
+stable.
+
 Worker indexes are stable: index `1` maps to VMID `9010`,
 `pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`,
 `pimox-worker-02`, and worker key `pimox02`. `LAB_PIMOX_SKIP_WORKER_INDEXES`

jeannie

@@ -1145,6 +1145,107 @@ pimox_generated_mac() {
         $((vmid & 255))
 }
 
+pimox_worker_static_ip() {
+    local index="$1"
+    local static_ips="$2"
+    local current_index=1
+    local ip
+
+    static_ips="${static_ips//,/ }"
+    for ip in ${static_ips}; do
+        if ((current_index == index)); then
+            printf '%s\n' "${ip}"
+            return 0
+        fi
+        current_index=$((current_index + 1))
+    done
+
+    return 1
+}
+
+validate_ipv4_cidr_or_host() {
+    local value="$1"
+
+    [[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]]
+}
+
+configure_pimox_worker_guest_network() {
+    local pimox_host="$1"
+    local pimox_user="$2"
+    local pimox_key="$3"
+    local vmid="$4"
+    local qm_bin="$5"
+    local worker_ip="$6"
+    local gateway="$7"
+    local dns_servers="$8"
+    local interface_name="$9"
+    local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}"
+    local deadline
+    local script
+    local encoded_script
+    local network_output
+
+    if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
+        echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2
+        return 1
+    fi
+    if [[ "${worker_ip}" != */* ]]; then
+        worker_ip="${worker_ip}/24"
+    fi
+    if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
+        echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2
+        return 1
+    fi
+    if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then
+        echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2
+        return 1
+    fi
+    if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
+        echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2
+        return 1
+    fi
+
+    script="$(cat <<EOF
+set -eu
+cat >/tmp/homelab-interfaces <<'INTERFACES'
+auto lo
+iface lo inet loopback
+
+auto ${interface_name}
+iface ${interface_name} inet static
+  address ${worker_ip}
+  gateway ${gateway}
+  dns-nameservers ${dns_servers}
+INTERFACES
+sudo cp /tmp/homelab-interfaces /etc/network/interfaces
+sudo systemctl disable --now systemd-networkd NetworkManager 2>/dev/null || true
+sudo systemctl enable networking >/dev/null 2>&1 || true
+sudo systemctl restart networking || true
+sudo ip addr flush dev ${interface_name} || true
+sudo ifup ${interface_name} || true
+sudo systemctl restart ssh 2>/dev/null || true
+ip -br addr show ${interface_name}
+ip route
+EOF
+)"
+    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
+
+    deadline=$((SECONDS + timeout_seconds))
+    while ((SECONDS < deadline)); do
+        if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
+            printf '%s\n' "${network_output}" | sed 's/^/    guest-network: /'
+            return 0
+        fi
+        sleep 5
+    done
+
+    echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2
+    if [[ -n "${network_output:-}" ]]; then
+        printf '%s\n' "${network_output}" | sed 's/^/    /' >&2
+    fi
+    return 1
+}
+
 cpuset_cpu_count() {
     local cpuset="$1"
     local count=0
@@ -1222,6 +1323,11 @@ ensure_pimox_worker_node() {
     local worker_storage="${19}"
     local worker_replace_existing="${20}"
     local worker_cpu_affinity="${21}"
+    local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
+    local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
+    local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}"
+    local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}"
+    local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}"
     local padded
     local vmid
     local worker_key
@@ -1229,6 +1335,7 @@ ensure_pimox_worker_node() {
     local node_name
     local mac
     local guest_ip
+    local static_ip
 
     printf -v padded '%02d' "${index}"
     vmid=$((worker_base_vmid + index - 1))
@@ -1320,6 +1427,35 @@ sudo '${qm_bin}' set '${vmid}' --onboot 1
 sudo '${qm_bin}' start '${vmid}'"
     fi
 
+    case "${worker_network_mode}" in
+        static)
+            if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then
+                echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2
+                exit 1
+            fi
+            echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..."
+            if ! configure_pimox_worker_guest_network \
+                "${pimox_host}" \
+                "${pimox_user}" \
+                "${pimox_key}" \
+                "${vmid}" \
+                "${qm_bin}" \
+                "${static_ip}" \
+                "${worker_gateway}" \
+                "${worker_dns_servers}" \
+                "${worker_interface}"; then
+                pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}"
+                exit 1
+            fi
+            ;;
+        dhcp)
+            ;;
+        *)
+            echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2
+            exit 1
+            ;;
+    esac
+
     if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
         echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2
         exit 1

tests/jeannie-unit

@@ -320,6 +320,16 @@ PY
     assert_contains "heal cooldown escalates diagnostics" "next diagnostic command: ./jeannie doctor-cluster --details" "${output}"
 }
 
+test_pimox_worker_static_ip_selection() {
+    assert_eq "pimox worker static IP index 1" "192.168.100.66" "$(pimox_worker_static_ip 1 "192.168.100.66 192.168.100.67")"
+    assert_eq "pimox worker static IP index 2" "192.168.100.67" "$(pimox_worker_static_ip 2 "192.168.100.66,192.168.100.67")"
+    if pimox_worker_static_ip 3 "192.168.100.66 192.168.100.67" >/dev/null; then
+        fail "pimox worker static IP missing index should fail"
+    else
+        pass "pimox worker static IP missing index should fail"
+    fi
+}
+
 test_pimox_worker_count_detects_generated_topology
 test_configured_worker_count_cannot_hide_existing_workers
 test_configured_worker_count_can_expand_target_set
@@ -332,6 +342,7 @@ test_report_renderer_empty_problems_message
 test_inventory_validator_accepts_current_inventory
 test_inventory_validator_rejects_ip_drift
 test_heal_cooldown_escalates_to_diagnostics
+test_pimox_worker_static_ip_selection
 
 if ((FAILURES > 0)); then
     printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2