diff --git a/README.md b/README.md index 2e9b504..ab86524 100644 --- a/README.md +++ b/README.md @@ -727,6 +727,15 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while allowing higher indexes to be automated. +Worker networking is deterministic by default. After cloning or reusing a VM, +the pipeline uses qemu-guest-agent to write a static `/etc/network/interfaces` +entry before waiting for SSH, so a missing DHCP lease cannot stall the build. +Defaults are `LAB_PIMOX_WORKER_NETWORK_MODE=static`, +`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, +`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and +`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Set +`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` to restore the old DHCP-only behavior. + Long-running `./jeannie up` steps keep compact output by default, but the active step line refreshes with elapsed time and the latest log line. Set `JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or diff --git a/README.md.tmpl b/README.md.tmpl index 0750f48..cc85c7b 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -727,6 +727,15 @@ Worker indexes are stable. Index `1` maps to VMID `9010`, node name so on. `LAB_PIMOX_SKIP_WORKER_INDEXES=1` leaves the first slot unmanaged while allowing higher indexes to be automated. +Worker networking is deterministic by default. After cloning or reusing a VM, +the pipeline uses qemu-guest-agent to write a static `/etc/network/interfaces` +entry before waiting for SSH, so a missing DHCP lease cannot stall the build. +Defaults are `LAB_PIMOX_WORKER_NETWORK_MODE=static`, +`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, +`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, and +`LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Set +`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` to restore the old DHCP-only behavior. + Long-running `./{{ main_script }} up` steps keep compact output by default, but the active step line refreshes with elapsed time and the latest log line. Set `JEANNIE_PROGRESS_INTERVAL_SECONDS=5` for faster heartbeats, or diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 5c920c3..c2fd546 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -142,6 +142,15 @@ destroy and recreate existing worker VMs from the current template. The pipeline refuses `LAB_PIMOX_WORKER_STORAGE=local` so only the template VM lives on local storage. +Worker networking defaults to deterministic static LAN addresses configured +through qemu-guest-agent after clone/start and before SSH wait: +`LAB_PIMOX_WORKER_NETWORK_MODE=static`, +`LAB_PIMOX_WORKER_STATIC_IPS="192.168.100.66 192.168.100.67 192.168.100.76"`, +`LAB_PIMOX_WORKER_GATEWAY=192.168.100.1`, `LAB_PIMOX_WORKER_INTERFACE=enp0s18`, +and `LAB_PIMOX_WORKER_DNS_SERVERS="192.168.100.89 1.1.1.1"`. Use +`LAB_PIMOX_WORKER_NETWORK_MODE=dhcp` only when DHCP reservations are known to be +stable. + Worker indexes are stable: index `1` maps to VMID `9010`, `pimox-worker-01`, and worker key `pimox01`; index `2` maps to VMID `9011`, `pimox-worker-02`, and worker key `pimox02`. `LAB_PIMOX_SKIP_WORKER_INDEXES` diff --git a/jeannie b/jeannie index 7e7728c..2eb00e0 100755 --- a/jeannie +++ b/jeannie @@ -1145,6 +1145,107 @@ pimox_generated_mac() { $((vmid & 255)) } +pimox_worker_static_ip() { + local index="$1" + local static_ips="$2" + local current_index=1 + local ip + + static_ips="${static_ips//,/ }" + for ip in ${static_ips}; do + if ((current_index == index)); then + printf '%s\n' "${ip}" + return 0 + fi + current_index=$((current_index + 1)) + done + + return 1 +} + +validate_ipv4_cidr_or_host() { + local value="$1" + + [[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]] +} + +configure_pimox_worker_guest_network() { + local pimox_host="$1" + local pimox_user="$2" + local pimox_key="$3" + local vmid="$4" + local qm_bin="$5" + local worker_ip="$6" + local gateway="$7" + local dns_servers="$8" + local interface_name="$9" + local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}" + local deadline + local script + local encoded_script + local network_output + + if ! validate_ipv4_cidr_or_host "${worker_ip}"; then + echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2 + return 1 + fi + if [[ "${worker_ip}" != */* ]]; then + worker_ip="${worker_ip}/24" + fi + if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2 + return 1 + fi + if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2 + return 1 + fi + if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2 + return 1 + fi + + script="$(cat </tmp/homelab-interfaces <<'INTERFACES' +auto lo +iface lo inet loopback + +auto ${interface_name} +iface ${interface_name} inet static + address ${worker_ip} + gateway ${gateway} + dns-nameservers ${dns_servers} +INTERFACES +sudo cp /tmp/homelab-interfaces /etc/network/interfaces +sudo systemctl disable --now systemd-networkd NetworkManager 2>/dev/null || true +sudo systemctl enable networking >/dev/null 2>&1 || true +sudo systemctl restart networking || true +sudo ip addr flush dev ${interface_name} || true +sudo ifup ${interface_name} || true +sudo systemctl restart ssh 2>/dev/null || true +ip -br addr show ${interface_name} +ip route +EOF +)" + encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" + + deadline=$((SECONDS + timeout_seconds)) + while ((SECONDS < deadline)); do + if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then + printf '%s\n' "${network_output}" | sed 's/^/ guest-network: /' + return 0 + fi + sleep 5 + done + + echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2 + if [[ -n "${network_output:-}" ]]; then + printf '%s\n' "${network_output}" | sed 's/^/ /' >&2 + fi + return 1 +} + cpuset_cpu_count() { local cpuset="$1" local count=0 @@ -1222,6 +1323,11 @@ ensure_pimox_worker_node() { local worker_storage="${19}" local worker_replace_existing="${20}" local worker_cpu_affinity="${21}" + local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" + local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" + local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" + local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" + local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" local padded local vmid local worker_key @@ -1229,6 +1335,7 @@ ensure_pimox_worker_node() { local node_name local mac local guest_ip + local static_ip printf -v padded '%02d' "${index}" vmid=$((worker_base_vmid + index - 1)) @@ -1320,6 +1427,35 @@ sudo '${qm_bin}' set '${vmid}' --onboot 1 sudo '${qm_bin}' start '${vmid}'" fi + case "${worker_network_mode}" in + static) + if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then + echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2 + exit 1 + fi + echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..." + if ! configure_pimox_worker_guest_network \ + "${pimox_host}" \ + "${pimox_user}" \ + "${pimox_key}" \ + "${vmid}" \ + "${qm_bin}" \ + "${static_ip}" \ + "${worker_gateway}" \ + "${worker_dns_servers}" \ + "${worker_interface}"; then + pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}" + exit 1 + fi + ;; + dhcp) + ;; + *) + echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2 + exit 1 + ;; + esac + if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2 exit 1 diff --git a/tests/jeannie-unit b/tests/jeannie-unit index 82fce38..ea4e634 100755 --- a/tests/jeannie-unit +++ b/tests/jeannie-unit @@ -320,6 +320,16 @@ PY assert_contains "heal cooldown escalates diagnostics" "next diagnostic command: ./jeannie doctor-cluster --details" "${output}" } +test_pimox_worker_static_ip_selection() { + assert_eq "pimox worker static IP index 1" "192.168.100.66" "$(pimox_worker_static_ip 1 "192.168.100.66 192.168.100.67")" + assert_eq "pimox worker static IP index 2" "192.168.100.67" "$(pimox_worker_static_ip 2 "192.168.100.66,192.168.100.67")" + if pimox_worker_static_ip 3 "192.168.100.66 192.168.100.67" >/dev/null; then + fail "pimox worker static IP missing index should fail" + else + pass "pimox worker static IP missing index should fail" + fi +} + test_pimox_worker_count_detects_generated_topology test_configured_worker_count_cannot_hide_existing_workers test_configured_worker_count_can_expand_target_set @@ -332,6 +342,7 @@ test_report_renderer_empty_problems_message test_inventory_validator_accepts_current_inventory test_inventory_validator_rejects_ip_drift test_heal_cooldown_escalates_to_diagnostics +test_pimox_worker_static_ip_selection if ((FAILURES > 0)); then printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2