Fix Pimox template PXE boot sequencing

bootstrap/provisioning/main.tf

@@ -124,7 +124,7 @@ resource "null_resource" "pimox_template_vm_create" {
     pimox_user       = var.pimox_user
     ssh_key_path     = var.pimox_ssh_key_path
     qm_bin           = var.pimox_qm_bin
-    builder_version  = "7"
+    builder_version  = "8"
     vmid             = tostring(var.pimox_template_vmid)
     name             = var.pimox_template_name
     cores            = tostring(var.pimox_template_cores)
@@ -207,7 +207,7 @@ sudo "$qm_cmd" create "$vmid" \
 
 sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
 sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
-sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0"
+sudo "$qm_cmd" set "$vmid" --boot "order=net0;scsi0"
 sudo "$qm_cmd" set "$vmid" --agent enabled=1
 if [ -n "${self.triggers.cpu_affinity}" ]; then
   affinity_output="$(sudo "$qm_cmd" set "$vmid" --affinity "${self.triggers.cpu_affinity}" 2>&1)" || {
@@ -223,6 +223,7 @@ if [ -n "${self.triggers.cpu_affinity}" ]; then
   }
 fi
 sudo "$qm_cmd" start "$vmid"
+sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0"
 EOT
     ]
   }
@@ -241,7 +242,7 @@ resource "null_resource" "pimox_template_vm_seal" {
     guest_host      = var.pimox_template_build_host
     guest_user      = var.pimox_template_build_user
     guest_key_path  = var.pimox_template_build_ssh_key_path
-    seal_version    = "4"
+    seal_version    = "5"
     timeout         = var.pimox_template_build_timeout
     timeout_seconds = tostring(var.pimox_template_build_timeout_seconds)
     guest_ip_prefix = var.pimox_template_guest_ip_prefix
@@ -263,6 +264,13 @@ guest_key="${self.triggers.guest_key_path}"
 timeout_seconds="${self.triggers.timeout_seconds}"
 guest_ip_prefix="${self.triggers.guest_ip_prefix}"
 vmid="${self.triggers.vmid}"
+known_hosts_file="${path.module}/../../.lab/pimox-template-known_hosts"
+last_known_hosts_ip=""
+last_ssh_output=""
+
+mkdir -p "$(dirname "$known_hosts_file")"
+touch "$known_hosts_file"
+chmod 0600 "$known_hosts_file"
 
 if ! command -v python3 >/dev/null 2>&1; then
   echo "python3 is required to discover the Pimox guest IP from qemu-guest-agent" >&2
@@ -274,7 +282,17 @@ ssh_pimox() {
 }
 
 ssh_guest() {
-  ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "$guest_user@$guest_host" "$@"
+  ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$guest_user@$guest_host" "$@"
+}
+
+debug_pimox_vm() {
+  ssh_pimox "set +e
+echo 'Pimox VM $vmid status:'
+sudo '$pimox_qm_bin' status '$vmid'
+echo 'Pimox VM $vmid config summary:'
+sudo '$pimox_qm_bin' config '$vmid' | grep -E '^(agent|bios|boot|efidisk0|net0|scsi0|serial0|vga):' || true
+echo 'Pimox VM $vmid guest-agent network-get-interfaces:'
+sudo '$pimox_qm_bin' guest cmd '$vmid' network-get-interfaces" >&2 || true
 }
 
 guest_ip_from_agent() {
@@ -319,13 +337,22 @@ while (( SECONDS < deadline )); do
   if [ -z "$guest_host" ]; then
     guest_host="$(guest_ip_from_agent || true)"
   fi
-  if [ -n "$guest_host" ] && ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
+  if [ -n "$guest_host" ]; then
-    break
+    if [ "$last_known_hosts_ip" != "$guest_host" ]; then
+      ssh-keygen -R "$guest_host" -f "$known_hosts_file" >/dev/null 2>&1 || true
+      last_known_hosts_ip="$guest_host"
+    fi
+    if last_ssh_output="$(ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh" 2>&1)"; then
+      break
+    fi
   fi
   if (( SECONDS >= next_log )); then
     elapsed=$((timeout_seconds - (deadline - SECONDS)))
     if [ -n "$guest_host" ]; then
       echo "Waiting for SSH and template preparation script on VM $vmid at $guest_host ($${elapsed}s elapsed)..."
+      if [ -n "$last_ssh_output" ]; then
+        echo "Last SSH failure: $last_ssh_output"
+      fi
     else
       echo "Waiting for VM $vmid to boot the installed guest and report an IP through qemu-guest-agent ($${elapsed}s elapsed)..."
     fi
@@ -336,11 +363,16 @@ done
 
 if [ -z "$guest_host" ]; then
   echo "Timed out waiting for VM $vmid to report a guest IP through qemu-guest-agent" >&2
+  debug_pimox_vm
   exit 1
 fi
 
 if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
   echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2
+  if [ -n "$last_ssh_output" ]; then
+    echo "Last SSH failure: $last_ssh_output" >&2
+  fi
+  debug_pimox_vm
   exit 1
 fi