Restore working Pimox provisioning path
Homelab Main / deploy (push) Successful in 1m23s Details

This commit is contained in:
juvdiaz 2026-05-28 00:47:08 -06:00
parent c3f08e3fdd
commit f46f962c2b
5 changed files with 16 additions and 88 deletions

View File

@ -102,12 +102,9 @@ LAB_PIMOX_PIPELINE=true ./lab.sh up
Defaults match the observed Pimox template VM shape: OVMF firmware, virtio Defaults match the observed Pimox template VM shape: OVMF firmware, virtio
networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with
2 cores, and 4 GiB memory. The Bullseye-pinned Pimox `qm` does not support 2 cores, and 4 GiB memory. Override `TF_VAR_pimox_template_scsi0`,
`--affinity`, so CPU affinity is disabled by default. Override `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or
`TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes.
`TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange
Pi template layout changes. `TF_VAR_pimox_template_cpu_affinity` remains an
opt-in for a newer `qm`; if unsupported, automation logs the skip and continues.
`./lab.sh up` also creates or reuses worker clones after the template exists. It `./lab.sh up` also creates or reuses worker clones after the template exists. It
defaults to two workers, VMIDs `9010` and `9011`, names like defaults to two workers, VMIDs `9010` and `9011`, names like

View File

@ -124,12 +124,11 @@ resource "null_resource" "pimox_template_vm_create" {
pimox_user = var.pimox_user pimox_user = var.pimox_user
ssh_key_path = var.pimox_ssh_key_path ssh_key_path = var.pimox_ssh_key_path
qm_bin = var.pimox_qm_bin qm_bin = var.pimox_qm_bin
builder_version = "11" builder_version = "4"
vmid = tostring(var.pimox_template_vmid) vmid = tostring(var.pimox_template_vmid)
name = var.pimox_template_name name = var.pimox_template_name
cores = tostring(var.pimox_template_cores) cores = tostring(var.pimox_template_cores)
memory = tostring(var.pimox_template_memory) memory = tostring(var.pimox_template_memory)
cpu_affinity = var.pimox_template_cpu_affinity
bridge = var.pimox_template_bridge bridge = var.pimox_template_bridge
net0 = local.pimox_template_net0 net0 = local.pimox_template_net0
scsi0 = var.pimox_template_scsi0 scsi0 = var.pimox_template_scsi0
@ -173,7 +172,7 @@ if ! ip link show "${self.triggers.bridge}" >/dev/null 2>&1; then
fi fi
if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then
if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$' && [ "$replace_existing" != "true" ]; then if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then
sudo "$qm_cmd" set "$vmid" --agent enabled=1 sudo "$qm_cmd" set "$vmid" --agent enabled=1
exit 0 exit 0
fi fi
@ -196,6 +195,7 @@ fi
sudo "$qm_cmd" create "$vmid" \ sudo "$qm_cmd" create "$vmid" \
--name "${self.triggers.name}" \ --name "${self.triggers.name}" \
--bios ovmf \ --bios ovmf \
--boot "order=scsi0;net0" \
--cores "${self.triggers.cores}" \ --cores "${self.triggers.cores}" \
--memory "${self.triggers.memory}" \ --memory "${self.triggers.memory}" \
--net0 "${self.triggers.net0}" \ --net0 "${self.triggers.net0}" \
@ -203,29 +203,13 @@ sudo "$qm_cmd" create "$vmid" \
--ostype l26 \ --ostype l26 \
--scsihw virtio-scsi-pci \ --scsihw virtio-scsi-pci \
--sockets 1 \ --sockets 1 \
--vga virtio --vga virtio \
--agent enabled=1
sudo "$qm_cmd" set "$vmid" --serial0 socket
sudo "$qm_cmd" set "$vmid" --vga serial0
sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
sudo "$qm_cmd" set "$vmid" --boot "order=net0;scsi0"
sudo "$qm_cmd" set "$vmid" --agent enabled=1 sudo "$qm_cmd" set "$vmid" --agent enabled=1
if [ -n "${self.triggers.cpu_affinity}" ]; then
affinity_output="$(sudo "$qm_cmd" set "$vmid" --affinity "${self.triggers.cpu_affinity}" 2>&1)" || {
case "$affinity_output" in
*"Unknown option: affinity"*)
echo "Pimox qm does not support --affinity; skipping CPU affinity ${self.triggers.cpu_affinity} for VM $vmid."
;;
*)
printf '%s\n' "$affinity_output" >&2
exit 1
;;
esac
}
fi
sudo "$qm_cmd" start "$vmid" sudo "$qm_cmd" start "$vmid"
sudo "$qm_cmd" set "$vmid" --boot "order=scsi0"
EOT EOT
] ]
} }
@ -244,7 +228,7 @@ resource "null_resource" "pimox_template_vm_seal" {
guest_host = var.pimox_template_build_host guest_host = var.pimox_template_build_host
guest_user = var.pimox_template_build_user guest_user = var.pimox_template_build_user
guest_key_path = var.pimox_template_build_ssh_key_path guest_key_path = var.pimox_template_build_ssh_key_path
seal_version = "9" seal_version = "4"
timeout = var.pimox_template_build_timeout timeout = var.pimox_template_build_timeout
timeout_seconds = tostring(var.pimox_template_build_timeout_seconds) timeout_seconds = tostring(var.pimox_template_build_timeout_seconds)
guest_ip_prefix = var.pimox_template_guest_ip_prefix guest_ip_prefix = var.pimox_template_guest_ip_prefix
@ -266,14 +250,6 @@ guest_key="${self.triggers.guest_key_path}"
timeout_seconds="${self.triggers.timeout_seconds}" timeout_seconds="${self.triggers.timeout_seconds}"
guest_ip_prefix="${self.triggers.guest_ip_prefix}" guest_ip_prefix="${self.triggers.guest_ip_prefix}"
vmid="${self.triggers.vmid}" vmid="${self.triggers.vmid}"
known_hosts_file="${path.module}/../../.lab/pimox-template-known_hosts"
last_known_hosts_ip=""
last_ssh_output=""
next_debug=0
mkdir -p "$(dirname "$known_hosts_file")"
touch "$known_hosts_file"
chmod 0600 "$known_hosts_file"
if ! command -v python3 >/dev/null 2>&1; then if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required to discover the Pimox guest IP from qemu-guest-agent" >&2 echo "python3 is required to discover the Pimox guest IP from qemu-guest-agent" >&2
@ -285,22 +261,7 @@ ssh_pimox() {
} }
ssh_guest() { ssh_guest() {
ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$guest_user@$guest_host" "$@" ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "$guest_user@$guest_host" "$@"
}
debug_pimox_vm() {
ssh_pimox "set +e
echo 'Pimox VM $vmid status:'
sudo '$pimox_qm_bin' status '$vmid'
echo 'Pimox VM $vmid config summary:'
sudo '$pimox_qm_bin' config '$vmid' | grep -E '^(agent|bios|boot|efidisk0|net0|scsi0|serial0|vga):' || true
echo 'Pimox VM $vmid guest-agent network-get-interfaces:'
if command -v timeout >/dev/null 2>&1; then
sudo timeout 20s '$pimox_qm_bin' guest cmd '$vmid' network-get-interfaces || true
else
sudo '$pimox_qm_bin' guest cmd '$vmid' network-get-interfaces || true
fi
echo 'If this VM is still in the installer, inspect the serial console on the Pimox host with: sudo $pimox_qm_bin terminal $vmid'" || true
} }
guest_ip_from_agent() { guest_ip_from_agent() {
@ -341,51 +302,32 @@ fi
deadline=$((SECONDS + timeout_seconds)) deadline=$((SECONDS + timeout_seconds))
next_log=$SECONDS next_log=$SECONDS
next_debug=$SECONDS
while (( SECONDS < deadline )); do while (( SECONDS < deadline )); do
if [ -z "$guest_host" ]; then if [ -z "$guest_host" ]; then
guest_host="$(guest_ip_from_agent || true)" guest_host="$(guest_ip_from_agent || true)"
fi fi
if [ -n "$guest_host" ]; then if [ -n "$guest_host" ] && ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
if [ "$last_known_hosts_ip" != "$guest_host" ]; then
ssh-keygen -R "$guest_host" -f "$known_hosts_file" >/dev/null 2>&1 || true
last_known_hosts_ip="$guest_host"
fi
if last_ssh_output="$(ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh" 2>&1)"; then
break break
fi fi
fi
if (( SECONDS >= next_log )); then if (( SECONDS >= next_log )); then
elapsed=$((timeout_seconds - (deadline - SECONDS))) elapsed=$((timeout_seconds - (deadline - SECONDS)))
if [ -n "$guest_host" ]; then if [ -n "$guest_host" ]; then
echo "Waiting for SSH and template preparation script on VM $vmid at $guest_host ($${elapsed}s elapsed)..." echo "Waiting for SSH and template preparation script on VM $vmid at $guest_host ($${elapsed}s elapsed)..."
if [ -n "$last_ssh_output" ]; then
echo "Last SSH failure: $last_ssh_output"
fi
else else
echo "Waiting for VM $vmid to boot the installed guest and report an IP through qemu-guest-agent ($${elapsed}s elapsed)..." echo "Waiting for VM $vmid to boot the installed guest and report an IP through qemu-guest-agent ($${elapsed}s elapsed)..."
fi fi
next_log=$((SECONDS + 60)) next_log=$((SECONDS + 60))
fi fi
if (( SECONDS >= next_debug )); then
debug_pimox_vm
next_debug=$((SECONDS + 60))
fi
sleep 15 sleep 15
done done
if [ -z "$guest_host" ]; then if [ -z "$guest_host" ]; then
echo "Timed out waiting for VM $vmid to report a guest IP through qemu-guest-agent" >&2 echo "Timed out waiting for VM $vmid to report a guest IP through qemu-guest-agent" >&2
debug_pimox_vm
exit 1 exit 1
fi fi
if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2 echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2
if [ -n "$last_ssh_output" ]; then
echo "Last SSH failure: $last_ssh_output" >&2
fi
debug_pimox_vm
exit 1 exit 1
fi fi
@ -405,7 +347,7 @@ resource "null_resource" "pimox_template_vm_finalize" {
pimox_user = var.pimox_user pimox_user = var.pimox_user
ssh_key_path = var.pimox_ssh_key_path ssh_key_path = var.pimox_ssh_key_path
qm_bin = var.pimox_qm_bin qm_bin = var.pimox_qm_bin
finalizer_version = "3" finalizer_version = "2"
vmid = tostring(var.pimox_template_vmid) vmid = tostring(var.pimox_template_vmid)
} }
@ -452,7 +394,7 @@ if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then
exit 1 exit 1
fi fi
sudo "$qm_cmd" set "$vmid" --boot "order=scsi0" sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0"
sudo "$qm_cmd" template "$vmid" sudo "$qm_cmd" template "$vmid"
EOT EOT
] ]

View File

@ -1,10 +1,7 @@
set default=0 set default=0
set timeout=5 set timeout=5
serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
terminal_input serial console
terminal_output serial console
menuentry 'Debian 13 arm64 homelab worker template' { menuentry 'Debian 13 arm64 homelab worker template' {
linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} DEBIAN_FRONTEND=text fb=false console=tty0 console=ttyS0,115200n8 --- console=tty0 console=ttyS0,115200n8 linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} --- quiet
initrd /debian-installer/arm64/initrd.gz initrd /debian-installer/arm64/initrd.gz
} }

View File

@ -18,8 +18,6 @@ d-i clock-setup/utc boolean true
d-i time/zone string ${timezone} d-i time/zone string ${timezone}
d-i partman-auto/disk string ${template_disk} d-i partman-auto/disk string ${template_disk}
d-i partman-auto/method string regular d-i partman-auto/method string regular
d-i partman-partitioning/choose_label string gpt
d-i partman-partitioning/default_label string gpt
d-i partman-auto/choose_recipe select atomic d-i partman-auto/choose_recipe select atomic
d-i partman-partitioning/confirm_write_new_label boolean true d-i partman-partitioning/confirm_write_new_label boolean true
d-i partman/choose_partition select finish d-i partman/choose_partition select finish
@ -32,7 +30,6 @@ d-i pkgsel/include string ${template_package_list}
d-i pkgsel/update-policy select none d-i pkgsel/update-policy select none
popularity-contest popularity-contest/participate boolean false popularity-contest popularity-contest/participate boolean false
d-i grub-installer/only_debian boolean true d-i grub-installer/only_debian boolean true
d-i grub-installer/force-efi-extra-removable boolean true
d-i grub-installer/bootdev string ${template_disk} d-i grub-installer/bootdev string ${template_disk}
d-i preseed/late_command string wget -O /target/usr/local/sbin/homelab-golden-node-prepare.sh ${provisioning_script_url}; chmod 0755 /target/usr/local/sbin/homelab-golden-node-prepare.sh; in-target /usr/local/sbin/homelab-golden-node-prepare.sh; wget -O /target/usr/local/sbin/homelab-prepare-template.sh ${prepare_template_script_url}; chmod 0755 /target/usr/local/sbin/homelab-prepare-template.sh d-i preseed/late_command string wget -O /target/usr/local/sbin/homelab-golden-node-prepare.sh ${provisioning_script_url}; chmod 0755 /target/usr/local/sbin/homelab-golden-node-prepare.sh; in-target /usr/local/sbin/homelab-golden-node-prepare.sh; wget -O /target/usr/local/sbin/homelab-prepare-template.sh ${prepare_template_script_url}; chmod 0755 /target/usr/local/sbin/homelab-prepare-template.sh
d-i finish-install/reboot_in_progress note d-i finish-install/reboot_in_progress note

View File

@ -199,11 +199,6 @@ variable "pimox_template_memory" {
default = 4096 default = 4096
} }
variable "pimox_template_cpu_affinity" {
type = string
default = ""
}
variable "pimox_template_bridge" { variable "pimox_template_bridge" {
type = string type = string
default = "vmbr0" default = "vmbr0"
@ -221,7 +216,7 @@ variable "pimox_template_scsi0" {
variable "pimox_template_efidisk0" { variable "pimox_template_efidisk0" {
type = string type = string
default = "local:1,efitype=4m,pre-enrolled-keys=0" default = "local:1,efitype=4m,pre-enrolled-keys=1"
} }
variable "pimox_template_replace_existing" { variable "pimox_template_replace_existing" {