terraform { required_version = ">= 1.0" required_providers { null = { source = "hashicorp/null" version = "~> 3.2" } } } locals { tftp_root = "${var.provisioning_install_dir}/tftp" http_root = "${var.provisioning_install_dir}/html" debian_netboot_base_url = var.debian_netboot_base_url != "" ? var.debian_netboot_base_url : "http://${var.debian_mirror_host}${var.debian_mirror_directory}/dists/${var.debian_suite}/main/installer-arm64/current/images/netboot/debian-installer/arm64" preseed_url = "http://${var.http_host}:${var.http_port}/preseed/debian13-arm64-worker.cfg" template_packages = distinct(concat([ "sudo", "openssh-server", "curl", "ca-certificates", "gnupg", "apt-transport-https", "qemu-guest-agent", "cloud-init", "containerd", "open-iscsi", "nfs-common", "iptables", "iproute2", "conntrack", "socat", "ebtables", "ethtool", "ipset", "ipvsadm", "jq", "vim-tiny", "chrony", "lvm2", "xfsprogs", ], var.additional_template_packages)) template_user_ssh_keys = distinct(compact(concat(var.template_user_ssh_authorized_keys, [try(trimspace(file(var.template_user_ssh_public_key_path)), "")]))) ssh_authorized_keys_base64 = base64encode(join("\n", local.template_user_ssh_keys)) node_dns_servers = join(" ", var.node_dns_servers) kernel_cgroup_boot_options = join(" ", var.kernel_cgroup_boot_options) pimox_template_net0 = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}" template_package_list = join(" ", local.template_packages) provisioning_http_base_url = "http://${var.http_host}:${var.http_port}" provisioning_script_url = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh" prepare_template_script_url = "${local.provisioning_http_base_url}/scripts/prepare-template.sh" dnsmasq_conf = templatefile("${path.module}/templates/dnsmasq.conf.tftpl", { provisioning_interface = var.provisioning_interface proxy_dhcp_range = var.proxy_dhcp_range pxe_boot_file = var.pxe_boot_file tftp_root = local.tftp_root }) nginx_conf = templatefile("${path.module}/templates/nginx.conf.tftpl", { http_port = tostring(var.http_port) http_root = local.http_root }) grub_cfg = templatefile("${path.module}/templates/grub.cfg.tftpl", { preseed_url = local.preseed_url template_hostname = var.template_hostname template_domain = var.template_domain }) preseed_cfg = templatefile("${path.module}/templates/preseed.cfg.tftpl", { locale = var.locale keyboard = var.keyboard timezone = var.timezone template_hostname = var.template_hostname template_domain = var.template_domain template_disk = var.template_disk template_user = var.template_user template_user_full_name = var.template_user_full_name template_user_password_hash = var.template_user_password_hash debian_mirror_host = var.debian_mirror_host debian_mirror_directory = var.debian_mirror_directory template_package_list = local.template_package_list provisioning_script_url = local.provisioning_script_url prepare_template_script_url = local.prepare_template_script_url }) golden_node_prepare = templatefile("${path.module}/templates/golden-node-prepare.sh.tftpl", { template_user = var.template_user ssh_authorized_keys_base64 = local.ssh_authorized_keys_base64 kubernetes_minor_version = var.kubernetes_minor_version kernel_cgroup_boot_options = local.kernel_cgroup_boot_options registry_endpoint = var.registry_endpoint node_dns_servers = local.node_dns_servers }) prepare_template = templatefile("${path.module}/templates/prepare-template.sh.tftpl", { template_hostname = var.template_hostname clone_hostname_prefix = var.clone_hostname_prefix template_domain = var.template_domain kernel_cgroup_boot_options = local.kernel_cgroup_boot_options }) config_hash = sha256(join("\n---\n", [ local.dnsmasq_conf, local.nginx_conf, local.grub_cfg, local.preseed_cfg, local.golden_node_prepare, local.prepare_template, local.debian_netboot_base_url, ])) } resource "null_resource" "pimox_template_vm_create" { count = var.pimox_template_builder_enabled ? 1 : 0 depends_on = [null_resource.provisioning_host] triggers = { pimox_host = var.pimox_host pimox_user = var.pimox_user ssh_key_path = var.pimox_ssh_key_path qm_bin = var.pimox_qm_bin builder_version = "8" vmid = tostring(var.pimox_template_vmid) name = var.pimox_template_name cores = tostring(var.pimox_template_cores) memory = tostring(var.pimox_template_memory) cpu_affinity = var.pimox_template_cpu_affinity bridge = var.pimox_template_bridge net0 = local.pimox_template_net0 scsi0 = var.pimox_template_scsi0 efidisk0 = var.pimox_template_efidisk0 replace_existing = tostring(var.pimox_template_replace_existing) } connection { type = "ssh" user = self.triggers.pimox_user private_key = file(self.triggers.ssh_key_path) host = self.triggers.pimox_host } provisioner "remote-exec" { inline = [ <&2 exit 1 fi if ! sudo -n true >/dev/null 2>&1; then echo "passwordless sudo is required for Pimox automation" >&2 exit 1 fi if ! ip link show "${self.triggers.bridge}" >/dev/null 2>&1; then echo "Pimox bridge ${self.triggers.bridge} does not exist. Refusing to change Orange Pi networking." >&2 exit 1 fi if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$' && [ "$replace_existing" != "true" ]; then sudo "$qm_cmd" set "$vmid" --agent enabled=1 exit 0 fi if [ "$replace_existing" != "true" ]; then echo "VM $vmid already exists and is not a template. Set pimox_template_replace_existing=true to rebuild it." >&2 exit 1 fi sudo "$qm_cmd" stop "$vmid" >/dev/null 2>&1 || true elapsed=0 while [ "$elapsed" -lt 300 ]; do if sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then break fi sleep 5 elapsed=$((elapsed + 5)) done sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid" fi sudo "$qm_cmd" create "$vmid" \ --name "${self.triggers.name}" \ --bios ovmf \ --cores "${self.triggers.cores}" \ --memory "${self.triggers.memory}" \ --net0 "${self.triggers.net0}" \ --numa 0 \ --ostype l26 \ --scsihw virtio-scsi-pci \ --sockets 1 \ --vga virtio sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --boot "order=net0;scsi0" sudo "$qm_cmd" set "$vmid" --agent enabled=1 if [ -n "${self.triggers.cpu_affinity}" ]; then affinity_output="$(sudo "$qm_cmd" set "$vmid" --affinity "${self.triggers.cpu_affinity}" 2>&1)" || { case "$affinity_output" in *"Unknown option: affinity"*) echo "Pimox qm does not support --affinity; skipping CPU affinity ${self.triggers.cpu_affinity} for VM $vmid." ;; *) printf '%s\n' "$affinity_output" >&2 exit 1 ;; esac } fi sudo "$qm_cmd" start "$vmid" sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0" EOT ] } } resource "null_resource" "pimox_template_vm_seal" { count = var.pimox_template_builder_enabled ? 1 : 0 depends_on = [null_resource.pimox_template_vm_create] triggers = { pimox_host = var.pimox_host pimox_user = var.pimox_user pimox_key_path = var.pimox_ssh_key_path pimox_qm_bin = var.pimox_qm_bin guest_host = var.pimox_template_build_host guest_user = var.pimox_template_build_user guest_key_path = var.pimox_template_build_ssh_key_path seal_version = "5" timeout = var.pimox_template_build_timeout timeout_seconds = tostring(var.pimox_template_build_timeout_seconds) guest_ip_prefix = var.pimox_template_guest_ip_prefix vmid = tostring(var.pimox_template_vmid) } provisioner "local-exec" { interpreter = ["/bin/bash", "-lc"] command = </dev/null 2>&1; then echo "python3 is required to discover the Pimox guest IP from qemu-guest-agent" >&2 exit 1 fi ssh_pimox() { ssh -i "$pimox_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$pimox_user@$pimox_host" "$@" } ssh_guest() { ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$guest_user@$guest_host" "$@" } debug_pimox_vm() { ssh_pimox "set +e echo 'Pimox VM $vmid status:' sudo '$pimox_qm_bin' status '$vmid' echo 'Pimox VM $vmid config summary:' sudo '$pimox_qm_bin' config '$vmid' | grep -E '^(agent|bios|boot|efidisk0|net0|scsi0|serial0|vga):' || true echo 'Pimox VM $vmid guest-agent network-get-interfaces:' sudo '$pimox_qm_bin' guest cmd '$vmid' network-get-interfaces" >&2 || true } guest_ip_from_agent() { guest_json="$(ssh_pimox "sudo '$pimox_qm_bin' guest cmd '$vmid' network-get-interfaces" 2>/dev/null || true)" if [ -z "$guest_json" ]; then return 1 fi GUEST_JSON="$guest_json" python3 - "$guest_ip_prefix" <<'PY' import json import os import sys prefix = sys.argv[1] try: interfaces = json.loads(os.environ.get("GUEST_JSON", "")) except Exception: sys.exit(1) for iface in interfaces or []: for address in iface.get("ip-addresses") or []: if address.get("ip-address-type") != "ipv4": continue ip = address.get("ip-address", "") if not ip or ip.startswith(("127.", "169.254.")): continue if prefix and not ip.startswith(prefix): continue print(ip) sys.exit(0) sys.exit(1) PY } if ssh_pimox "sudo '$pimox_qm_bin' config '$vmid' | grep -q '^template: 1$'"; then ssh_pimox "sudo '$pimox_qm_bin' set '$vmid' --agent enabled=1" exit 0 fi deadline=$((SECONDS + timeout_seconds)) next_log=$SECONDS while (( SECONDS < deadline )); do if [ -z "$guest_host" ]; then guest_host="$(guest_ip_from_agent || true)" fi if [ -n "$guest_host" ]; then if [ "$last_known_hosts_ip" != "$guest_host" ]; then ssh-keygen -R "$guest_host" -f "$known_hosts_file" >/dev/null 2>&1 || true last_known_hosts_ip="$guest_host" fi if last_ssh_output="$(ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh" 2>&1)"; then break fi fi if (( SECONDS >= next_log )); then elapsed=$((timeout_seconds - (deadline - SECONDS))) if [ -n "$guest_host" ]; then echo "Waiting for SSH and template preparation script on VM $vmid at $guest_host ($${elapsed}s elapsed)..." if [ -n "$last_ssh_output" ]; then echo "Last SSH failure: $last_ssh_output" fi else echo "Waiting for VM $vmid to boot the installed guest and report an IP through qemu-guest-agent ($${elapsed}s elapsed)..." fi next_log=$((SECONDS + 60)) fi sleep 15 done if [ -z "$guest_host" ]; then echo "Timed out waiting for VM $vmid to report a guest IP through qemu-guest-agent" >&2 debug_pimox_vm exit 1 fi if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2 if [ -n "$last_ssh_output" ]; then echo "Last SSH failure: $last_ssh_output" >&2 fi debug_pimox_vm exit 1 fi ssh_guest "sudo /usr/local/sbin/homelab-prepare-template.sh" ssh_guest "sudo nohup sh -c 'sleep 2; poweroff' >/dev/null 2>&1 &" || true EOT } } resource "null_resource" "pimox_template_vm_finalize" { count = var.pimox_template_builder_enabled ? 1 : 0 depends_on = [null_resource.pimox_template_vm_seal] triggers = { pimox_host = var.pimox_host pimox_user = var.pimox_user ssh_key_path = var.pimox_ssh_key_path qm_bin = var.pimox_qm_bin finalizer_version = "2" vmid = tostring(var.pimox_template_vmid) } connection { type = "ssh" user = self.triggers.pimox_user private_key = file(self.triggers.ssh_key_path) host = self.triggers.pimox_host } provisioner "remote-exec" { inline = [ <&2 exit 1 fi if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then sudo "$qm_cmd" set "$vmid" --agent enabled=1 exit 0 fi elapsed=0 while [ "$elapsed" -lt 600 ]; do if sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then break fi sleep 5 elapsed=$((elapsed + 5)) done if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then echo "Timed out waiting for VM $vmid to stop before template conversion" >&2 exit 1 fi sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0" sudo "$qm_cmd" template "$vmid" EOT ] } } resource "null_resource" "provisioning_host" { triggers = { host = var.provisioning_host user = var.provisioning_user ssh_key_path = var.provisioning_ssh_key_path install_dir = var.provisioning_install_dir tftp_root = local.tftp_root http_root = local.http_root netboot_base_url = local.debian_netboot_base_url pxe_boot_file = var.pxe_boot_file http_port = tostring(var.http_port) config_hash = local.config_hash provisioning_layer = "1" provisioner_version = "3" } connection { type = "ssh" user = self.triggers.user private_key = file(self.triggers.ssh_key_path) host = self.triggers.host } provisioner "remote-exec" { inline = [ "rm -rf /tmp/homelab-provisioning", "mkdir -p /tmp/homelab-provisioning", ] } provisioner "file" { content = local.dnsmasq_conf destination = "/tmp/homelab-provisioning/dnsmasq.conf" } provisioner "file" { content = local.nginx_conf destination = "/tmp/homelab-provisioning/nginx.conf" } provisioner "file" { content = local.grub_cfg destination = "/tmp/homelab-provisioning/grub.cfg" } provisioner "file" { content = local.preseed_cfg destination = "/tmp/homelab-provisioning/preseed.cfg" } provisioner "file" { content = local.golden_node_prepare destination = "/tmp/homelab-provisioning/golden-node-prepare.sh" } provisioner "file" { content = local.prepare_template destination = "/tmp/homelab-provisioning/prepare-template.sh" } provisioner "remote-exec" { inline = [ </dev/null | grep -q "install ok installed"; then missing_packages="$missing_packages $package" fi done if [ -n "$missing_packages" ]; then sudo apt-get update sudo apt-get install -y --no-install-recommends $missing_packages fi } install_missing_packages ca-certificates curl dnsmasq nginx python3 sudo install -d -m 0755 \ "$install_dir" \ "$tftp_root" \ "$tftp_root/debian-installer/arm64" \ "$tftp_root/debian-installer/arm64/grub" \ "$tftp_root/grub" \ "$http_root" \ "$http_root/preseed" \ "$http_root/scripts" \ "$http_root/debian-installer/arm64" sudo cp "$tmp_dir/grub.cfg" "$tftp_root/grub/grub.cfg" sudo cp "$tmp_dir/grub.cfg" "$tftp_root/debian-installer/arm64/grub/grub.cfg" sudo cp "$tmp_dir/preseed.cfg" "$http_root/preseed/debian13-arm64-worker.cfg" sudo cp "$tmp_dir/golden-node-prepare.sh" "$http_root/scripts/golden-node-prepare.sh" sudo cp "$tmp_dir/prepare-template.sh" "$http_root/scripts/prepare-template.sh" sudo chmod 0644 \ "$tftp_root/grub/grub.cfg" \ "$tftp_root/debian-installer/arm64/grub/grub.cfg" \ "$http_root/preseed/debian13-arm64-worker.cfg" sudo chmod 0755 "$http_root/scripts/golden-node-prepare.sh" "$http_root/scripts/prepare-template.sh" for asset in linux initrd.gz "$pxe_boot_file"; do sudo curl -fsSL "$netboot_base_url/$asset" -o "$tftp_root/debian-installer/arm64/$asset.tmp" sudo mv "$tftp_root/debian-installer/arm64/$asset.tmp" "$tftp_root/debian-installer/arm64/$asset" done sudo cp "$tftp_root/debian-installer/arm64/$pxe_boot_file" "$tftp_root/$pxe_boot_file" sudo cp "$tftp_root/debian-installer/arm64/linux" "$http_root/debian-installer/arm64/linux" sudo cp "$tftp_root/debian-installer/arm64/initrd.gz" "$http_root/debian-installer/arm64/initrd.gz" sudo cp "$tmp_dir/dnsmasq.conf" /etc/dnsmasq.d/homelab-pxe.conf sudo cp "$tmp_dir/nginx.conf" /etc/nginx/sites-available/homelab-provisioning.conf sudo ln -sfn ../sites-available/homelab-provisioning.conf /etc/nginx/sites-enabled/homelab-provisioning.conf sudo nginx -t sudo systemctl enable --now nginx >/dev/null sudo systemctl restart nginx sudo dnsmasq --test --conf-file=/etc/dnsmasq.d/homelab-pxe.conf sudo systemctl enable --now dnsmasq >/dev/null sudo systemctl restart dnsmasq EOT ] } }