Add homelab provisioning automation

2026-05-26 11:46:38 -06:00 · 2026-05-26 11:46:38 -06:00 · 11ea473c7f
parent b0a5a0bd67
commit 11ea473c7f
21 changed files with 1252 additions and 69 deletions
--- a/README.md
+++ b/README.md
@ -9,6 +9,8 @@ The lab is intentionally small but production-shaped:

 - a Debian amd64 host runs the kubeadm control plane and local deployment tools
 - a Raspberry Pi arm64 node runs selected workloads
+- a provisioning layer can PXE boot Debian 13 arm64 VMs for Pimox worker
+  templates
 - OpenTofu owns the bootstrap layers for cluster, platform, apps, and edge
 - Argo CD continuously reconciles Kubernetes manifests from this repo
 - a local registry stores the website and demos images built for the worker
@ -22,14 +24,21 @@ accidentally modify the cluster.

 ## Flow

-1. `bootstrap/cluster`
+1. `bootstrap/provisioning`
+   - prepares a Debian server as a PXE and preseed service for arm64 VMs
+   - serves Debian 13 arm64 netboot assets through TFTP and HTTP
+   - creates a golden image install path with Kubernetes, containerd,
+     qemu-guest-agent, cloud-init, and storage client packages ready
+   - stays out of `./lab.sh up` so VM template creation remains manual
+
+2. `bootstrap/cluster`
   - creates the kubeadm control plane on the Debian amd64 node
   - joins worker nodes such as Raspberry Pi arm64 nodes
   - configures Calico-compatible pod CIDR
   - configures containerd to pull from the in-cluster NodePort registry
   - creates retained host directories under `/var/openebs/local`

-2. `bootstrap/platform`
+3. `bootstrap/platform`
   - installs a minimal Calico deployment through the Tigera operator
   - installs OpenEBS
   - creates `openebs-hostpath-retain`
@ -37,12 +46,12 @@ accidentally modify the cluster.
   - registers the private GitOps repo without storing the SSH private key in
     Terraform state

-3. `bootstrap/apps`
+4. `bootstrap/apps`
   - registers Argo CD Applications from the `applications` map
   - default apps are `container-registry`, `gitea`, `website-production`, and
     `demos-static`

-4. `bootstrap/edge`
+5. `bootstrap/edge`
   - connects to the OCI jump box
   - uploads nginx, HAProxy, Varnish, and Squid configs
   - obtains and renews Let's Encrypt certificates for the configured hostname
@ -108,6 +117,11 @@ hostname.

 ## Adding Nodes

+For Pimox on Orange Pi 5 Plus, use `bootstrap/provisioning` to create a Debian
+13 arm64 golden image first. The layer serves PXE, preseed, and guest-prep
+assets from the Debian homelab server, then the installed VM can be sealed and
+converted to a Pimox template. Details are in `bootstrap/provisioning/README.md`.
+
 Add entries to `bootstrap/cluster/variables.tf` or a `.tfvars` file:

 ```hcl
@ -293,7 +307,8 @@ targets. For a single-node rebuild, set it to an empty string.
 The website is a PHP app under `apps/website`. It includes a home page, CV page,
 blog page, and demos page, plus a lightweight translation flow backed by Ollama.
 Static language files live in `apps/website/lang`; unsupported browser languages
-can be translated by the client and saved through `save_lang.php`.
+can be translated by the client and saved through `save_lang.php` as runtime
+JSON data on the website PVC.

 The CV page has two client-side presentation modes:

--- a/apps/website/Dockerfile
+++ b/apps/website/Dockerfile
@ -37,7 +37,7 @@ RUN usermod -u 1000 apache && \
    mkdir -p /run/apache2 /var/log/apache2 /tmp/website-lang && \
    chown -R apache:apache /run/apache2 /var/log/apache2 /tmp/website-lang /var/www/localhost/htdocs/db

-ENV WEBSITE_LANG_WRITE_DIR=/tmp/website-lang
+ENV WEBSITE_LANG_WRITE_DIR=/var/www/localhost/htdocs/db/lang

 USER apache

--- a/apps/website/lang_helper.php
+++ b/apps/website/lang_helper.php
@ -2,34 +2,71 @@

 $staticLangDir = __DIR__ . '/lang';
 $runtimeLangDir = getenv('WEBSITE_LANG_WRITE_DIR') ?: null;
-$langFiles = glob($staticLangDir . '/*.php') ?: [];
+$staticLangFiles = glob($staticLangDir . '/*.php') ?: [];
+$runtimeLangFiles = [];
 if ($runtimeLangDir && is_dir($runtimeLangDir)) {
-    $langFiles = array_merge($langFiles, glob($runtimeLangDir . '/*.php') ?: []);
+    $runtimeLangFiles = glob($runtimeLangDir . '/*.json') ?: [];
 }

 $availableLangs = array_values(array_unique(array_map(
-    fn($f) => basename($f, '.php'),
-    $langFiles
+    fn($f) => preg_replace('/\.(php|json)$/', '', basename($f)),
+    array_merge($staticLangFiles, $runtimeLangFiles)
 )));
+$availableLangs = array_values(array_filter(
+    $availableLangs,
+    fn($code) => is_string($code) && preg_match('/^[a-z]{2,5}$/', $code)
+));

 function getLang($supported) {
-    if (isset($_GET['lang']) && in_array($_GET['lang'], $supported)) {
+    if (isset($_GET['lang']) && in_array($_GET['lang'], $supported, true)) {
        return $_GET['lang'];
    }
-    $browser = substr($_SERVER['HTTP_ACCEPT_LANGUAGE'] ?? 'nah', 0, 2);
-    return in_array($browser, $supported) ? $browser : 'nah';
+    $browser = strtolower(substr($_SERVER['HTTP_ACCEPT_LANGUAGE'] ?? 'nah', 0, 2));
+    return in_array($browser, $supported, true) ? $browser : 'nah';
+}
+
+function cleanRuntimeTranslation(string $value, int $maxLength = 2000): string {
+    $value = strip_tags($value);
+    $value = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $value) ?? '';
+    $value = trim($value);
+    if (strlen($value) > $maxLength) {
+        $value = substr($value, 0, $maxLength);
+    }
+    return $value;
+}
+
+function loadRuntimeTranslations(string $file, array $base): array {
+    $raw = file_get_contents($file);
+    if ($raw === false || strlen($raw) > 131072) {
+        return [];
+    }
+
+    $decoded = json_decode($raw, true);
+    if (!is_array($decoded)) {
+        return [];
+    }
+
+    $translations = [];
+    foreach ($decoded as $key => $value) {
+        if (!is_string($key) || !array_key_exists($key, $base) || (!is_string($value) && !is_numeric($value))) {
+            continue;
+        }
+        $translations[$key] = cleanRuntimeTranslation((string) $value);
+    }
+    return $translations;
 }

 $lang = getLang($availableLangs);

-$file = $runtimeLangDir ? "$runtimeLangDir/$lang.php" : '';
-if (!$file || !file_exists($file)) {
-    $file = "$staticLangDir/$lang.php";
-}
-if (!file_exists($file)) {
-    $lang = 'nah';
-    $file = "$staticLangDir/nah.php";
-}
-
 $en = include "$staticLangDir/en.php";
-$text = array_replace($en, include $file);
+$staticFile = "$staticLangDir/$lang.php";
+$runtimeFile = $runtimeLangDir ? "$runtimeLangDir/$lang.json" : '';
+
+if (file_exists($staticFile)) {
+    $text = array_replace($en, include $staticFile);
+} elseif ($runtimeFile && file_exists($runtimeFile)) {
+    $text = array_replace($en, loadRuntimeTranslations($runtimeFile, $en));
+} else {
+    $lang = 'nah';
+    $text = array_replace($en, include "$staticLangDir/nah.php");
+}
--- a/apps/website/save_lang.php
+++ b/apps/website/save_lang.php
@ -2,58 +2,70 @@

 header('Content-Type: application/json');

-if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
-    http_response_code(405);
-    echo json_encode(['error' => 'Method not allowed']);
+function translation_response(int $status, array $body): never {
+    http_response_code($status);
+    echo json_encode($body, JSON_UNESCAPED_SLASHES);
    exit;
 }

+function clean_translation_value(string $value, int $maxLength = 2000): string {
+    $value = strip_tags($value);
+    $value = preg_replace('/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/u', '', $value) ?? '';
+    $value = trim($value);
+    if (strlen($value) > $maxLength) {
+        $value = substr($value, 0, $maxLength);
+    }
+    return $value;
+}
+
+if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
+    translation_response(405, ['error' => 'Method not allowed']);
+}
+
+if ((int) ($_SERVER['CONTENT_LENGTH'] ?? 0) > 131072) {
+    translation_response(413, ['error' => 'Request too large']);
+}
+
 $body = json_decode(file_get_contents('php://input'), true);

-if (!isset($body['lang'], $body['translations'])) {
-    http_response_code(400);
-    echo json_encode(['error' => 'Missing lang or translations']);
-    exit;
+if (!is_array($body) || !isset($body['lang'], $body['translations']) || !is_array($body['translations'])) {
+    translation_response(400, ['error' => 'Missing lang or translations']);
 }

 $lang         = preg_replace('/[^a-z]/', '', strtolower($body['lang']));
 $translations = $body['translations'];

 if (strlen($lang) < 2 || strlen($lang) > 5) {
-    http_response_code(400);
-    echo json_encode(['error' => 'Invalid lang code']);
-    exit;
+    translation_response(400, ['error' => 'Invalid lang code']);
 }

 $base = include __DIR__ . '/lang/en.php';

+if (file_exists(__DIR__ . "/lang/$lang.php")) {
+    translation_response(409, ['error' => 'Static language already exists']);
+}
+
 foreach ($translations as $key => $value) {
-    if (array_key_exists($key, $base)) {
-        $base[$key] = $value;
+    if (is_string($key) && array_key_exists($key, $base) && (is_string($value) || is_numeric($value))) {
+        $base[$key] = clean_translation_value((string) $value);
    }
 }

-$lines = ["<?php", "return ["];
-foreach ($base as $key => $value) {
-    $key   = addslashes($key);
-    $value = addslashes($value);
-    $lines[] = "    '$key' => '$value',";
+$content = json_encode($base, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT) . "\n";
+if ($content === false) {
+    translation_response(500, ['error' => 'Could not encode language file']);
 }
-$lines[] = "];";
-$content = implode("\n", $lines) . "\n";

-$langDir = getenv('WEBSITE_LANG_WRITE_DIR') ?: (__DIR__ . '/lang');
+$langDir = getenv('WEBSITE_LANG_WRITE_DIR') ?: (sys_get_temp_dir() . '/website-lang');
 if (!is_dir($langDir) && !mkdir($langDir, 0755, true)) {
-    http_response_code(500);
-    echo json_encode(['error' => 'Could not create language directory']);
-    exit;
+    translation_response(500, ['error' => 'Could not create language directory']);
 }

-$path = "$langDir/$lang.php";
-if (file_put_contents($path, $content) === false) {
-    http_response_code(500);
-    echo json_encode(['error' => 'Could not write language file']);
-    exit;
+$path = "$langDir/$lang.json";
+$tmpPath = "$path.tmp." . bin2hex(random_bytes(6));
+if (file_put_contents($tmpPath, $content, LOCK_EX) === false || !rename($tmpPath, $path)) {
+    @unlink($tmpPath);
+    translation_response(500, ['error' => 'Could not write language file']);
 }

 echo json_encode(['success' => true, 'lang' => $lang]);
--- a/apps/website/translation.js
+++ b/apps/website/translation.js
@ -71,7 +71,7 @@ async function saveLang(lang, translations) {
    });
    const data = await res.json();
    if (data.success) {
-      console.log(`Saved lang/${lang}.php — static next visit`);
+      console.log(`Saved runtime translation ${lang}.json`);
    } else {
      console.warn('Save failed:', data.error);
    }
--- a/apps/website/web-app.yaml
+++ b/apps/website/web-app.yaml
@ -65,7 +65,7 @@ spec:
                - ALL
          env:
            - name: WEBSITE_LANG_WRITE_DIR
-              value: /tmp/website-lang
+              value: /var/www/localhost/htdocs/db/lang
            - name: WEBSITE_IDEAS_WRITE_DIR
              value: /var/www/localhost/htdocs/db/ideas
          ports:
--- a/bootstrap/edge/templates/default.conf.tftpl
+++ b/bootstrap/edge/templates/default.conf.tftpl
@ -1,13 +1,10 @@
-# WAF-like rules
 map $request_uri $blocked_uris {
    default 0;
    ~*(/\.env|/\.git|/\.aws|/wp-admin) 1;
 }

-# Rate limiting zone
 limit_req_zone $binary_remote_addr zone=one:10m rate=10r/s;

-# Cache zones
 proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=static_assets:10m max_size=100m inactive=24h;
 proxy_cache_path /var/cache/nginx_dynamic levels=1:2 keys_zone=dynamic_content:5m max_size=50m inactive=1h;

@ -15,7 +12,6 @@ upstream haproxy_backend {
    server haproxy-dev:9000;
 }

-# Cloudflare IP ranges
 set_real_ip_from 173.245.48.0/20;
 set_real_ip_from 103.21.244.0/22;
 set_real_ip_from 103.22.200.0/22;
--- a/bootstrap/edge/templates/docker-compose.yml.tftpl
+++ b/bootstrap/edge/templates/docker-compose.yml.tftpl
@ -20,25 +20,18 @@ services:
    depends_on:
      - varnish-dev
      - squid-dev
-    ports:
-      - "9000:9000"
-      - "8404:8404"
    volumes:
      - ./config_files/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro

  varnish-dev:
    image: varnish:fresh-alpine
    restart: unless-stopped
-    ports:
-      - "6081:80"
    volumes:
      - ./config_files/default.vcl:/etc/varnish/default.vcl:ro

  squid-dev:
    image: ubuntu/squid:latest
    restart: unless-stopped
-    ports:
-      - "3128:3128"
    volumes:
      - ./config_files/squid.conf:/etc/squid/squid.conf:ro
      - squid_cache:/var/spool/squid
--- a/bootstrap/platform/main.tf
+++ b/bootstrap/platform/main.tf
@ -39,10 +39,10 @@ resource "null_resource" "calico_helm_recovery" {
  depends_on = [helm_release.calico_crds]

  triggers = {
-    always          = timestamp()
    kubeconfig_path = var.kubeconfig_path
    namespace       = var.calico.namespace
    release_name    = "calico"
+    release_version = var.calico.version
  }

  provisioner "local-exec" {
--- a/bootstrap/provisioning/.terraform.lock.hcl
+++ b/bootstrap/provisioning/.terraform.lock.hcl
@ -0,0 +1,39 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/hashicorp/null" {
+  version     = "3.3.0"
+  constraints = "~> 3.2"
+  hashes = [
+    "h1:0r7+t8CqzjfBgHgEiJGBCw+McEUdRXliMdF+Hk29d8o=",
+    "h1:EvvCOc4FJY3NitSm6BpzCcUPU53LayVCB/tPOxYmy7U=",
+    "h1:IDVnZXNCh0u4LfeSazc9z1v/kNz+92Eej7ePWV6SbyE=",
+    "h1:Iw2c0n9/4fS92N5WnJ3CCSwSUXZO953oHp9gj3pWCaM=",
+    "h1:JofS1og3hPN0ANjH+gNjxrJyyk6znodpC/F0qhp4eEk=",
+    "h1:QIBhsJ4+5+t0vFEgJwtezNLT31tsptFHOEyGAAhLR1o=",
+    "h1:RjjoL9qRPwNTwLdtJsYUaFvunbPM2/oujf2DcUcitOE=",
+    "h1:SSirA+z2VWTs1s+TCAx8vVKg9jh6cRjxqc8LYi2iQTI=",
+    "h1:U2XZc7hxcpcWp/C2S9LtuGUimhMOD2UT5xAEJJQQQaU=",
+    "h1:bPG+xE5UonkJv3y/Yn9Q7OfbP2qHU/QKiS31nwfe7S0=",
+    "h1:eODLdk/pARc4yxChAFtwseVmBr+r5fF9yGOvUhwGEyM=",
+    "h1:iFj1oM5ZPENspsPqK1kcvZzyP95jJE/CM0rlu0MfIss=",
+    "h1:mdu+qpyVmjDDLMrcL1JFy+cSyF58I3TFJwB5NssCZ58=",
+    "h1:tJmep6aoBeDH77XsYU65HAbi0RAjxtsmbCOXmnqT13U=",
+    "h1:tdMTn1evBLd6KCeLqWdQXCpF07hBu3n5rY6N3rXw3Rc=",
+    "zh:083dcc0bec53f8abfa3f2aa2ce9d732a9675338fd60ae7d61162e25db7cb08bf",
+    "zh:19f7456b5a2ad16595860974714bfdb25b87bc16356ea9d5c7453892aaa27864",
+    "zh:222c0ed1fed4e4c677ebe626104dbfdba66763e264de0d9c27c58ce60104ee69",
+    "zh:271711d6caa7dd5a4e9b79fe8c679fab61a840bcf80040a0f5ebb425d1b27d97",
+    "zh:5adcf35f30baaea13f80c2a2c774deb9369892719493049687e23476c9dff40f",
+    "zh:5bcfd19df16e73d7f0ad75bd09e2b3b86cf6700d09822d585d68304b71de1d97",
+    "zh:604edecf263e38674decb35bb4e0e048fdc951f26fa103c33065ff9728f0313b",
+    "zh:782acbfb4fa4807e273e588fe45b4aaea9dd0fd1136f76ec3200f6f4db3af8d6",
+    "zh:84411a596d528fe67294e5c1cfd0c2036b08802497bcc4215ce518924f3c9a4a",
+    "zh:85e79eecf3f5348975cffec3016b0eba3baf605646102d4348796ccd2df2e5f6",
+    "zh:95669535ca17aeefef307ebfd59ce6930953173baae5637e8cbbf0297ec7ad58",
+    "zh:d04d9b177747bfd66b4a45b5d911a2a7822aa8451f5e35621971fb7a4206b530",
+    "zh:e6d9c924475283e90833450a14a732f4deb6d9bb131db8f86ab856e894270836",
+    "zh:ebcab0c8a1334c86ed7cfa53f571a17ad6d27e9901f27a8854ea622a74b54bb6",
+    "zh:ef9c757bb2c83d2103811a3d86b6ec5be06b0ffc337b84db1582d023bce7cdcd",
+  ]
+}
--- a/bootstrap/provisioning/README.md
+++ b/bootstrap/provisioning/README.md
@ -0,0 +1,95 @@
+# Homelab Provisioning Layer
+
+This layer prepares a Debian server to PXE boot a Debian 13 arm64 VM and install a reusable worker-node golden image for Pimox on Orange Pi 5 Plus.
+
+It is intentionally separate from `./lab.sh up`. Run it manually from the Debian homelab server only when you want to create or refresh the provisioning service.
+
+## What It Installs
+
+- `dnsmasq` for proxyDHCP and TFTP
+- `nginx` for preseed, installer, and guest-prep scripts
+- Debian 13 arm64 netboot assets under `/opt/homelab-provisioning`
+- a preseed file for unattended Debian install
+- guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path
+- kernel boot options for cgroup support through `TF_VAR_kernel_cgroup_boot_options`
+- a template sealing script at `/usr/local/sbin/homelab-prepare-template.sh` inside the installed VM that verifies cgroup boot state before sealing
+
+## Apply From Debian
+
+Find the LAN interface on the Debian server:
+
+```bash
+ip -br addr
+```
+
+Apply the provisioning layer:
+
+```bash
+cd ~/my-homelab-configs
+tofu -chdir=bootstrap/provisioning init
+TF_VAR_provisioning_interface=enp1s0 tofu -chdir=bootstrap/provisioning apply
+```
+
+Override `TF_VAR_provisioning_interface` with the interface that serves the Pimox VM network.
+
+The default VM user is `jv`. The account uses `/home/jv/.ssh/id_ed25519.pub`
+from the Debian server when that key exists, and the password is locked by
+default. Set `TF_VAR_template_user_ssh_authorized_keys` or
+`TF_VAR_template_user_password_hash` before applying if you want different
+access.
+
+Clones should get their intended hostname through cloud-init or the later clone
+automation. If they boot with the template hostname, the first-boot service
+generates a unique fallback name using `TF_VAR_clone_hostname_prefix`.
+
+## Pimox VM Template Flow
+
+Create an arm64 VM in Pimox with UEFI firmware, a virtio disk, and a NIC on the same LAN as the Debian provisioning host. Put network boot first.
+
+PXE should load `grubaa64.efi`, boot the Debian installer, fetch the preseed from `http://192.168.100.68:8088/preseed/debian13-arm64-worker.cfg`, and install the golden image.
+
+If your Pimox firmware needs a different Debian arm64 EFI loader, override `TF_VAR_pxe_boot_file`.
+
+After the first successful boot, run this inside the VM before converting it to a template:
+
+```bash
+sudo /usr/local/sbin/homelab-prepare-template.sh
+sudo poweroff
+```
+
+Convert the powered-off VM to a Pimox template. Clone it for each new worker, set a unique hostname and IP address through cloud-init or DHCP reservation, then add it to `bootstrap/cluster/variables.tf` or a `.tfvars` file:
+
+```hcl
+worker_nodes = {
+  pimox01 = {
+    host         = "192.168.100.90"
+    user         = "jv"
+    node_name    = "pimox01"
+    ssh_key_path = "/home/jv/.ssh/id_ed25519"
+  }
+}
+```
+
+Run the cluster layer from the Debian homelab server after the cloned VM is reachable over SSH.
+
+## Optional Pimox Automation
+
+Set `TF_VAR_pimox_template_builder_enabled=true` to have this layer SSH into the
+Pimox host, create the template-build VM with `qm`, boot it from PXE, wait for
+the installed VM over SSH, run `/usr/local/sbin/homelab-prepare-template.sh`,
+power it off, switch boot order back to disk first, and run `qm template`.
+
+The automation needs a predictable temporary IP for the installed VM. Use a DHCP
+reservation for the generated or configured MAC address, then set:
+
+```bash
+export TF_VAR_pimox_template_builder_enabled=true
+export TF_VAR_pimox_host=192.168.100.91
+export TF_VAR_pimox_template_build_host=192.168.100.90
+```
+
+Defaults match the observed Pimox VM shape: OVMF firmware, virtio networking,
+virtio-scsi disk, `vmbr0`, `local` storage, 2 vCPU, and 2 GiB memory. Override
+`TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`,
+`TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange
+Pi storage layout changes.
--- a/bootstrap/provisioning/main.tf
+++ b/bootstrap/provisioning/main.tf
@ -0,0 +1,407 @@
+terraform {
+  required_version = ">= 1.0"
+  required_providers {
+    null = {
+      source  = "hashicorp/null"
+      version = "~> 3.2"
+    }
+  }
+}
+
+locals {
+  tftp_root = "${var.provisioning_install_dir}/tftp"
+  http_root = "${var.provisioning_install_dir}/html"
+
+  debian_netboot_base_url = var.debian_netboot_base_url != "" ? var.debian_netboot_base_url : "http://${var.debian_mirror_host}${var.debian_mirror_directory}/dists/${var.debian_suite}/main/installer-arm64/current/images/netboot/debian-installer/arm64"
+  preseed_url             = "http://${var.http_host}:${var.http_port}/preseed/debian13-arm64-worker.cfg"
+
+  template_packages = distinct(concat([
+    "sudo",
+    "openssh-server",
+    "curl",
+    "ca-certificates",
+    "gnupg",
+    "apt-transport-https",
+    "qemu-guest-agent",
+    "cloud-init",
+    "containerd",
+    "open-iscsi",
+    "nfs-common",
+    "iptables",
+    "iproute2",
+    "conntrack",
+    "socat",
+    "ebtables",
+    "ethtool",
+    "ipset",
+    "ipvsadm",
+    "jq",
+    "vim-tiny",
+    "chrony",
+    "lvm2",
+    "xfsprogs",
+  ], var.additional_template_packages))
+
+  template_user_ssh_keys      = distinct(compact(concat(var.template_user_ssh_authorized_keys, [try(trimspace(file(var.template_user_ssh_public_key_path)), "")])))
+  ssh_authorized_keys_base64  = base64encode(join("\n", local.template_user_ssh_keys))
+  node_dns_servers            = join(" ", var.node_dns_servers)
+  kernel_cgroup_boot_options  = join(" ", var.kernel_cgroup_boot_options)
+  pimox_template_net0         = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}"
+  template_package_list       = join(" ", local.template_packages)
+  provisioning_http_base_url  = "http://${var.http_host}:${var.http_port}"
+  provisioning_script_url     = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh"
+  prepare_template_script_url = "${local.provisioning_http_base_url}/scripts/prepare-template.sh"
+
+  dnsmasq_conf = templatefile("${path.module}/templates/dnsmasq.conf.tftpl", {
+    provisioning_interface = var.provisioning_interface
+    proxy_dhcp_range       = var.proxy_dhcp_range
+    pxe_boot_file          = var.pxe_boot_file
+    tftp_root              = local.tftp_root
+  })
+
+  nginx_conf = templatefile("${path.module}/templates/nginx.conf.tftpl", {
+    http_port = tostring(var.http_port)
+    http_root = local.http_root
+  })
+
+  grub_cfg = templatefile("${path.module}/templates/grub.cfg.tftpl", {
+    preseed_url       = local.preseed_url
+    template_hostname = var.template_hostname
+    template_domain   = var.template_domain
+  })
+
+  preseed_cfg = templatefile("${path.module}/templates/preseed.cfg.tftpl", {
+    locale                      = var.locale
+    keyboard                    = var.keyboard
+    timezone                    = var.timezone
+    template_hostname           = var.template_hostname
+    template_domain             = var.template_domain
+    template_disk               = var.template_disk
+    template_user               = var.template_user
+    template_user_full_name     = var.template_user_full_name
+    template_user_password_hash = var.template_user_password_hash
+    debian_mirror_host          = var.debian_mirror_host
+    debian_mirror_directory     = var.debian_mirror_directory
+    template_package_list       = local.template_package_list
+    provisioning_script_url     = local.provisioning_script_url
+    prepare_template_script_url = local.prepare_template_script_url
+  })
+
+  golden_node_prepare = templatefile("${path.module}/templates/golden-node-prepare.sh.tftpl", {
+    template_user              = var.template_user
+    ssh_authorized_keys_base64 = local.ssh_authorized_keys_base64
+    kubernetes_minor_version   = var.kubernetes_minor_version
+    kernel_cgroup_boot_options = local.kernel_cgroup_boot_options
+    registry_endpoint          = var.registry_endpoint
+    node_dns_servers           = local.node_dns_servers
+  })
+
+  prepare_template = templatefile("${path.module}/templates/prepare-template.sh.tftpl", {
+    template_hostname          = var.template_hostname
+    clone_hostname_prefix      = var.clone_hostname_prefix
+    template_domain            = var.template_domain
+    kernel_cgroup_boot_options = local.kernel_cgroup_boot_options
+  })
+
+  config_hash = sha256(join("\n---\n", [
+    local.dnsmasq_conf,
+    local.nginx_conf,
+    local.grub_cfg,
+    local.preseed_cfg,
+    local.golden_node_prepare,
+    local.prepare_template,
+    local.debian_netboot_base_url,
+  ]))
+}
+
+resource "null_resource" "pimox_template_vm_create" {
+  count = var.pimox_template_builder_enabled ? 1 : 0
+
+  depends_on = [null_resource.provisioning_host]
+
+  triggers = {
+    pimox_host       = var.pimox_host
+    pimox_user       = var.pimox_user
+    ssh_key_path     = var.pimox_ssh_key_path
+    vmid             = tostring(var.pimox_template_vmid)
+    name             = var.pimox_template_name
+    cores            = tostring(var.pimox_template_cores)
+    memory           = tostring(var.pimox_template_memory)
+    net0             = local.pimox_template_net0
+    scsi0            = var.pimox_template_scsi0
+    efidisk0         = var.pimox_template_efidisk0
+    replace_existing = tostring(var.pimox_template_replace_existing)
+  }
+
+  connection {
+    type        = "ssh"
+    user        = self.triggers.pimox_user
+    private_key = file(self.triggers.ssh_key_path)
+    host        = self.triggers.pimox_host
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      <<EOT
+set -eu
+
+vmid="${self.triggers.vmid}"
+replace_existing="${self.triggers.replace_existing}"
+
+if ! command -v qm >/dev/null 2>&1; then
+  echo "qm is not installed on this Pimox host" >&2
+  exit 1
+fi
+
+if sudo qm status "$vmid" >/dev/null 2>&1; then
+  if sudo qm config "$vmid" | grep -q '^template: 1$'; then
+    exit 0
+  fi
+  if [ "$replace_existing" != "true" ]; then
+    echo "VM $vmid already exists and is not a template. Set pimox_template_replace_existing=true to rebuild it." >&2
+    exit 1
+  fi
+  sudo qm stop "$vmid" >/dev/null 2>&1 || true
+  elapsed=0
+  while [ "$elapsed" -lt 300 ]; do
+    if sudo qm status "$vmid" | grep -q 'status: stopped'; then
+      break
+    fi
+    sleep 5
+    elapsed=$((elapsed + 5))
+  done
+  sudo qm destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo qm destroy "$vmid"
+fi
+
+sudo qm create "$vmid" \
+  --name "${self.triggers.name}" \
+  --bios ovmf \
+  --boot "order=net0;scsi0" \
+  --cores "${self.triggers.cores}" \
+  --memory "${self.triggers.memory}" \
+  --net0 "${self.triggers.net0}" \
+  --numa 0 \
+  --ostype l26 \
+  --scsihw virtio-scsi-pci \
+  --sockets 1 \
+  --vga virtio
+
+sudo qm set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
+sudo qm set "$vmid" --scsi0 "${self.triggers.scsi0}"
+sudo qm start "$vmid"
+EOT
+    ]
+  }
+}
+
+resource "null_resource" "pimox_template_vm_seal" {
+  count = var.pimox_template_builder_enabled ? 1 : 0
+
+  depends_on = [null_resource.pimox_template_vm_create]
+
+  triggers = {
+    host         = var.pimox_template_build_host
+    user         = var.pimox_template_build_user
+    ssh_key_path = var.pimox_template_build_ssh_key_path
+    timeout      = var.pimox_template_build_timeout
+    vmid         = tostring(var.pimox_template_vmid)
+  }
+
+  connection {
+    type        = "ssh"
+    user        = self.triggers.user
+    private_key = file(self.triggers.ssh_key_path)
+    host        = self.triggers.host
+    timeout     = self.triggers.timeout
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      <<EOT
+set -eu
+
+if [ -z "${self.triggers.host}" ]; then
+  echo "pimox_template_build_host must point to the installed VM before template sealing can run" >&2
+  exit 1
+fi
+
+sudo /usr/local/sbin/homelab-prepare-template.sh
+sudo nohup sh -c 'sleep 2; poweroff' >/dev/null 2>&1 &
+EOT
+    ]
+  }
+}
+
+resource "null_resource" "pimox_template_vm_finalize" {
+  count = var.pimox_template_builder_enabled ? 1 : 0
+
+  depends_on = [null_resource.pimox_template_vm_seal]
+
+  triggers = {
+    pimox_host   = var.pimox_host
+    pimox_user   = var.pimox_user
+    ssh_key_path = var.pimox_ssh_key_path
+    vmid         = tostring(var.pimox_template_vmid)
+  }
+
+  connection {
+    type        = "ssh"
+    user        = self.triggers.pimox_user
+    private_key = file(self.triggers.ssh_key_path)
+    host        = self.triggers.pimox_host
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      <<EOT
+set -eu
+
+vmid="${self.triggers.vmid}"
+elapsed=0
+while [ "$elapsed" -lt 600 ]; do
+  if sudo qm status "$vmid" | grep -q 'status: stopped'; then
+    break
+  fi
+  sleep 5
+  elapsed=$((elapsed + 5))
+done
+
+if ! sudo qm status "$vmid" | grep -q 'status: stopped'; then
+  echo "Timed out waiting for VM $vmid to stop before template conversion" >&2
+  exit 1
+fi
+
+sudo qm set "$vmid" --boot "order=scsi0;net0"
+sudo qm template "$vmid"
+EOT
+    ]
+  }
+}
+
+resource "null_resource" "provisioning_host" {
+  triggers = {
+    host               = var.provisioning_host
+    user               = var.provisioning_user
+    ssh_key_path       = var.provisioning_ssh_key_path
+    install_dir        = var.provisioning_install_dir
+    tftp_root          = local.tftp_root
+    http_root          = local.http_root
+    netboot_base_url   = local.debian_netboot_base_url
+    pxe_boot_file      = var.pxe_boot_file
+    http_port          = tostring(var.http_port)
+    config_hash        = local.config_hash
+    provisioning_layer = "1"
+  }
+
+  connection {
+    type        = "ssh"
+    user        = self.triggers.user
+    private_key = file(self.triggers.ssh_key_path)
+    host        = self.triggers.host
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "rm -rf /tmp/homelab-provisioning",
+      "mkdir -p /tmp/homelab-provisioning",
+    ]
+  }
+
+  provisioner "file" {
+    content     = local.dnsmasq_conf
+    destination = "/tmp/homelab-provisioning/dnsmasq.conf"
+  }
+
+  provisioner "file" {
+    content     = local.nginx_conf
+    destination = "/tmp/homelab-provisioning/nginx.conf"
+  }
+
+  provisioner "file" {
+    content     = local.grub_cfg
+    destination = "/tmp/homelab-provisioning/grub.cfg"
+  }
+
+  provisioner "file" {
+    content     = local.preseed_cfg
+    destination = "/tmp/homelab-provisioning/preseed.cfg"
+  }
+
+  provisioner "file" {
+    content     = local.golden_node_prepare
+    destination = "/tmp/homelab-provisioning/golden-node-prepare.sh"
+  }
+
+  provisioner "file" {
+    content     = local.prepare_template
+    destination = "/tmp/homelab-provisioning/prepare-template.sh"
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      <<EOT
+set -eu
+
+install_dir="${self.triggers.install_dir}"
+tftp_root="${self.triggers.tftp_root}"
+http_root="${self.triggers.http_root}"
+netboot_base_url="${self.triggers.netboot_base_url}"
+pxe_boot_file="${self.triggers.pxe_boot_file}"
+tmp_dir="/tmp/homelab-provisioning"
+
+install_missing_packages() {
+  missing_packages=""
+  for package in "$@"; do
+    if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
+      missing_packages="$missing_packages $package"
+    fi
+  done
+  if [ -n "$missing_packages" ]; then
+    sudo apt-get update
+    sudo apt-get install -y --no-install-recommends $missing_packages
+  fi
+}
+
+install_missing_packages ca-certificates curl dnsmasq nginx
+
+sudo install -d -m 0755 \
+  "$install_dir" \
+  "$tftp_root" \
+  "$tftp_root/debian-installer/arm64" \
+  "$tftp_root/grub" \
+  "$http_root" \
+  "$http_root/preseed" \
+  "$http_root/scripts" \
+  "$http_root/debian-installer/arm64"
+
+sudo cp "$tmp_dir/grub.cfg" "$tftp_root/grub/grub.cfg"
+sudo cp "$tmp_dir/preseed.cfg" "$http_root/preseed/debian13-arm64-worker.cfg"
+sudo cp "$tmp_dir/golden-node-prepare.sh" "$http_root/scripts/golden-node-prepare.sh"
+sudo cp "$tmp_dir/prepare-template.sh" "$http_root/scripts/prepare-template.sh"
+sudo chmod 0644 "$tftp_root/grub/grub.cfg" "$http_root/preseed/debian13-arm64-worker.cfg"
+sudo chmod 0755 "$http_root/scripts/golden-node-prepare.sh" "$http_root/scripts/prepare-template.sh"
+
+for asset in linux initrd.gz "$pxe_boot_file"; do
+  sudo curl -fsSL "$netboot_base_url/$asset" -o "$tftp_root/debian-installer/arm64/$asset.tmp"
+  sudo mv "$tftp_root/debian-installer/arm64/$asset.tmp" "$tftp_root/debian-installer/arm64/$asset"
+done
+
+sudo cp "$tftp_root/debian-installer/arm64/$pxe_boot_file" "$tftp_root/$pxe_boot_file"
+sudo cp "$tftp_root/debian-installer/arm64/linux" "$http_root/debian-installer/arm64/linux"
+sudo cp "$tftp_root/debian-installer/arm64/initrd.gz" "$http_root/debian-installer/arm64/initrd.gz"
+
+sudo cp "$tmp_dir/dnsmasq.conf" /etc/dnsmasq.d/homelab-pxe.conf
+sudo cp "$tmp_dir/nginx.conf" /etc/nginx/sites-available/homelab-provisioning.conf
+sudo ln -sfn ../sites-available/homelab-provisioning.conf /etc/nginx/sites-enabled/homelab-provisioning.conf
+
+sudo nginx -t
+sudo systemctl enable --now nginx >/dev/null
+sudo systemctl restart nginx
+
+sudo dnsmasq --test --conf-file=/etc/dnsmasq.d/homelab-pxe.conf
+sudo systemctl enable --now dnsmasq >/dev/null
+sudo systemctl restart dnsmasq
+EOT
+    ]
+  }
+}
--- a/bootstrap/provisioning/outputs.tf
+++ b/bootstrap/provisioning/outputs.tf
@ -0,0 +1,19 @@
+output "provisioning_http_base_url" {
+  value = local.provisioning_http_base_url
+}
+
+output "preseed_url" {
+  value = local.preseed_url
+}
+
+output "pxe_boot_file" {
+  value = var.pxe_boot_file
+}
+
+output "tftp_root" {
+  value = local.tftp_root
+}
+
+output "pimox_template_vmid" {
+  value = var.pimox_template_builder_enabled ? var.pimox_template_vmid : null
+}
--- a/bootstrap/provisioning/templates/dnsmasq.conf.tftpl
+++ b/bootstrap/provisioning/templates/dnsmasq.conf.tftpl
@ -0,0 +1,10 @@
+port=0
+interface=${provisioning_interface}
+bind-interfaces
+log-dhcp
+enable-tftp
+tftp-root=${tftp_root}
+dhcp-range=${proxy_dhcp_range}
+dhcp-match=set:efi-arm64,option:client-arch,11
+dhcp-boot=tag:efi-arm64,${pxe_boot_file}
+pxe-service=tag:efi-arm64,ARM64_EFI,Debian 13 arm64 automated install,${pxe_boot_file}
--- a/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl
+++ b/bootstrap/provisioning/templates/golden-node-prepare.sh.tftpl
@ -0,0 +1,173 @@
+#!/bin/sh
+set -eu
+
+install_ssh_key() {
+  if [ -z "${ssh_authorized_keys_base64}" ]; then
+    return 0
+  fi
+
+  install -d -m 0700 -o ${template_user} -g ${template_user} /home/${template_user}/.ssh
+  printf '%s' '${ssh_authorized_keys_base64}' | base64 -d >/home/${template_user}/.ssh/authorized_keys
+  chown ${template_user}:${template_user} /home/${template_user}/.ssh/authorized_keys
+  chmod 0600 /home/${template_user}/.ssh/authorized_keys
+}
+
+configure_sudo() {
+  printf '%s ALL=(ALL) NOPASSWD:ALL\n' '${template_user}' >/etc/sudoers.d/90-homelab-${template_user}
+  chmod 0440 /etc/sudoers.d/90-homelab-${template_user}
+}
+
+configure_dns() {
+  dns_servers="${node_dns_servers}"
+  if [ -z "$dns_servers" ]; then
+    return 0
+  fi
+
+  if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
+    mkdir -p /etc/systemd/resolved.conf.d
+    {
+      echo "[Resolve]"
+      printf 'DNS=%s\n' "$dns_servers"
+      printf 'FallbackDNS=%s\n' "$dns_servers"
+      echo "DNSSEC=no"
+    } >/etc/systemd/resolved.conf.d/homelab-k8s.conf
+  fi
+}
+
+configure_kubernetes_prereqs() {
+  swapoff -a || true
+  systemctl mask swap.target >/dev/null 2>&1 || true
+  awk '
+    /^[[:space:]]*#/ { print; next }
+    $3 == "swap" { next }
+    { print }
+  ' /etc/fstab >/etc/fstab.homelab
+  mv /etc/fstab.homelab /etc/fstab
+
+  printf 'overlay\nbr_netfilter\nip_vs\nip_vs_rr\nip_vs_wrr\nip_vs_sh\nnf_conntrack\n' >/etc/modules-load.d/k8s.conf
+  modprobe overlay || true
+  modprobe br_netfilter || true
+  modprobe ip_vs || true
+  modprobe ip_vs_rr || true
+  modprobe ip_vs_wrr || true
+  modprobe ip_vs_sh || true
+  modprobe nf_conntrack || true
+
+  cat >/etc/sysctl.d/99-kubernetes-cri.conf <<'SYSCTL'
+net.bridge.bridge-nf-call-iptables = 1
+net.bridge.bridge-nf-call-ip6tables = 1
+net.ipv4.ip_forward = 1
+SYSCTL
+  sysctl --system >/dev/null || true
+}
+
+configure_kernel_boot_options() {
+  boot_options="${kernel_cgroup_boot_options}"
+  if [ -z "$boot_options" ] || [ ! -f /etc/default/grub ]; then
+    return 0
+  fi
+
+  current_options="$(sed -n 's/^GRUB_CMDLINE_LINUX_DEFAULT=//p' /etc/default/grub | tail -n 1 | sed 's/^"//; s/"$//')"
+  for option in $boot_options; do
+    case " $current_options " in
+      *" $option "*) ;;
+      *) current_options="$current_options $option" ;;
+    esac
+  done
+  current_options="$(printf '%s' "$current_options" | awk '{$1=$1; print}')"
+
+  escaped_options="$(printf '%s' "$current_options" | sed 's/[\/&]/\\&/g')"
+  if grep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub; then
+    sed -i "s/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT=\"$escaped_options\"/" /etc/default/grub
+  else
+    printf 'GRUB_CMDLINE_LINUX_DEFAULT="%s"\n' "$current_options" >>/etc/default/grub
+  fi
+
+  if command -v update-grub >/dev/null 2>&1; then
+    update-grub
+  elif command -v grub-mkconfig >/dev/null 2>&1 && [ -d /boot/grub ]; then
+    grub-mkconfig -o /boot/grub/grub.cfg
+  fi
+}
+
+install_kubernetes_tools() {
+  install -d -m 0755 /etc/apt/keyrings
+  curl -fsSL "https://pkgs.k8s.io/core:/stable:/${kubernetes_minor_version}/deb/Release.key" | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+  printf 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${kubernetes_minor_version}/deb/ /\n' >/etc/apt/sources.list.d/kubernetes.list
+  apt-get update
+  apt-get install -y --no-install-recommends kubelet kubeadm kubectl
+  apt-mark hold kubelet kubeadm kubectl
+}
+
+configure_containerd() {
+  mkdir -p /etc/containerd /etc/containerd/certs.d/${registry_endpoint}
+  containerd config default >/etc/containerd/config.toml
+  sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
+
+  config_version="$(awk -F= '/^[[:space:]]*version[[:space:]]*=/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' /etc/containerd/config.toml)"
+  if [ "$config_version" = "3" ]; then
+    registry_table='[plugins."io.containerd.cri.v1.images".registry]'
+  else
+    registry_table='[plugins."io.containerd.grpc.v1.cri".registry]'
+  fi
+
+  awk -v registry_table="$registry_table" '
+    $0 == registry_table { in_registry = 1; found = 1; print; next }
+    in_registry && /^\[/ {
+      if (!wrote) {
+        print "  config_path = \"/etc/containerd/certs.d\""
+      }
+      in_registry = 0
+      wrote = 0
+    }
+    in_registry && /^[[:space:]]*config_path[[:space:]]*=/ {
+      print "  config_path = \"/etc/containerd/certs.d\""
+      wrote = 1
+      next
+    }
+    { print }
+    END {
+      if (in_registry && !wrote) {
+        print "  config_path = \"/etc/containerd/certs.d\""
+      }
+      if (!found) {
+        print ""
+        print registry_table
+        print "  config_path = \"/etc/containerd/certs.d\""
+      }
+    }
+  ' /etc/containerd/config.toml >/etc/containerd/config.toml.homelab
+  mv /etc/containerd/config.toml.homelab /etc/containerd/config.toml
+
+  cat >/etc/containerd/certs.d/${registry_endpoint}/hosts.toml <<'HOSTS'
+server = "http://${registry_endpoint}"
+
+[host."http://${registry_endpoint}"]
+  capabilities = ["pull", "resolve", "push"]
+  skip_verify = true
+HOSTS
+
+  cat >/etc/crictl.yaml <<'CRICTL'
+runtime-endpoint: unix:///run/containerd/containerd.sock
+image-endpoint: unix:///run/containerd/containerd.sock
+timeout: 10
+debug: false
+CRICTL
+}
+
+enable_services() {
+  systemctl enable qemu-guest-agent >/dev/null 2>&1 || true
+  systemctl enable containerd >/dev/null 2>&1 || true
+  systemctl enable kubelet >/dev/null 2>&1 || true
+  systemctl enable iscsid >/dev/null 2>&1 || true
+  systemctl enable ssh >/dev/null 2>&1 || true
+}
+
+install_ssh_key
+configure_sudo
+configure_dns
+configure_kubernetes_prereqs
+configure_kernel_boot_options
+install_kubernetes_tools
+configure_containerd
+enable_services
--- a/bootstrap/provisioning/templates/grub.cfg.tftpl
+++ b/bootstrap/provisioning/templates/grub.cfg.tftpl
@ -0,0 +1,7 @@
+set default=0
+set timeout=5
+
+menuentry 'Debian 13 arm64 homelab worker template' {
+    linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} --- quiet
+    initrd /debian-installer/arm64/initrd.gz
+}
--- a/bootstrap/provisioning/templates/nginx.conf.tftpl
+++ b/bootstrap/provisioning/templates/nginx.conf.tftpl
@ -0,0 +1,10 @@
+server {
+  listen ${http_port};
+  server_name _;
+  root ${http_root};
+  autoindex on;
+
+  location / {
+    try_files $uri $uri/ =404;
+  }
+}
--- a/bootstrap/provisioning/templates/prepare-template.sh.tftpl
+++ b/bootstrap/provisioning/templates/prepare-template.sh.tftpl
@ -0,0 +1,95 @@
+#!/bin/sh
+set -eu
+
+verify_kernel_cgroups() {
+  boot_options="${kernel_cgroup_boot_options}"
+  cmdline=" $(cat /proc/cmdline) "
+  for option in $boot_options; do
+    case "$cmdline" in
+      *" $option "*) ;;
+      *)
+        echo "Missing kernel boot option: $option" >&2
+        exit 1
+        ;;
+    esac
+  done
+
+  if [ ! -d /sys/fs/cgroup ]; then
+    echo "Missing /sys/fs/cgroup" >&2
+    exit 1
+  fi
+
+  if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+    if ! grep -qw memory /sys/fs/cgroup/cgroup.controllers; then
+      echo "Missing memory controller in cgroup v2" >&2
+      exit 1
+    fi
+    return 0
+  fi
+
+  if ! awk '$1 == "memory" && $4 == "1" { found = 1 } END { exit found ? 0 : 1 }' /proc/cgroups; then
+    echo "Missing enabled memory cgroup controller" >&2
+    exit 1
+  fi
+}
+
+verify_kernel_cgroups
+
+cat >/usr/local/sbin/homelab-firstboot-node.sh <<'SCRIPT'
+#!/bin/sh
+set -eu
+
+systemd-machine-id-setup
+ssh-keygen -A
+
+template_hostname='${template_hostname}'
+clone_hostname_prefix='${clone_hostname_prefix}'
+template_domain='${template_domain}'
+current_hostname="$(hostnamectl --static 2>/dev/null || hostname)"
+
+if [ -z "$current_hostname" ] || [ "$current_hostname" = "$template_hostname" ] || [ "$current_hostname" = "localhost" ]; then
+  machine_id="$(cat /etc/machine-id 2>/dev/null | tr -dc '[:xdigit:]' | cut -c1-8)"
+  if [ -z "$machine_id" ]; then
+    machine_id="$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n')"
+  fi
+  current_hostname="$clone_hostname_prefix-$machine_id"
+  hostnamectl set-hostname "$current_hostname"
+fi
+
+if grep -q '^127[.]0[.]1[.]1[[:space:]]' /etc/hosts; then
+  sed -i "s/^127[.]0[.]1[.]1[[:space:]].*/127.0.1.1 $current_hostname.$template_domain $current_hostname/" /etc/hosts
+else
+  printf '127.0.1.1 %s.%s %s\n' "$current_hostname" "$template_domain" "$current_hostname" >>/etc/hosts
+fi
+
+rm -f /etc/homelab-template-sealed
+systemctl disable homelab-firstboot-node.service >/dev/null 2>&1 || true
+SCRIPT
+
+chmod 0755 /usr/local/sbin/homelab-firstboot-node.sh
+
+cat >/etc/systemd/system/homelab-firstboot-node.service <<'SERVICE'
+[Unit]
+Description=Initialize cloned homelab node identity
+Before=ssh.service
+ConditionPathExists=/etc/homelab-template-sealed
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/homelab-firstboot-node.sh
+
+[Install]
+WantedBy=multi-user.target
+SERVICE
+
+systemctl enable homelab-firstboot-node.service >/dev/null
+cloud-init clean --logs >/dev/null 2>&1 || true
+rm -f /etc/ssh/ssh_host_*
+rm -f /var/lib/dbus/machine-id
+ln -sf /etc/machine-id /var/lib/dbus/machine-id
+: >/etc/machine-id
+touch /etc/homelab-template-sealed
+apt-get clean
+rm -rf /tmp/* /var/tmp/*
+sync
+echo "Power off this VM and convert it to a Pimox template."
--- a/bootstrap/provisioning/templates/preseed.cfg.tftpl
+++ b/bootstrap/provisioning/templates/preseed.cfg.tftpl
@ -0,0 +1,35 @@
+d-i debian-installer/locale string ${locale}
+d-i keyboard-configuration/xkb-keymap select ${keyboard}
+d-i netcfg/choose_interface select auto
+d-i netcfg/get_hostname string ${template_hostname}
+d-i netcfg/get_domain string ${template_domain}
+d-i hw-detect/load_firmware boolean true
+d-i mirror/country string manual
+d-i mirror/http/hostname string ${debian_mirror_host}
+d-i mirror/http/directory string ${debian_mirror_directory}
+d-i mirror/http/proxy string
+d-i passwd/root-login boolean false
+d-i passwd/user-fullname string ${template_user_full_name}
+d-i passwd/username string ${template_user}
+d-i passwd/user-password-crypted password ${template_user_password_hash}
+d-i user-setup/allow-password-weak boolean true
+d-i user-setup/encrypt-home boolean false
+d-i clock-setup/utc boolean true
+d-i time/zone string ${timezone}
+d-i partman-auto/disk string ${template_disk}
+d-i partman-auto/method string regular
+d-i partman-auto/choose_recipe select atomic
+d-i partman-partitioning/confirm_write_new_label boolean true
+d-i partman/choose_partition select finish
+d-i partman/confirm boolean true
+d-i partman/confirm_nooverwrite boolean true
+d-i apt-setup/non-free-firmware boolean true
+d-i apt-setup/services-select multiselect security, updates
+tasksel tasksel/first multiselect standard, ssh-server
+d-i pkgsel/include string ${template_package_list}
+d-i pkgsel/update-policy select none
+popularity-contest popularity-contest/participate boolean false
+d-i grub-installer/only_debian boolean true
+d-i grub-installer/bootdev string default
+d-i preseed/late_command string wget -O /target/usr/local/sbin/homelab-golden-node-prepare.sh ${provisioning_script_url}; chmod 0755 /target/usr/local/sbin/homelab-golden-node-prepare.sh; in-target /usr/local/sbin/homelab-golden-node-prepare.sh; wget -O /target/usr/local/sbin/homelab-prepare-template.sh ${prepare_template_script_url}; chmod 0755 /target/usr/local/sbin/homelab-prepare-template.sh
+d-i finish-install/reboot_in_progress note
--- a/bootstrap/provisioning/variables.tf
+++ b/bootstrap/provisioning/variables.tf
@ -0,0 +1,240 @@
+variable "provisioning_host" {
+  type    = string
+  default = "192.168.100.68"
+}
+
+variable "provisioning_user" {
+  type    = string
+  default = "jv"
+}
+
+variable "provisioning_ssh_key_path" {
+  type    = string
+  default = "/home/jv/.ssh/id_ed25519"
+}
+
+variable "provisioning_install_dir" {
+  type    = string
+  default = "/opt/homelab-provisioning"
+}
+
+variable "provisioning_interface" {
+  type = string
+}
+
+variable "proxy_dhcp_range" {
+  type    = string
+  default = "192.168.100.0,proxy"
+}
+
+variable "http_host" {
+  type    = string
+  default = "192.168.100.68"
+}
+
+variable "http_port" {
+  type    = number
+  default = 8088
+}
+
+variable "debian_suite" {
+  type    = string
+  default = "trixie"
+}
+
+variable "debian_mirror_host" {
+  type    = string
+  default = "deb.debian.org"
+}
+
+variable "debian_mirror_directory" {
+  type    = string
+  default = "/debian"
+}
+
+variable "debian_netboot_base_url" {
+  type    = string
+  default = ""
+}
+
+variable "pxe_boot_file" {
+  type    = string
+  default = "grubaa64.efi"
+}
+
+variable "template_hostname" {
+  type    = string
+  default = "homelab-arm64-template"
+}
+
+variable "clone_hostname_prefix" {
+  type    = string
+  default = "homelab-worker"
+}
+
+variable "template_domain" {
+  type    = string
+  default = "homelab.local"
+}
+
+variable "template_disk" {
+  type    = string
+  default = "/dev/vda"
+}
+
+variable "template_user" {
+  type    = string
+  default = "jv"
+}
+
+variable "template_user_full_name" {
+  type    = string
+  default = "Homelab Operator"
+}
+
+variable "template_user_password_hash" {
+  type    = string
+  default = "!"
+}
+
+variable "template_user_ssh_public_key_path" {
+  type    = string
+  default = "/home/jv/.ssh/id_ed25519.pub"
+}
+
+variable "template_user_ssh_authorized_keys" {
+  type    = list(string)
+  default = []
+}
+
+variable "locale" {
+  type    = string
+  default = "en_US.UTF-8"
+}
+
+variable "keyboard" {
+  type    = string
+  default = "us"
+}
+
+variable "timezone" {
+  type    = string
+  default = "UTC"
+}
+
+variable "kubernetes_minor_version" {
+  type    = string
+  default = "v1.33"
+}
+
+variable "kernel_cgroup_boot_options" {
+  type = list(string)
+  default = [
+    "systemd.unified_cgroup_hierarchy=1",
+    "cgroup_enable=memory",
+    "cgroup_memory=1",
+  ]
+}
+
+variable "registry_endpoint" {
+  type    = string
+  default = "192.168.100.68:30500"
+}
+
+variable "node_dns_servers" {
+  type = list(string)
+  default = [
+    "1.1.1.1",
+    "8.8.8.8",
+  ]
+}
+
+variable "additional_template_packages" {
+  type    = list(string)
+  default = []
+}
+
+variable "pimox_template_builder_enabled" {
+  type    = bool
+  default = false
+}
+
+variable "pimox_host" {
+  type    = string
+  default = "192.168.100.91"
+}
+
+variable "pimox_user" {
+  type    = string
+  default = "jv"
+}
+
+variable "pimox_ssh_key_path" {
+  type    = string
+  default = "/home/jv/.ssh/id_ed25519"
+}
+
+variable "pimox_template_vmid" {
+  type    = number
+  default = 9000
+}
+
+variable "pimox_template_name" {
+  type    = string
+  default = "debian13-arm64-k8s-template"
+}
+
+variable "pimox_template_cores" {
+  type    = number
+  default = 2
+}
+
+variable "pimox_template_memory" {
+  type    = number
+  default = 2048
+}
+
+variable "pimox_template_bridge" {
+  type    = string
+  default = "vmbr0"
+}
+
+variable "pimox_template_mac" {
+  type    = string
+  default = ""
+}
+
+variable "pimox_template_scsi0" {
+  type    = string
+  default = "local:15"
+}
+
+variable "pimox_template_efidisk0" {
+  type    = string
+  default = "local:1,efitype=4m,pre-enrolled-keys=1"
+}
+
+variable "pimox_template_replace_existing" {
+  type    = bool
+  default = false
+}
+
+variable "pimox_template_build_host" {
+  type    = string
+  default = ""
+}
+
+variable "pimox_template_build_user" {
+  type    = string
+  default = "jv"
+}
+
+variable "pimox_template_build_ssh_key_path" {
+  type    = string
+  default = "/home/jv/.ssh/id_ed25519"
+}
+
+variable "pimox_template_build_timeout" {
+  type    = string
+  default = "60m"
+}
--- a/lab.sh
+++ b/lab.sh
@ -527,7 +527,7 @@ install_gitea_runner() {
    local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}"
    local runner_user="${GITEA_RUNNER_USER:-jv}"
    local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}"
-    local missing_packages=""
+    local missing_packages=()

    require_debian_server "install-gitea-runner"

@ -546,12 +546,12 @@ install_gitea_runner() {

    for package in ca-certificates curl git nodejs python3; do
        if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
-            missing_packages="$missing_packages $package"
+            missing_packages+=("$package")
        fi
    done
-    if [[ -n "${missing_packages}" ]]; then
+    if [[ ${#missing_packages[@]} -gt 0 ]]; then
        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends ${missing_packages}
+        sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
    fi

    sudo curl -fsSL \