Expand PXE provisioning rescue menu

README.md

@@ -54,6 +54,7 @@ accidentally modify the cluster.
 1. `bootstrap/provisioning`
    - prepares a Debian server as a PXE and preseed service for arm64 VMs
    - serves Debian 13 arm64 netboot assets through TFTP and HTTP
+   - exposes a PXE rescue menu and HTTP rescue page for manual repair paths
    - creates a golden image install path with Kubernetes, containerd,
      qemu-guest-agent, cloud-init, and storage client packages ready
    - is driven by `./jeannie up` when Pimox is reachable, without changing

README.md.tmpl

@@ -54,6 +54,7 @@ accidentally modify the cluster.
 1. `bootstrap/provisioning`
    - prepares a Debian server as a PXE and preseed service for arm64 VMs
    - serves Debian 13 arm64 netboot assets through TFTP and HTTP
+   - exposes a PXE rescue menu and HTTP rescue page for manual repair paths
    - creates a golden image install path with Kubernetes, containerd,
      qemu-guest-agent, cloud-init, and storage client packages ready
    - is driven by `./{{ main_script }} up` when Pimox is reachable, without changing

apps/demos-static/public/homelab-catalog/index.html

@@ -152,6 +152,14 @@
             <td><span class="muted">not exposed</span></td>
             <td><code>bootstrap/provisioning/README.md</code></td>
           </tr>
+          <tr>
+            <td>PXE Rescue Toolkit</td>
+            <td>debian (control-plane, 192.168.100.73)</td>
+            <td>OpenTofu / bootstrap/provisioning</td>
+            <td><a href="http://192.168.100.73:8088/rescue/">http://192.168.100.73:8088/rescue/</a></td>
+            <td><span class="muted">not exposed</span></td>
+            <td><code>bootstrap/provisioning/README.md</code></td>
+          </tr>
           <tr>
             <td>Ollama</td>
             <td>debian (control-plane, 192.168.100.73)</td>

bootstrap/provisioning/README.md

@@ -13,6 +13,7 @@ to edit Orange Pi host networking.
 - `nginx` for preseed, installer, and guest-prep scripts
 - Debian 13 arm64 netboot assets under `/opt/homelab-provisioning`
 - a preseed file for unattended Debian install
+- a GRUB PXE rescue menu and HTTP rescue page at `/rescue/`
 - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path
 - Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36`
 - `containerd.io` from Docker's Debian repository so worker runtimes use the
@@ -60,7 +61,25 @@ generates a unique fallback name using `TF_VAR_clone_hostname_prefix`.
 
 Create an arm64 VM in Pimox with UEFI firmware, a virtio disk, and a NIC on the same LAN as the Debian provisioning host. Put network boot first.
 
-PXE should load `grubaa64.efi`, boot the Debian installer, fetch the preseed from `http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg`, and install the golden image.
+PXE should load `grubaa64.efi` and display a GRUB menu. The default entry keeps
+the automated template flow intact: it boots the Debian installer, fetches the
+preseed from `http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg`, and
+installs the golden image.
+
+The same menu also includes rescue-oriented entries:
+
+- Debian 13 arm64 rescue shell
+- Debian 13 arm64 expert/manual install
+- Debian 13 arm64 automated install, verbose
+- UEFI firmware settings
+- Exit PXE and continue local boot
+- Reboot
+
+The Debian provisioning host also publishes a rescue landing page:
+
+```bash
+curl http://192.168.100.73:8088/rescue/
+```
 
 If your Pimox firmware needs a different Debian arm64 EFI loader, override `TF_VAR_pxe_boot_file`.
 

bootstrap/provisioning/main.tf

@@ -64,9 +64,17 @@ locals {
   })
 
   grub_cfg = templatefile("${path.module}/templates/grub.cfg.tftpl", {
-    preseed_url       = local.preseed_url
+    preseed_url            = local.preseed_url
-    template_hostname = var.template_hostname
+    template_hostname      = var.template_hostname
-    template_domain   = var.template_domain
+    template_domain        = var.template_domain
+    pxe_default_menu_entry = tostring(var.pxe_default_menu_entry)
+    pxe_menu_timeout       = tostring(var.pxe_menu_timeout)
+  })
+
+  rescue_index = templatefile("${path.module}/templates/rescue-index.html.tftpl", {
+    http_base_url = local.provisioning_http_base_url
+    preseed_url   = local.preseed_url
+    tftp_root     = local.tftp_root
   })
 
   preseed_cfg = templatefile("${path.module}/templates/preseed.cfg.tftpl", {
@@ -106,6 +114,7 @@ locals {
     local.dnsmasq_conf,
     local.nginx_conf,
     local.grub_cfg,
+    local.rescue_index,
     local.preseed_cfg,
     local.golden_node_prepare,
     local.prepare_template,
@@ -421,7 +430,7 @@ resource "null_resource" "provisioning_host" {
     http_port           = tostring(var.http_port)
     config_hash         = local.config_hash
     provisioning_layer  = "1"
-    provisioner_version = "3"
+    provisioner_version = "4"
   }
 
   connection {
@@ -453,6 +462,11 @@ resource "null_resource" "provisioning_host" {
     destination = "/tmp/homelab-provisioning/grub.cfg"
   }
 
+  provisioner "file" {
+    content     = local.rescue_index
+    destination = "/tmp/homelab-provisioning/rescue-index.html"
+  }
+
   provisioner "file" {
     content     = local.preseed_cfg
     destination = "/tmp/homelab-provisioning/preseed.cfg"
@@ -503,17 +517,20 @@ sudo install -d -m 0755 \
   "$tftp_root/grub" \
   "$http_root" \
   "$http_root/preseed" \
+  "$http_root/rescue" \
   "$http_root/scripts" \
   "$http_root/debian-installer/arm64"
 
 sudo cp "$tmp_dir/grub.cfg" "$tftp_root/grub/grub.cfg"
 sudo cp "$tmp_dir/grub.cfg" "$tftp_root/debian-installer/arm64/grub/grub.cfg"
+sudo cp "$tmp_dir/rescue-index.html" "$http_root/rescue/index.html"
 sudo cp "$tmp_dir/preseed.cfg" "$http_root/preseed/debian13-arm64-worker.cfg"
 sudo cp "$tmp_dir/golden-node-prepare.sh" "$http_root/scripts/golden-node-prepare.sh"
 sudo cp "$tmp_dir/prepare-template.sh" "$http_root/scripts/prepare-template.sh"
 sudo chmod 0644 \
   "$tftp_root/grub/grub.cfg" \
   "$tftp_root/debian-installer/arm64/grub/grub.cfg" \
+  "$http_root/rescue/index.html" \
   "$http_root/preseed/debian13-arm64-worker.cfg"
 sudo chmod 0755 "$http_root/scripts/golden-node-prepare.sh" "$http_root/scripts/prepare-template.sh"
 

bootstrap/provisioning/outputs.tf

@@ -6,6 +6,10 @@ output "preseed_url" {
   value = local.preseed_url
 }
 
+output "pxe_rescue_url" {
+  value = "${local.provisioning_http_base_url}/rescue/"
+}
+
 output "pxe_boot_file" {
   value = var.pxe_boot_file
 }

bootstrap/provisioning/templates/grub.cfg.tftpl

@@ -1,7 +1,34 @@
-set default=0
+set default=${pxe_default_menu_entry}
-set timeout=5
+set timeout=${pxe_menu_timeout}
 
-menuentry 'Debian 13 arm64 homelab worker template' {
+menuentry 'Debian 13 arm64 automated homelab worker template' --id debian-auto-template {
     linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} --- quiet
     initrd /debian-installer/arm64/initrd.gz
 }
+
+menuentry 'Debian 13 arm64 rescue shell' --id debian-rescue {
+    linux /debian-installer/arm64/linux rescue/enable=true priority=low interface=auto hostname=rescue-${template_hostname} domain=${template_domain} ---
+    initrd /debian-installer/arm64/initrd.gz
+}
+
+menuentry 'Debian 13 arm64 expert/manual install' --id debian-expert {
+    linux /debian-installer/arm64/linux priority=low interface=auto hostname=manual-${template_hostname} domain=${template_domain} ---
+    initrd /debian-installer/arm64/initrd.gz
+}
+
+menuentry 'Debian 13 arm64 automated install, verbose' --id debian-auto-template-verbose {
+    linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} ---
+    initrd /debian-installer/arm64/initrd.gz
+}
+
+menuentry 'UEFI firmware settings' --id uefi-firmware {
+    fwsetup
+}
+
+menuentry 'Exit PXE and continue local boot' --id localboot {
+    exit
+}
+
+menuentry 'Reboot' --id reboot {
+    reboot
+}

bootstrap/provisioning/templates/rescue-index.html.tftpl

@@ -0,0 +1,75 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Homelab PXE Rescue Toolkit</title>
+  <style>
+    body {
+      margin: 0;
+      background: #0b1020;
+      color: #e5edf7;
+      font-family: Arial, sans-serif;
+      line-height: 1.55;
+    }
+    main {
+      width: min(980px, calc(100% - 32px));
+      margin: 0 auto;
+      padding: 32px 0 48px;
+    }
+    h1, h2 {
+      color: #ffffff;
+    }
+    .panel {
+      background: #11182c;
+      border: 1px solid #263551;
+      border-radius: 8px;
+      margin: 18px 0;
+      padding: 18px;
+    }
+    code {
+      background: #050816;
+      border: 1px solid #263551;
+      border-radius: 5px;
+      padding: 2px 6px;
+    }
+    a {
+      color: #7dd3fc;
+    }
+  </style>
+</head>
+<body>
+  <main>
+    <h1>Homelab PXE Rescue Toolkit</h1>
+    <p>This page is served by the Debian provisioning host at <code>${http_base_url}</code>. The same host serves TFTP boot assets from <code>${tftp_root}</code>.</p>
+
+    <section class="panel">
+      <h2>PXE Menu Entries</h2>
+      <ul>
+        <li><strong>Automated homelab worker template:</strong> unattended Debian install using <a href="${preseed_url}">preseed</a>.</li>
+        <li><strong>Rescue shell:</strong> Debian Installer rescue mode for mounting and repairing an installed system.</li>
+        <li><strong>Expert/manual install:</strong> interactive Debian Installer path for one-off machines.</li>
+        <li><strong>Automated install, verbose:</strong> same preseeded install without quiet boot.</li>
+        <li><strong>UEFI firmware settings:</strong> returns into firmware setup when supported.</li>
+        <li><strong>Exit PXE and continue local boot:</strong> leaves the GRUB netboot path.</li>
+      </ul>
+    </section>
+
+    <section class="panel">
+      <h2>Useful URLs</h2>
+      <ul>
+        <li><a href="/preseed/debian13-arm64-worker.cfg">/preseed/debian13-arm64-worker.cfg</a></li>
+        <li><a href="/scripts/golden-node-prepare.sh">/scripts/golden-node-prepare.sh</a></li>
+        <li><a href="/scripts/prepare-template.sh">/scripts/prepare-template.sh</a></li>
+        <li><a href="/debian-installer/arm64/linux">/debian-installer/arm64/linux</a></li>
+        <li><a href="/debian-installer/arm64/initrd.gz">/debian-installer/arm64/initrd.gz</a></li>
+      </ul>
+    </section>
+
+    <section class="panel">
+      <h2>Notes</h2>
+      <p>The default menu entry remains the automated install so the Pimox template builder can still boot unattended. Use the firmware console to select rescue or manual install when recovering a host.</p>
+    </section>
+  </main>
+</body>
+</html>

bootstrap/provisioning/variables.tf

@@ -62,6 +62,16 @@ variable "pxe_boot_file" {
   default = "grubaa64.efi"
 }
 
+variable "pxe_menu_timeout" {
+  type    = number
+  default = 10
+}
+
+variable "pxe_default_menu_entry" {
+  type    = number
+  default = 0
+}
+
 variable "template_hostname" {
   type    = string
   default = "homelab-arm64-template"

docs/service-catalog.md

@@ -29,6 +29,7 @@ Update `homelab.yml`, then run `scripts/render-service-catalog`.
 | Unbound | rpi4 (dns-edge-worker, 192.168.100.89) | Docker Compose / jeannie rpi-services | `192.168.100.89:53` |  | `infra/rpi-services/README.md` |
 | Uptime Kuma | rpi4 (dns-edge-worker, 192.168.100.89) | Docker Compose / jeannie rpi-services | [local](http://192.168.100.89:3001/) |  | `infra/rpi-services/README.md` |
 | Provisioning PXE/HTTP | debian (control-plane, 192.168.100.73) | OpenTofu / bootstrap/provisioning | [local](http://192.168.100.73:8088/) |  | `bootstrap/provisioning/README.md` |
+| PXE Rescue Toolkit | debian (control-plane, 192.168.100.73) | OpenTofu / bootstrap/provisioning | [local](http://192.168.100.73:8088/rescue/) |  | `bootstrap/provisioning/README.md` |
 | Ollama | debian (control-plane, 192.168.100.73) | Local Debian service | [local](http://192.168.100.73:11434) |  | `apps/website/ollama/README.md` |
 | OCI Edge | oci_edge | OpenTofu / bootstrap/edge | `/opt/homelab-edge` | [public](https://lab2025.duckdns.org) | `docs/runbooks/edge-failures.md` |
 | Heimdall | Kubernetes | Argo CD / apps/heimdall | [local](http://192.168.100.240/) | [public](https://heimdall.lab2025.duckdns.org/) | `apps/heimdall/` |

homelab.yml

@@ -81,6 +81,7 @@ services:
     host: debian
     http_port: 8088
     preseed_url: http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg
+    rescue_url: http://192.168.100.73:8088/rescue/
   ollama:
     host: debian
     url: http://192.168.100.73:11434

scripts/render-service-catalog

@@ -182,6 +182,14 @@ def service_rows(values: dict[str, object]) -> list[dict[str, str]]:
             "public": "",
             "docs": "bootstrap/provisioning/README.md",
         },
+        {
+            "name": "PXE Rescue Toolkit",
+            "host": "debian",
+            "managed_by": "OpenTofu / bootstrap/provisioning",
+            "local": value(values, "services.provisioning.rescue_url"),
+            "public": "",
+            "docs": "bootstrap/provisioning/README.md",
+        },
         {
             "name": "Ollama",
             "host": "debian",