diff --git a/README.md b/README.md index ae1eb98..4391890 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ accidentally modify the cluster. 1. `bootstrap/provisioning` - prepares a Debian server as a PXE and preseed service for arm64 VMs - serves Debian 13 arm64 netboot assets through TFTP and HTTP + - exposes a PXE rescue menu and HTTP rescue page for manual repair paths - creates a golden image install path with Kubernetes, containerd, qemu-guest-agent, cloud-init, and storage client packages ready - is driven by `./jeannie up` when Pimox is reachable, without changing diff --git a/README.md.tmpl b/README.md.tmpl index eb17983..f94cdbd 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -54,6 +54,7 @@ accidentally modify the cluster. 1. `bootstrap/provisioning` - prepares a Debian server as a PXE and preseed service for arm64 VMs - serves Debian 13 arm64 netboot assets through TFTP and HTTP + - exposes a PXE rescue menu and HTTP rescue page for manual repair paths - creates a golden image install path with Kubernetes, containerd, qemu-guest-agent, cloud-init, and storage client packages ready - is driven by `./{{ main_script }} up` when Pimox is reachable, without changing diff --git a/apps/demos-static/public/homelab-catalog/index.html b/apps/demos-static/public/homelab-catalog/index.html index 3558ab0..1e41a68 100644 --- a/apps/demos-static/public/homelab-catalog/index.html +++ b/apps/demos-static/public/homelab-catalog/index.html @@ -152,6 +152,14 @@ not exposed bootstrap/provisioning/README.md + + PXE Rescue Toolkit + debian (control-plane, 192.168.100.73) + OpenTofu / bootstrap/provisioning + http://192.168.100.73:8088/rescue/ + not exposed + bootstrap/provisioning/README.md + Ollama debian (control-plane, 192.168.100.73) diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 18cb0d6..5c920c3 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -13,6 +13,7 @@ to edit Orange Pi host networking. - `nginx` for preseed, installer, and guest-prep scripts - Debian 13 arm64 netboot assets under `/opt/homelab-provisioning` - a preseed file for unattended Debian install +- a GRUB PXE rescue menu and HTTP rescue page at `/rescue/` - guest prep scripts that install Kubernetes tools, containerd, qemu guest agent, cloud-init, OpenEBS dependencies, cgroup prerequisites, swap disablement, and the local registry trust path - Kubernetes tools from the pinned minor in `TF_VAR_kubernetes_minor_version`, defaulting to `v1.36` - `containerd.io` from Docker's Debian repository so worker runtimes use the @@ -60,7 +61,25 @@ generates a unique fallback name using `TF_VAR_clone_hostname_prefix`. Create an arm64 VM in Pimox with UEFI firmware, a virtio disk, and a NIC on the same LAN as the Debian provisioning host. Put network boot first. -PXE should load `grubaa64.efi`, boot the Debian installer, fetch the preseed from `http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg`, and install the golden image. +PXE should load `grubaa64.efi` and display a GRUB menu. The default entry keeps +the automated template flow intact: it boots the Debian installer, fetches the +preseed from `http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg`, and +installs the golden image. + +The same menu also includes rescue-oriented entries: + +- Debian 13 arm64 rescue shell +- Debian 13 arm64 expert/manual install +- Debian 13 arm64 automated install, verbose +- UEFI firmware settings +- Exit PXE and continue local boot +- Reboot + +The Debian provisioning host also publishes a rescue landing page: + +```bash +curl http://192.168.100.73:8088/rescue/ +``` If your Pimox firmware needs a different Debian arm64 EFI loader, override `TF_VAR_pxe_boot_file`. diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index fc2d23d..799a2a2 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -64,9 +64,17 @@ locals { }) grub_cfg = templatefile("${path.module}/templates/grub.cfg.tftpl", { - preseed_url = local.preseed_url - template_hostname = var.template_hostname - template_domain = var.template_domain + preseed_url = local.preseed_url + template_hostname = var.template_hostname + template_domain = var.template_domain + pxe_default_menu_entry = tostring(var.pxe_default_menu_entry) + pxe_menu_timeout = tostring(var.pxe_menu_timeout) + }) + + rescue_index = templatefile("${path.module}/templates/rescue-index.html.tftpl", { + http_base_url = local.provisioning_http_base_url + preseed_url = local.preseed_url + tftp_root = local.tftp_root }) preseed_cfg = templatefile("${path.module}/templates/preseed.cfg.tftpl", { @@ -106,6 +114,7 @@ locals { local.dnsmasq_conf, local.nginx_conf, local.grub_cfg, + local.rescue_index, local.preseed_cfg, local.golden_node_prepare, local.prepare_template, @@ -421,7 +430,7 @@ resource "null_resource" "provisioning_host" { http_port = tostring(var.http_port) config_hash = local.config_hash provisioning_layer = "1" - provisioner_version = "3" + provisioner_version = "4" } connection { @@ -453,6 +462,11 @@ resource "null_resource" "provisioning_host" { destination = "/tmp/homelab-provisioning/grub.cfg" } + provisioner "file" { + content = local.rescue_index + destination = "/tmp/homelab-provisioning/rescue-index.html" + } + provisioner "file" { content = local.preseed_cfg destination = "/tmp/homelab-provisioning/preseed.cfg" @@ -503,17 +517,20 @@ sudo install -d -m 0755 \ "$tftp_root/grub" \ "$http_root" \ "$http_root/preseed" \ + "$http_root/rescue" \ "$http_root/scripts" \ "$http_root/debian-installer/arm64" sudo cp "$tmp_dir/grub.cfg" "$tftp_root/grub/grub.cfg" sudo cp "$tmp_dir/grub.cfg" "$tftp_root/debian-installer/arm64/grub/grub.cfg" +sudo cp "$tmp_dir/rescue-index.html" "$http_root/rescue/index.html" sudo cp "$tmp_dir/preseed.cfg" "$http_root/preseed/debian13-arm64-worker.cfg" sudo cp "$tmp_dir/golden-node-prepare.sh" "$http_root/scripts/golden-node-prepare.sh" sudo cp "$tmp_dir/prepare-template.sh" "$http_root/scripts/prepare-template.sh" sudo chmod 0644 \ "$tftp_root/grub/grub.cfg" \ "$tftp_root/debian-installer/arm64/grub/grub.cfg" \ + "$http_root/rescue/index.html" \ "$http_root/preseed/debian13-arm64-worker.cfg" sudo chmod 0755 "$http_root/scripts/golden-node-prepare.sh" "$http_root/scripts/prepare-template.sh" diff --git a/bootstrap/provisioning/outputs.tf b/bootstrap/provisioning/outputs.tf index 9459f22..f6ec291 100644 --- a/bootstrap/provisioning/outputs.tf +++ b/bootstrap/provisioning/outputs.tf @@ -6,6 +6,10 @@ output "preseed_url" { value = local.preseed_url } +output "pxe_rescue_url" { + value = "${local.provisioning_http_base_url}/rescue/" +} + output "pxe_boot_file" { value = var.pxe_boot_file } diff --git a/bootstrap/provisioning/templates/grub.cfg.tftpl b/bootstrap/provisioning/templates/grub.cfg.tftpl index 8222e9e..fb05c30 100644 --- a/bootstrap/provisioning/templates/grub.cfg.tftpl +++ b/bootstrap/provisioning/templates/grub.cfg.tftpl @@ -1,7 +1,34 @@ -set default=0 -set timeout=5 +set default=${pxe_default_menu_entry} +set timeout=${pxe_menu_timeout} -menuentry 'Debian 13 arm64 homelab worker template' { +menuentry 'Debian 13 arm64 automated homelab worker template' --id debian-auto-template { linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} --- quiet initrd /debian-installer/arm64/initrd.gz } + +menuentry 'Debian 13 arm64 rescue shell' --id debian-rescue { + linux /debian-installer/arm64/linux rescue/enable=true priority=low interface=auto hostname=rescue-${template_hostname} domain=${template_domain} --- + initrd /debian-installer/arm64/initrd.gz +} + +menuentry 'Debian 13 arm64 expert/manual install' --id debian-expert { + linux /debian-installer/arm64/linux priority=low interface=auto hostname=manual-${template_hostname} domain=${template_domain} --- + initrd /debian-installer/arm64/initrd.gz +} + +menuentry 'Debian 13 arm64 automated install, verbose' --id debian-auto-template-verbose { + linux /debian-installer/arm64/linux auto=true priority=critical url=${preseed_url} interface=auto hostname=${template_hostname} domain=${template_domain} --- + initrd /debian-installer/arm64/initrd.gz +} + +menuentry 'UEFI firmware settings' --id uefi-firmware { + fwsetup +} + +menuentry 'Exit PXE and continue local boot' --id localboot { + exit +} + +menuentry 'Reboot' --id reboot { + reboot +} diff --git a/bootstrap/provisioning/templates/rescue-index.html.tftpl b/bootstrap/provisioning/templates/rescue-index.html.tftpl new file mode 100644 index 0000000..7c121b8 --- /dev/null +++ b/bootstrap/provisioning/templates/rescue-index.html.tftpl @@ -0,0 +1,75 @@ + + + + + + Homelab PXE Rescue Toolkit + + + +
+

Homelab PXE Rescue Toolkit

+

This page is served by the Debian provisioning host at ${http_base_url}. The same host serves TFTP boot assets from ${tftp_root}.

+ +
+

PXE Menu Entries

+ +
+ +
+

Useful URLs

+ +
+ +
+

Notes

+

The default menu entry remains the automated install so the Pimox template builder can still boot unattended. Use the firmware console to select rescue or manual install when recovering a host.

+
+
+ + diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index fb0b1f9..b09fc0b 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -62,6 +62,16 @@ variable "pxe_boot_file" { default = "grubaa64.efi" } +variable "pxe_menu_timeout" { + type = number + default = 10 +} + +variable "pxe_default_menu_entry" { + type = number + default = 0 +} + variable "template_hostname" { type = string default = "homelab-arm64-template" diff --git a/docs/service-catalog.md b/docs/service-catalog.md index 5ca026b..f607460 100644 --- a/docs/service-catalog.md +++ b/docs/service-catalog.md @@ -29,6 +29,7 @@ Update `homelab.yml`, then run `scripts/render-service-catalog`. | Unbound | rpi4 (dns-edge-worker, 192.168.100.89) | Docker Compose / jeannie rpi-services | `192.168.100.89:53` | | `infra/rpi-services/README.md` | | Uptime Kuma | rpi4 (dns-edge-worker, 192.168.100.89) | Docker Compose / jeannie rpi-services | [local](http://192.168.100.89:3001/) | | `infra/rpi-services/README.md` | | Provisioning PXE/HTTP | debian (control-plane, 192.168.100.73) | OpenTofu / bootstrap/provisioning | [local](http://192.168.100.73:8088/) | | `bootstrap/provisioning/README.md` | +| PXE Rescue Toolkit | debian (control-plane, 192.168.100.73) | OpenTofu / bootstrap/provisioning | [local](http://192.168.100.73:8088/rescue/) | | `bootstrap/provisioning/README.md` | | Ollama | debian (control-plane, 192.168.100.73) | Local Debian service | [local](http://192.168.100.73:11434) | | `apps/website/ollama/README.md` | | OCI Edge | oci_edge | OpenTofu / bootstrap/edge | `/opt/homelab-edge` | [public](https://lab2025.duckdns.org) | `docs/runbooks/edge-failures.md` | | Heimdall | Kubernetes | Argo CD / apps/heimdall | [local](http://192.168.100.240/) | [public](https://heimdall.lab2025.duckdns.org/) | `apps/heimdall/` | diff --git a/homelab.yml b/homelab.yml index 1e9eee5..ceca651 100644 --- a/homelab.yml +++ b/homelab.yml @@ -81,6 +81,7 @@ services: host: debian http_port: 8088 preseed_url: http://192.168.100.73:8088/preseed/debian13-arm64-worker.cfg + rescue_url: http://192.168.100.73:8088/rescue/ ollama: host: debian url: http://192.168.100.73:11434 diff --git a/scripts/render-service-catalog b/scripts/render-service-catalog index 13cfed1..95db758 100755 --- a/scripts/render-service-catalog +++ b/scripts/render-service-catalog @@ -182,6 +182,14 @@ def service_rows(values: dict[str, object]) -> list[dict[str, str]]: "public": "", "docs": "bootstrap/provisioning/README.md", }, + { + "name": "PXE Rescue Toolkit", + "host": "debian", + "managed_by": "OpenTofu / bootstrap/provisioning", + "local": value(values, "services.provisioning.rescue_url"), + "public": "", + "docs": "bootstrap/provisioning/README.md", + }, { "name": "Ollama", "host": "debian",