Add Jeannie man page

README.md

@@ -237,6 +237,11 @@ running the full pipeline:
 ./jeannie fix-debian-docker-root
 ```
 
+When `./jeannie up` sees an existing tracked kubeadm control plane but
+the Kubernetes API is down, it starts the saved runtime before applying cluster
+changes. Use `./jeannie rebuild-cluster` only when you intentionally
+want to destroy and recreate the cluster.
+
 ## Validation
 
 Useful checks after a rebuild:

README.md.tmpl

@@ -237,6 +237,11 @@ running the full pipeline:
 ./{{ main_script }} fix-debian-docker-root
 ```
 
+When `./{{ main_script }} up` sees an existing tracked kubeadm control plane but
+the Kubernetes API is down, it starts the saved runtime before applying cluster
+changes. Use `./{{ main_script }} rebuild-cluster` only when you intentionally
+want to destroy and recreate the cluster.
+
 ## Validation
 
 Useful checks after a rebuild:

bootstrap/cluster/main.tf

@@ -381,19 +381,6 @@ EOT
   }
 }
 
-data "external" "kubeadm_join_command" {
-  depends_on = [null_resource.kubeadm_control_plane]
-  program = [
-    "bash",
-    "-lc",
-    <<EOT
-set -euo pipefail
-cmd="$(sudo kubeadm token create --print-join-command)"
-printf '{"cmd":"%s"}\n' "$(printf '%s' "$cmd" | sed 's/\\/\\\\/g; s/"/\\"/g')"
-EOT
-  ]
-}
-
 resource "null_resource" "worker_nodes_required" {
   triggers = {
     worker_count = tostring(length(var.worker_nodes))
@@ -411,7 +398,7 @@ resource "null_resource" "kubeadm_worker" {
   for_each = var.worker_nodes
 
   depends_on = [
-    data.external.kubeadm_join_command,
+    null_resource.kubeadm_control_plane,
     null_resource.worker_nodes_required,
   ]
 
@@ -892,24 +879,47 @@ fi
 if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
   reset_worker_join_state
 fi
+EOT
+    ]
+  }
 
-if [ ! -f /etc/kubernetes/kubelet.conf ]; then
-  sudo systemctl stop kubelet 2>/dev/null || true
-  if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then
+  provisioner "local-exec" {
+    command = <<EOT
+set -euo pipefail
+
+ssh_target='${self.triggers.user}@${self.triggers.host}'
+ssh_key='${self.triggers.ssh_key_path}'
+node_name='${self.triggers.node_name}'
+
+if ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+  'test -f /etc/kubernetes/kubelet.conf && test -f /var/lib/kubelet/config.yaml'; then
+  exit 0
+fi
+
+join_cmd="$(sudo kubeadm token create --print-join-command)"
+remote_join_script="$(mktemp)"
+trap 'rm -f "$remote_join_script"' EXIT
+
+cat >"$remote_join_script" <<JOIN_EOT
+set -eu
+sudo systemctl stop kubelet 2>/dev/null || true
+if ! sudo $join_cmd --node-name=$node_name; then
   sudo systemctl status kubelet --no-pager -l || true
   sudo journalctl -u kubelet --no-pager -n 160 || true
   exit 1
-  fi
-  if [ ! -f /etc/kubernetes/kubelet.conf ] || [ ! -f /var/lib/kubelet/config.yaml ]; then
+fi
+if [ ! -f /etc/kubernetes/kubelet.conf ] || [ ! -f /var/lib/kubelet/config.yaml ]; then
   echo "kubeadm join completed without creating kubelet configuration" >&2
   sudo ls -la /etc/kubernetes /var/lib/kubelet 2>/dev/null || true
   sudo systemctl status kubelet containerd --no-pager -l || true
   sudo journalctl -u kubelet --no-pager -n 160 || true
   exit 1
-  fi
 fi
+JOIN_EOT
+
+ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+  'bash -s' <"$remote_join_script"
 EOT
-    ]
   }
 }
 

bootstrap/edge/main.tf

@@ -46,6 +46,7 @@ resource "null_resource" "edge_services" {
     host                = var.edge_host
     user                = var.edge_user
     install_dir         = var.edge_install_dir
+    install_docker      = tostring(var.edge_install_docker)
     server_name         = var.server_name
     server_names        = join(" ", local.server_names)
     certbot_domain_args = join(" ", [for name in local.server_names : "-d ${name}"])
@@ -105,6 +106,7 @@ resource "null_resource" "edge_services" {
 set -eu
 
 install_dir="${self.triggers.install_dir}"
+install_docker="${self.triggers.install_docker}"
 server_name="${self.triggers.server_name}"
 server_names="${self.triggers.server_names}"
 certbot_domain_args="${self.triggers.certbot_domain_args}"
@@ -144,7 +146,12 @@ if ! curl -sS --connect-timeout 10 "http://$gitea_backend_host:$gitea_backend_po
 fi
 
 if ! command -v docker >/dev/null 2>&1; then
+  if [ "$install_docker" = "true" ]; then
     curl -fsSL https://get.docker.com | sudo sh
+  else
+    echo "Docker is not installed on the edge host. Install Docker first, or set edge_install_docker=true to allow get.docker.com installation." >&2
+    exit 1
+  fi
 fi
 
 if ! sudo docker compose version >/dev/null 2>&1; then

bootstrap/edge/variables.tf

@@ -18,6 +18,11 @@ variable "edge_install_dir" {
   default = "/opt/homelab-edge"
 }
 
+variable "edge_install_docker" {
+  type    = bool
+  default = false
+}
+
 variable "server_name" {
   type    = string
   default = "lab2025.duckdns.org"
@@ -82,5 +87,11 @@ variable "haproxy_stats_user" {
 
 variable "haproxy_stats_password" {
   type      = string
-  default = "adminpassword"
+  sensitive = true
+  default   = ""
+
+  validation {
+    condition     = length(var.haproxy_stats_password) >= 12 && var.haproxy_stats_password != "adminpassword"
+    error_message = "haproxy_stats_password must be set explicitly to a non-default value with at least 12 characters."
+  }
 }

docs/jeannie.1.md

@@ -0,0 +1,289 @@
+# JEANNIE(1)
+
+## NAME
+
+jeannie - operate the personal homelab control pipeline
+
+## SYNOPSIS
+
+```sh
+./jeannie COMMAND [ARG...]
+```
+
+## DESCRIPTION
+
+`jeannie` is the main homelab operator script. It deploys and checks the Debian
+control host, Gitea, Raspberry Pi services, Pimox workers, Kubernetes platform
+components, applications, public edge routing, secrets tooling, and local AI
+helpers.
+
+Run mutating infrastructure commands from the Debian homelab host. Commands that
+can destroy or rewrite state have explicit confirmation environment variables.
+The canonical script name is `metadata.main_script` in `homelab.yml`.
+
+## COMMANDS
+
+### Core Lifecycle
+
+`up`
+: Deploy the full homelab pipeline. Runs early preflight, deploys Gitea and RPi
+services, runs full preflight and pre-apply doctor checks, then applies Pimox,
+OpenWrt, cluster, platform, apps, and edge stacks.
+
+`plan [all|provisioning|cluster|platform|apps|edge]`
+: Run OpenTofu plans without applying changes. The default target is `all`.
+
+`rebuild-cluster`
+: Back up current local OpenTofu state, destroy cluster state through the guarded
+nuke path, rebuild Pimox workers, recreate the cluster, and redeploy platform,
+apps, and edge.
+
+`stop-cluster`
+: Stop the Kubernetes control-plane runtime and worker VMs without destroying
+cluster state.
+
+`start-cluster`
+: Start the Kubernetes control-plane runtime and desired worker VMs.
+
+`status`
+: Print a compact health report for the configured homelab.
+
+`nuke`
+: Destroy Kubernetes state and Pimox worker VMs. Requires
+`LAB_CONFIRM_NUKE=homelab`.
+
+### Deploy Targets
+
+`apps`
+: Deploy platform applications and build or publish application images when
+needed.
+
+`deploy-gitea`
+: Deploy the external Gitea Docker Compose service on the Debian host.
+
+`rpi-services`
+: Deploy Pi-hole, Unbound, and Uptime Kuma on the Raspberry Pi.
+
+`bootstrap-gitea-repo`
+: Ensure the configured Gitea repository, branch, remotes, and host SSH key are
+ready.
+
+`openwrt`
+: Apply the OpenWrt VM stack.
+
+### Gitea Operations
+
+`backup-gitea`
+: Run a Gitea backup on the Debian host.
+
+`drill-gitea-restore`
+: Run the Gitea restore drill workflow.
+
+`install-gitea-runner [TOKEN]`
+: Install or repair the Gitea Actions runner. If `TOKEN` is omitted, the command
+uses the configured token source.
+
+### Checks And Doctors
+
+`preflight`
+: Run homelab preflight checks. This verifies inventory-derived host settings,
+Docker roots, Tailscale IPs, Gitea reachability, Pimox storage, and OCI edge SSH.
+
+`doctor-preapply`
+: Run blocking readiness checks used by `up` before high-blast-radius applies.
+It checks Debian free disk, Docker and containerd, Raspberry Pi Docker
+writability, Pimox storage, OCI edge disk, and Pi-hole DNS.
+
+`doctor-versions`
+: Check Kubernetes component version alignment.
+
+`doctor-edge`
+: Diagnose public edge routing, Traefik reachability, and Gitea edge exposure.
+
+`doctor-gitea`
+: Diagnose Debian Gitea container, local access, edge access, and repository
+health.
+
+`doctor-rpi`
+: Diagnose Raspberry Pi services, Pi-hole DNS, Unbound DNS, fallback DNS,
+Uptime Kuma, and Docker root state.
+
+`doctor-cluster`
+: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, and app health.
+
+`inventory-check`
+: Validate `homelab.yml` with a real YAML parser when available.
+
+`secrets-check`
+: Validate SOPS and age configuration plus committed encrypted secret file
+structure.
+
+`tailnet-policy-check`
+: Validate the repo-managed Tailscale ACL policy.
+
+`ai-check`
+: Check local AI helper prerequisites, knowledge index, and Ollama availability
+when the backstage helper is enabled.
+
+### Maintenance
+
+`state-backup`
+: Create a local tarball backup of OpenTofu state and generated homelab state
+files.
+
+`fix-debian-docker-root`
+: Repair or migrate the Debian Docker root according to inventory settings.
+
+`move-prometheus-stack-workers`
+: Move Prometheus stack workloads off the control plane. Requires
+`LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true` because Prometheus PVC data is deleted.
+
+`secrets-init`
+: Initialize local age and SOPS configuration on the Debian host.
+
+`ai-index`
+: Build the local homelab knowledge index for the backstage AI helper.
+
+`ollama-setup`
+: Install or configure Ollama on the Debian host and pull the configured local
+model.
+
+`website-translation-model`
+: Prepare the website translation model assets.
+
+`website-ollama-listen`
+: Configure the Ollama service listen address for website use.
+
+## COMMON ENVIRONMENT
+
+`HOMELAB_INVENTORY_FILE`
+: Path to the canonical non-secret inventory. Defaults to `./homelab.yml`.
+
+`HOMELAB_STATE_DIR`
+: Local runtime state directory. Defaults to
+`${XDG_DATA_HOME:-$HOME/.local/share}/homelab`.
+
+`JEANNIE_LOG_DIR`
+: Directory for command logs. Defaults to `$HOMELAB_STATE_DIR/logs`.
+
+`JEANNIE_VERBOSE=true`
+: Print full step output as it runs. Without this, noisy commands are compact and
+full output is written to the log file.
+
+`LAB_AUTO_APPROVE=false`
+: Disable automatic OpenTofu approval for apply paths that support confirmation.
+
+`LAB_PIMOX_WORKER_COUNT`
+: Desired Pimox worker count when not supplied by generated cluster topology
+state.
+
+`LAB_SKIP_PREFLIGHT=true`
+: Skip preflight checks. Use only for targeted troubleshooting.
+
+`LAB_PREFLIGHT_ROOT_MIN_FREE_GIB`
+: Minimum Debian `/` free space for `doctor-preapply`. Default is `10`.
+
+`LAB_PREFLIGHT_DATA_MIN_FREE_GIB`
+: Minimum Debian `/data` free space for `doctor-preapply`. Default is `20`.
+
+`LAB_PREFLIGHT_EDGE_MIN_FREE_GIB`
+: Minimum OCI edge `/` free space for `doctor-preapply`. Default is `2`.
+
+`LAB_GITEA_INSTALL_DOCKER=true`
+: Allow `deploy-gitea` to install Docker through `get.docker.com` when Docker is
+missing. Default is to fail and ask for host bootstrap.
+
+`LAB_RPI_INSTALL_DOCKER=true`
+: Allow `rpi-services` to install Docker through `get.docker.com` on the
+Raspberry Pi when Docker is missing. Default is to fail.
+
+`TF_VAR_edge_install_docker=true`
+: Allow the edge OpenTofu stack to install Docker on the OCI host when Docker is
+missing. Default is to fail.
+
+`TF_VAR_haproxy_stats_password`
+: Required HAProxy stats password for the edge stack. It must be set explicitly
+and must not use the old default.
+
+`LAB_CONFIRM_NUKE=homelab`
+: Required confirmation for direct `./jeannie nuke`.
+
+`LAB_NUKE_DESTROY_PIMOX_WORKERS=true`
+: Also destroy Pimox worker VMs during `nuke`.
+
+`LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true`
+: Required confirmation for `move-prometheus-stack-workers`.
+
+## FILES
+
+`homelab.yml`
+: Canonical non-secret inventory for hosts, IPs, domains, ports, and the main
+script name.
+
+`.lab/cluster-workers.auto.tfvars.json`
+: Generated worker topology passed to the cluster stack.
+
+`.lab/pimox-workers.tsv`
+: Generated Pimox worker inventory.
+
+`.lab/manual-workers.tsv`
+: User-managed manual worker inventory.
+
+`$HOMELAB_STATE_DIR/logs`
+: Jeannie command logs.
+
+`$HOMELAB_STATE_DIR/tofu-state-backups`
+: Local OpenTofu and generated state backups.
+
+## EXAMPLES
+
+Start a three-worker homelab:
+
+```sh
+LAB_PIMOX_WORKER_COUNT=3 ./jeannie up
+```
+
+Preview all OpenTofu changes:
+
+```sh
+./jeannie plan
+```
+
+Run focused edge diagnostics:
+
+```sh
+./jeannie doctor-edge
+```
+
+Run the blocking pre-apply doctor directly:
+
+```sh
+./jeannie doctor-preapply
+```
+
+Create a local state backup:
+
+```sh
+./jeannie state-backup
+```
+
+Destroy the cluster path intentionally:
+
+```sh
+LAB_CONFIRM_NUKE=homelab ./jeannie nuke
+```
+
+## EXIT STATUS
+
+`0`
+: Command completed successfully.
+
+Non-zero
+: A check, deployment step, or external tool failed. Check the printed error and
+the referenced Jeannie log file for full command output.
+
+## SEE ALSO
+
+`README.md`, `docs/service-catalog.md`, `docs/runbooks/edge-failures.md`,
+`docs/runbooks/gitea-failures.md`,
+`docs/runbooks/cluster-stop-start-failures.md`, `docs/secrets.md`.

infra/rpi-services/bootstrap.sh

@@ -290,7 +290,12 @@ seed_uptime_kuma_monitors() {
 install_missing_packages ca-certificates curl iptables python3 rsync
 
 if ! command -v docker >/dev/null 2>&1; then
+    if [[ "${LAB_RPI_INSTALL_DOCKER:-false}" == "true" ]]; then
         curl -fsSL https://get.docker.com | sudo sh
+    else
+        echo "Docker is not installed on the RPi host. Install Docker first, or rerun with LAB_RPI_INSTALL_DOCKER=true to allow get.docker.com installation." >&2
+        exit 1
+    fi
 fi
 sudo systemctl enable docker >/dev/null
 

scripts/render-docs

@@ -99,6 +99,8 @@ main() {
         return 1
     fi
 
+    "${REPO_ROOT}/scripts/validate-homelab-inventory" "${INVENTORY_FILE}"
+
     main_script="$(read_inventory_value metadata.main_script)"
     if [[ -z "${main_script}" ]]; then
         printf 'homelab.yml metadata.main_script must not be empty\n' >&2

scripts/validate-homelab-inventory

@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+INVENTORY_FILE="${1:-${REPO_ROOT}/homelab.yml}"
+
+if [[ ! -s "${INVENTORY_FILE}" ]]; then
+    printf 'Missing homelab inventory: %s\n' "${INVENTORY_FILE}" >&2
+    exit 1
+fi
+
+if command -v python3 >/dev/null 2>&1 &&
+    python3 - "${INVENTORY_FILE}" <<'PY'
+import ipaddress
+import sys
+
+try:
+    import yaml
+except ImportError:
+    raise SystemExit(42)
+
+inventory_file = sys.argv[1]
+
+with open(inventory_file, encoding="utf-8") as handle:
+    document = yaml.safe_load(handle)
+
+failures = []
+
+
+def get(path):
+    value = document
+    for part in path.split("."):
+        if not isinstance(value, dict) or part not in value:
+            return None
+        value = value[part]
+    return value
+
+
+def require(path, expected_type=str):
+    value = get(path)
+    if value is None or value == "":
+        failures.append(f"{path} is required")
+        return None
+    if expected_type is not None and not isinstance(value, expected_type):
+        failures.append(f"{path} must be {expected_type.__name__}, got {type(value).__name__}")
+        return None
+    return value
+
+
+if not isinstance(document, dict):
+    failures.append("inventory root must be a YAML mapping")
+else:
+    for path in (
+        "metadata.main_script",
+        "domain.base",
+        "domain.public_url",
+        "domain.gitea_url",
+        "network.lan_cidr",
+        "network.lan_ip_prefix",
+        "network.metallb.traefik_ip",
+        "hosts.debian.lan_ip",
+        "hosts.debian.tailscale_ip",
+        "hosts.debian.docker_root",
+        "hosts.rpi4.lan_ip",
+        "hosts.rpi4.tailscale_ip",
+        "hosts.opi5_pimox.lan_ip",
+        "hosts.opi5_pimox.worker_storage",
+        "hosts.oci_edge.public_ip",
+        "services.gitea.root_url",
+        "services.local_registry.endpoint",
+        "services.traefik.load_balancer_ip",
+    ):
+        require(path)
+    require("pimox.default_worker_count", None)
+
+    subdomains = require("domain.subdomains", list)
+    if isinstance(subdomains, list):
+        for index, value in enumerate(subdomains):
+            if not isinstance(value, str) or not value:
+                failures.append(f"domain.subdomains[{index}] must be a non-empty string")
+
+    for path in (
+        "network.lan_cidr",
+        "network.metallb.traefik_ip",
+        "hosts.debian.lan_ip",
+        "hosts.debian.tailscale_ip",
+        "hosts.rpi4.lan_ip",
+        "hosts.rpi4.tailscale_ip",
+        "hosts.opi5_pimox.lan_ip",
+        "hosts.oci_edge.public_ip",
+        "services.traefik.load_balancer_ip",
+    ):
+        value = get(path)
+        if not value:
+            continue
+        try:
+            if path.endswith("lan_cidr"):
+                ipaddress.ip_network(str(value), strict=False)
+            else:
+                ipaddress.ip_address(str(value))
+        except ValueError:
+            failures.append(f"{path} is not a valid IP address/CIDR: {value}")
+
+    worker_count = get("pimox.default_worker_count")
+    if worker_count is not None:
+        try:
+            if int(worker_count) < 0:
+                failures.append("pimox.default_worker_count must be non-negative")
+        except (TypeError, ValueError):
+            failures.append("pimox.default_worker_count must be an integer")
+
+if failures:
+    for failure in failures:
+        print(f"inventory error: {failure}", file=sys.stderr)
+    raise SystemExit(1)
+
+print(f"inventory ok: {inventory_file}")
+PY
+then
+    exit 0
+else
+    status=$?
+    if [[ "${status}" != "42" ]]; then
+        exit "${status}"
+    fi
+fi
+
+if command -v ruby >/dev/null 2>&1; then
+    ruby - "${INVENTORY_FILE}" <<'RUBY'
+require "ipaddr"
+require "yaml"
+
+inventory_file = ARGV.fetch(0)
+document = YAML.load_file(inventory_file)
+failures = []
+
+def get(document, path)
+  path.split(".").reduce(document) do |value, part|
+    return nil unless value.is_a?(Hash) && value.key?(part)
+    value[part]
+  end
+end
+
+required_paths = %w[
+  metadata.main_script
+  domain.base
+  domain.public_url
+  domain.gitea_url
+  network.lan_cidr
+  network.lan_ip_prefix
+  network.metallb.traefik_ip
+  hosts.debian.lan_ip
+  hosts.debian.tailscale_ip
+  hosts.debian.docker_root
+  hosts.rpi4.lan_ip
+  hosts.rpi4.tailscale_ip
+  hosts.opi5_pimox.lan_ip
+  hosts.opi5_pimox.worker_storage
+  hosts.oci_edge.public_ip
+  services.gitea.root_url
+  services.local_registry.endpoint
+  services.traefik.load_balancer_ip
+  pimox.default_worker_count
+]
+
+unless document.is_a?(Hash)
+  failures << "inventory root must be a YAML mapping"
+else
+  required_paths.each do |path|
+    value = get(document, path)
+    failures << "#{path} is required" if value.nil? || value == ""
+  end
+
+  subdomains = get(document, "domain.subdomains")
+  if !subdomains.is_a?(Array)
+    failures << "domain.subdomains must be Array"
+  else
+    subdomains.each_with_index do |value, index|
+      failures << "domain.subdomains[#{index}] must be a non-empty string" unless value.is_a?(String) && !value.empty?
+    end
+  end
+
+  %w[
+    network.metallb.traefik_ip
+    hosts.debian.lan_ip
+    hosts.debian.tailscale_ip
+    hosts.rpi4.lan_ip
+    hosts.rpi4.tailscale_ip
+    hosts.opi5_pimox.lan_ip
+    hosts.oci_edge.public_ip
+    services.traefik.load_balancer_ip
+  ].each do |path|
+    value = get(document, path)
+    next if value.nil? || value == ""
+    begin
+      IPAddr.new(value.to_s)
+    rescue ArgumentError
+      failures << "#{path} is not a valid IP address: #{value}"
+    end
+  end
+
+  cidr = get(document, "network.lan_cidr")
+  begin
+    IPAddr.new(cidr.to_s) if cidr
+  rescue ArgumentError
+    failures << "network.lan_cidr is not a valid CIDR: #{cidr}"
+  end
+
+  worker_count = get(document, "pimox.default_worker_count")
+  failures << "pimox.default_worker_count must be non-negative" if worker_count && worker_count.to_i < 0
+end
+
+if failures.any?
+  failures.each { |failure| warn "inventory error: #{failure}" }
+  exit 1
+end
+
+puts "inventory ok: #{inventory_file}"
+RUBY
+    exit 0
+fi
+
+cat >&2 <<'EOF'
+No real YAML parser is available.
+Install python3-yaml or ruby, then rerun scripts/validate-homelab-inventory.
+EOF
+exit 1