Add Tailscale bootstrap for Pimox workers

README.md

@@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set
 must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID
 `9010` and VMID `9011`.
 
+Pimox workers can join the tailnet during cluster worker bootstrap. Keep the
+ephemeral or pre-authorized auth key outside Git, either in
+`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in
+`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with:
+
+```bash
+LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./jeannie up
+```
+
+Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT
+from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach
+tailnet-only services. Override with
+`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or
+`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not
+wanted. `./jeannie doctor-cluster` checks Pimox worker tailnet egress
+against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and
+`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`.
+
 OpenWrt firewall VM automation is available as a standalone command because it
 attaches to both WAN and LAN bridges. Run `./jeannie openwrt` after `vmbr1`
 already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM
@@ -456,6 +474,10 @@ next step:
 ./jeannie doctor-cluster
 ```
 
+`doctor-cluster` also checks whether Pimox workers have Tailscale installed,
+`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet
+probe service.
+
 ## RPi Services
 
 The Raspberry Pi runs lightweight always-on services outside Kubernetes from

README.md.tmpl

@@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set
 must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID
 `9010` and VMID `9011`.
 
+Pimox workers can join the tailnet during cluster worker bootstrap. Keep the
+ephemeral or pre-authorized auth key outside Git, either in
+`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in
+`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with:
+
+```bash
+LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./{{ main_script }} up
+```
+
+Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT
+from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach
+tailnet-only services. Override with
+`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or
+`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not
+wanted. `./{{ main_script }} doctor-cluster` checks Pimox worker tailnet egress
+against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and
+`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`.
+
 OpenWrt firewall VM automation is available as a standalone command because it
 attaches to both WAN and LAN bridges. Run `./{{ main_script }} openwrt` after `vmbr1`
 already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM
@@ -456,6 +474,10 @@ next step:
 ./{{ main_script }} doctor-cluster
 ```
 
+`doctor-cluster` also checks whether Pimox workers have Tailscale installed,
+`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet
+probe service.
+
 ## RPi Services
 
 The Raspberry Pi runs lightweight always-on services outside Kubernetes from

bootstrap/cluster/main.tf

@@ -415,6 +415,10 @@ resource "null_resource" "kubeadm_worker" {
     worker_rejoin_reset_version          = "2"
     node_dns_servers                     = join(" ", var.node_dns_servers)
     persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
+    worker_tailscale_version             = "1"
+    worker_tailscale_enabled             = tostring(var.worker_tailscale_enabled)
+    worker_tailscale_accept_routes       = tostring(var.worker_tailscale_accept_routes)
+    worker_tailscale_pod_egress_snat     = tostring(var.worker_tailscale_pod_egress_snat)
     tailscale_nodeport_version           = "3"
     tailscale_nodeport_enabled           = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false"
     tailscale_nodeport_peer_ip           = var.tailscale_nodeport_access.peer_ip
@@ -478,6 +482,84 @@ DOCKER_SOURCES
   sudo apt-mark hold containerd.io
 }
 
+install_tailscale() {
+  if [ "${self.triggers.worker_tailscale_enabled}" != "true" ]; then
+    return 0
+  fi
+
+  if command -v tailscale >/dev/null 2>&1 && systemctl list-unit-files tailscaled.service >/dev/null 2>&1; then
+    sudo systemctl enable --now tailscaled
+    return 0
+  fi
+
+  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
+  sudo install -d -m 0755 /usr/share/keyrings
+  curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.noarmor.gpg" |
+    sudo tee /usr/share/keyrings/tailscale-archive-keyring.gpg >/dev/null
+  curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.tailscale-keyring.list" |
+    sudo tee /etc/apt/sources.list.d/tailscale.list >/dev/null
+  sudo apt-get update
+  sudo apt-get install -y --no-install-recommends tailscale
+  sudo systemctl enable --now tailscaled
+}
+
+configure_tailscale_pod_egress_snat() {
+  local enabled="$1"
+  local pod_cidr="$2"
+
+  if [ "${self.triggers.worker_tailscale_enabled}" != "true" ] || [ "$enabled" != "true" ]; then
+    sudo systemctl disable --now homelab-tailscale-pod-egress.service >/dev/null 2>&1 || true
+    return 0
+  fi
+
+  sudo mkdir -p /usr/local/sbin /etc/sysctl.d
+  sudo tee /etc/sysctl.d/98-homelab-tailscale-pod-egress.conf >/dev/null <<POD_EGRESS_SYSCTL_EOT
+net.ipv4.ip_forward = 1
+net.ipv4.conf.all.rp_filter = 0
+net.ipv4.conf.tailscale0.rp_filter = 0
+POD_EGRESS_SYSCTL_EOT
+
+  sudo tee /usr/local/sbin/homelab-tailscale-pod-egress.sh >/dev/null <<POD_EGRESS_SCRIPT_EOT
+#!/bin/sh
+set -eu
+
+sysctl -w net.ipv4.ip_forward=1 >/dev/null
+sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
+sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true
+
+if ! ip link show tailscale0 >/dev/null 2>&1; then
+  echo "tailscale0 is not present; skipping Tailscale pod egress SNAT"
+  exit 0
+fi
+
+iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -j ACCEPT 2>/dev/null ||
+  iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -j ACCEPT
+iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
+  iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+iptables -t nat -C POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE 2>/dev/null ||
+  iptables -t nat -A POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE
+POD_EGRESS_SCRIPT_EOT
+  sudo chmod 0755 /usr/local/sbin/homelab-tailscale-pod-egress.sh
+
+  sudo tee /etc/systemd/system/homelab-tailscale-pod-egress.service >/dev/null <<'POD_EGRESS_SERVICE_EOT'
+[Unit]
+Description=Homelab Tailscale pod egress SNAT
+After=network-online.target tailscaled.service kubelet.service
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/homelab-tailscale-pod-egress.sh
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
+POD_EGRESS_SERVICE_EOT
+  sudo systemctl daemon-reload
+  sudo systemctl enable homelab-tailscale-pod-egress.service >/dev/null
+  sudo systemctl restart homelab-tailscale-pod-egress.service
+}
+
 configure_node_dns() {
   dns_servers="${self.triggers.node_dns_servers}"
   if [ -z "$dns_servers" ]; then
@@ -705,6 +787,7 @@ REGISTRY_EOT
 configure_node_dns
 install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
 install_containerd_runtime
+install_tailscale
 install_cni_plugins
 sudo systemctl enable --now iscsid
 sudo systemctl enable kubelet || true
@@ -739,6 +822,10 @@ if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
   printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
 fi
 
+configure_tailscale_pod_egress_snat \
+  "${self.triggers.worker_tailscale_pod_egress_snat}" \
+  "${self.triggers.tailscale_nodeport_pod_cidr}"
+
 configure_tailscale_nodeport_access() {
   local enabled="$1"
   local peer_ip="$2"
@@ -887,6 +974,52 @@ EOT
     command = <<EOT
 set -euo pipefail
 
+if [ '${self.triggers.worker_tailscale_enabled}' != 'true' ]; then
+  exit 0
+fi
+
+ssh_target='${self.triggers.user}@${self.triggers.host}'
+ssh_key='${self.triggers.ssh_key_path}'
+node_name='${self.triggers.node_name}'
+accept_routes='${self.triggers.worker_tailscale_accept_routes}'
+
+if ! ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+  'command -v tailscale >/dev/null 2>&1 && sudo systemctl is-active --quiet tailscaled'; then
+  echo "tailscale is not installed or tailscaled is not active on $ssh_target" >&2
+  exit 1
+fi
+
+if ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+  'tailscale status --json >/dev/null 2>&1'; then
+  ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+    "sudo tailscale set --accept-routes=$accept_routes"
+  exit 0
+fi
+
+auth_key="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}"
+auth_key_file="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-$${HOME}/.config/homelab/tailscale-pimox-worker.authkey}"
+if [ -z "$auth_key" ] && [ -s "$auth_key_file" ]; then
+  auth_key="$(tr -d '\n\r' <"$auth_key_file")"
+fi
+if [ -z "$auth_key" ]; then
+  echo "Pimox worker Tailscale is enabled, but no auth key was provided." >&2
+  echo "Set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY or create $auth_key_file outside Git." >&2
+  exit 1
+fi
+
+printf '%s\n' "$auth_key" |
+  ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+    "read -r auth_key && sudo tailscale up --auth-key \"\$auth_key\" --hostname '$node_name' --accept-routes=$accept_routes --reset"
+
+ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
+  'sudo systemctl restart homelab-tailscale-pod-egress.service 2>/dev/null || true'
+EOT
+  }
+
+  provisioner "local-exec" {
+    command = <<EOT
+set -euo pipefail
+
 ssh_target='${self.triggers.user}@${self.triggers.host}'
 ssh_key='${self.triggers.ssh_key_path}'
 node_name='${self.triggers.node_name}'

bootstrap/cluster/variables.tf

@@ -79,6 +79,21 @@ variable "force_worker_rejoin" {
   default = false
 }
 
+variable "worker_tailscale_enabled" {
+  type    = bool
+  default = false
+}
+
+variable "worker_tailscale_accept_routes" {
+  type    = bool
+  default = true
+}
+
+variable "worker_tailscale_pod_egress_snat" {
+  type    = bool
+  default = true
+}
+
 variable "tailscale_nodeport_access" {
   type = object({
     enabled           = bool

docs/jeannie.1.md

@@ -279,7 +279,8 @@ health.
 Uptime Kuma, and Docker root state.
 
 `doctor-cluster`
-: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, and app health.
+: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, app health, and
+Pimox worker tailnet egress.
 
 `inventory-check`
 : Validate `homelab.yml` with a real YAML parser when available.

jeannie

@@ -196,6 +196,9 @@ export_homelab_inventory_tf_vars() {
     export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}"
     export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}"
     export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}"
+    export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}"
+    export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}"
+    export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}"
 }
 
 export_homelab_inventory_tf_vars
@@ -4607,6 +4610,53 @@ status_run() {
     fi
 }
 
+check_pimox_worker_tailnet_egress() {
+    local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
+    local probe_host="${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}"
+    local probe_port="${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}"
+    local failures=0
+    local found=0
+    local worker_key
+    local host
+    local user
+    local node_name
+    local key_path
+
+    if [[ ! -s "${spec_file}" ]]; then
+        echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE."
+        return 1
+    fi
+    if [[ -z "${probe_host}" ]]; then
+        echo "No tailnet probe host configured; set LAB_PIMOX_WORKER_TAILNET_PROBE_HOST or LAB_DEBIAN_TAILSCALE_IP."
+        return 1
+    fi
+
+    while IFS=$'\t' read -r worker_key host user node_name key_path; do
+        [[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue
+        found=1
+        printf '%-28s ' "${node_name}"
+        if ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" "
+set -eu
+command -v tailscale >/dev/null
+sudo systemctl is-active --quiet tailscaled
+tailscale ip -4 >/dev/null
+timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}'
+" >/dev/null 2>&1; then
+            printf 'ok - %s:%s reachable over tailnet\n' "${probe_host}" "${probe_port}"
+        else
+            printf 'fail - cannot reach %s:%s over tailnet\n' "${probe_host}" "${probe_port}"
+            failures=$((failures + 1))
+        fi
+    done <"${spec_file}"
+
+    if ((found == 0)); then
+        echo "No Pimox workers were listed in ${spec_file}."
+        return 1
+    fi
+
+    ((failures == 0))
+}
+
 status_http() {
     local name="$1"
     local url="$2"
@@ -5455,6 +5505,7 @@ doctor_cluster() {
     status_systemd_units kubelet containerd docker
     status_kubernetes
     status_pimox_workers
+    status_run "pimox worker tailnet egress" check_pimox_worker_tailnet_egress
     status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/"
 
     printf '\n-- local CRI containers\n'
@@ -5480,6 +5531,7 @@ Kubeconfig path: ${KUBECONFIG_PATH}
 Traefik LoadBalancer IP: ${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}
 Pimox host: ${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}@${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}
 Worker storage: ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}
+Pimox worker tailnet probe: ${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-unset}}:${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}
 Known runbook: docs/runbooks/cluster-stop-start-failures.md
 EOF
 }