Add Jeannie worker tailnet wrapper

README.md

@@ -196,6 +196,13 @@ ephemeral or pre-authorized auth key outside Git, either in
 LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./jeannie up
 ```
 
+For existing workers, prefer the Jeannie wrapper because it reads the generated
+worker keys and exports the correct OpenTofu variables:
+
+```bash
+./jeannie workers tailnet
+```
+
 When applying `bootstrap/cluster` directly, use the OpenTofu variable name
 instead of the Jeannie `LAB_` wrapper:
 
@@ -459,7 +466,8 @@ TLS coverage, and visible Uptime Kuma monitor coverage so stale routes and
 missing monitors are easier to spot.
 
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
-`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+`list`, `tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and
+`rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the
 cluster drain, VM stop, and `./jeannie up` path stay ordered.
 

README.md.tmpl

@@ -196,6 +196,13 @@ ephemeral or pre-authorized auth key outside Git, either in
 LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./{{ main_script }} up
 ```
 
+For existing workers, prefer the Jeannie wrapper because it reads the generated
+worker keys and exports the correct OpenTofu variables:
+
+```bash
+./{{ main_script }} workers tailnet
+```
+
 When applying `bootstrap/cluster` directly, use the OpenTofu variable name
 instead of the Jeannie `LAB_` wrapper:
 
@@ -459,7 +466,8 @@ TLS coverage, and visible Uptime Kuma monitor coverage so stale routes and
 missing monitors are easier to spot.
 
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
-`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+`list`, `tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and
+`rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the
 cluster drain, VM stop, and `./{{ main_script }} up` path stay ordered.
 

bootstrap/cluster/main.tf

@@ -971,6 +971,7 @@ EOT
   }
 
   provisioner "local-exec" {
+    interpreter = ["/usr/bin/env", "bash", "-c"]
     command     = <<EOT
 set -euo pipefail
 
@@ -1017,6 +1018,7 @@ EOT
   }
 
   provisioner "local-exec" {
+    interpreter = ["/usr/bin/env", "bash", "-c"]
     command     = <<EOT
 set -euo pipefail
 

docs/jeannie.1.md

@@ -177,7 +177,8 @@ visible Uptime Kuma monitor coverage.
 
 `workers <command>`
 : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
-`start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
+`tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and
+`rebalance`.
 
 `change-journal {list|path}`
 : List local journal entries written before risky commands, or print the journal

jeannie

@@ -4621,6 +4621,7 @@ check_pimox_worker_tailnet_egress() {
     local user
     local node_name
     local key_path
+    local output
 
     if [[ ! -s "${spec_file}" ]]; then
         echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE."
@@ -4635,16 +4636,18 @@ check_pimox_worker_tailnet_egress() {
         [[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue
         found=1
         printf '%-28s ' "${node_name}"
-        if ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" "
+        if output="$(ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" "
 set -eu
-command -v tailscale >/dev/null
+command -v tailscale >/dev/null || { echo 'tailscale command missing'; exit 1; }
-sudo systemctl is-active --quiet tailscaled
+sudo systemctl is-active --quiet tailscaled || { echo 'tailscaled is not active'; exit 1; }
-tailscale ip -4 >/dev/null
+tailscale_ip=\"\$(tailscale ip -4 2>/dev/null | head -n 1 || true)\"
-timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}'
+[ -n \"\$tailscale_ip\" ] || { echo 'worker is not joined to Tailscale'; exit 1; }
-" >/dev/null 2>&1; then
+timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' || { echo 'tailnet TCP probe failed'; exit 1; }
-            printf 'ok - %s:%s reachable over tailnet\n' "${probe_host}" "${probe_port}"
+echo \"tailscale_ip=\$tailscale_ip\"
+" 2>&1)"; then
+            printf 'ok - %s:%s reachable over tailnet (%s)\n' "${probe_host}" "${probe_port}" "${output}"
         else
-            printf 'fail - cannot reach %s:%s over tailnet\n' "${probe_host}" "${probe_port}"
+            printf 'fail - %s\n' "$(printf '%s' "${output}" | head -n 1)"
             failures=$((failures + 1))
         fi
     done <"${spec_file}"
@@ -6194,7 +6197,7 @@ Cluster Lifecycle
   rebuild-cluster                            Recreate the cluster through the guarded path.
   stop-cluster                               Stop Kubernetes and worker VMs without destroy.
   start-cluster                              Start Kubernetes and desired worker VMs.
-  workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>
+  workers <list|tailnet|start|stop|drain|uncordon|recreate-plan|rebalance>
                                              Manage Pimox/Kubernetes workers.
   move-prometheus-stack-workers              Move monitoring workloads to worker nodes.
   doctor-versions                            Check Kubernetes/container tooling versions.

scripts/workers

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
 KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
 PIMOX_HOST="${LAB_PIMOX_HOST:-192.168.100.80}"
 PIMOX_USER="${LAB_PIMOX_USER:-jv}"
@@ -17,6 +18,7 @@ Usage: ./jeannie workers <command> [args]
 
 Commands:
   list                       List desired Pimox workers, VM status, and node status
+  tailnet                    Install/join Tailscale on generated Pimox workers
   start [index|all]          Start Pimox worker VM(s)
   stop [index|all]           Gracefully stop Pimox worker VM(s)
   drain <node>               Drain a Kubernetes worker node
@@ -202,10 +204,73 @@ rebalance() {
     echo "Restarting pending/failed pods is intentionally manual. Use kubectl delete pod for specific pods after checking events."
 }
 
+tailnet_setup() {
+    local var_file="${LAB_CLUSTER_VAR_FILE:-${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json}"
+    local auth_key_file="${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-${HOME}/.config/homelab/tailscale-pimox-worker.authkey}"
+    local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
+    local accept_routes="${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-true}"
+    local pod_egress_snat="${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-true}"
+    local keys=()
+    local key
+    local cmd
+
+    if [ ! -s "$var_file" ]; then
+        echo "Missing generated worker var file: $var_file" >&2
+        echo "Run ./jeannie up once, or set LAB_CLUSTER_VAR_FILE." >&2
+        exit 1
+    fi
+    if [ -z "${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}" ] && [ ! -s "$auth_key_file" ]; then
+        echo "Missing Pimox worker Tailscale auth key." >&2
+        echo "Create $auth_key_file outside Git or set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY." >&2
+        exit 1
+    fi
+
+    while IFS= read -r key; do
+        [ -n "$key" ] && keys+=("$key")
+    done < <(python3 - "$var_file" "$worker_key_prefix" <<'PY'
+import json
+import sys
+
+var_file, prefix = sys.argv[1:3]
+with open(var_file, encoding="utf-8") as handle:
+    worker_nodes = (json.load(handle).get("worker_nodes") or {})
+for key in sorted(worker_nodes):
+    if key.startswith(prefix):
+        print(key)
+PY
+)
+
+    if ((${#keys[@]} == 0)); then
+        echo "No Pimox worker keys found in $var_file with prefix '$worker_key_prefix'." >&2
+        exit 1
+    fi
+
+    cmd=(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" apply -var-file="$var_file")
+    for key in "${keys[@]}"; do
+        cmd+=("-replace=null_resource.kubeadm_worker[\"${key}\"]")
+    done
+
+    printf 'Re-running cluster worker bootstrap with Tailscale enabled for keys: %s\n' "${keys[*]}"
+    if [ -n "${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}" ]; then
+        printf 'Auth key source: LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY\n'
+    else
+        printf 'Auth key source: %s\n' "$auth_key_file"
+    fi
+
+    env \
+        TF_VAR_worker_tailscale_enabled=true \
+        TF_VAR_worker_tailscale_accept_routes="$accept_routes" \
+        TF_VAR_worker_tailscale_pod_egress_snat="$pod_egress_snat" \
+        "${cmd[@]}"
+}
+
 case "${1:-}" in
     list)
         list_workers
         ;;
+    tailnet)
+        tailnet_setup
+        ;;
     start)
         start_workers "${2:-all}"
         ;;