From ef83e71088d93319008150f3928452ce73ea4545 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 21:45:19 -0600 Subject: [PATCH] Add Jeannie worker tailnet wrapper --- README.md | 10 +++++- README.md.tmpl | 10 +++++- bootstrap/cluster/main.tf | 6 ++-- docs/jeannie.1.md | 3 +- jeannie | 21 +++++++------ scripts/workers | 65 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 101 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e47a9c5..f8cb653 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,13 @@ ephemeral or pre-authorized auth key outside Git, either in LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./jeannie up ``` +For existing workers, prefer the Jeannie wrapper because it reads the generated +worker keys and exports the correct OpenTofu variables: + +```bash +./jeannie workers tailnet +``` + When applying `bootstrap/cluster` directly, use the OpenTofu variable name instead of the Jeannie `LAB_` wrapper: @@ -459,7 +466,8 @@ TLS coverage, and visible Uptime Kuma monitor coverage so stale routes and missing monitors are easier to spot. `workers` provides named Kubernetes/Pimox worker lifecycle operations: -`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. +`list`, `tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and +`rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the cluster drain, VM stop, and `./jeannie up` path stay ordered. diff --git a/README.md.tmpl b/README.md.tmpl index e2cd9b9..de8b99b 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -196,6 +196,13 @@ ephemeral or pre-authorized auth key outside Git, either in LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./{{ main_script }} up ``` +For existing workers, prefer the Jeannie wrapper because it reads the generated +worker keys and exports the correct OpenTofu variables: + +```bash +./{{ main_script }} workers tailnet +``` + When applying `bootstrap/cluster` directly, use the OpenTofu variable name instead of the Jeannie `LAB_` wrapper: @@ -459,7 +466,8 @@ TLS coverage, and visible Uptime Kuma monitor coverage so stale routes and missing monitors are easier to spot. `workers` provides named Kubernetes/Pimox worker lifecycle operations: -`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. +`list`, `tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and +`rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the cluster drain, VM stop, and `./{{ main_script }} up` path stay ordered. diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index af8287d..8743cd5 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -971,7 +971,8 @@ EOT } provisioner "local-exec" { - command = <` : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`, -`start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. +`tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and +`rebalance`. `change-journal {list|path}` : List local journal entries written before risky commands, or print the journal diff --git a/jeannie b/jeannie index 0fa0b92..3af3406 100755 --- a/jeannie +++ b/jeannie @@ -4621,6 +4621,7 @@ check_pimox_worker_tailnet_egress() { local user local node_name local key_path + local output if [[ ! -s "${spec_file}" ]]; then echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE." @@ -4635,16 +4636,18 @@ check_pimox_worker_tailnet_egress() { [[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue found=1 printf '%-28s ' "${node_name}" - if ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" " + if output="$(ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" " set -eu -command -v tailscale >/dev/null -sudo systemctl is-active --quiet tailscaled -tailscale ip -4 >/dev/null -timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' -" >/dev/null 2>&1; then - printf 'ok - %s:%s reachable over tailnet\n' "${probe_host}" "${probe_port}" +command -v tailscale >/dev/null || { echo 'tailscale command missing'; exit 1; } +sudo systemctl is-active --quiet tailscaled || { echo 'tailscaled is not active'; exit 1; } +tailscale_ip=\"\$(tailscale ip -4 2>/dev/null | head -n 1 || true)\" +[ -n \"\$tailscale_ip\" ] || { echo 'worker is not joined to Tailscale'; exit 1; } +timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' || { echo 'tailnet TCP probe failed'; exit 1; } +echo \"tailscale_ip=\$tailscale_ip\" +" 2>&1)"; then + printf 'ok - %s:%s reachable over tailnet (%s)\n' "${probe_host}" "${probe_port}" "${output}" else - printf 'fail - cannot reach %s:%s over tailnet\n' "${probe_host}" "${probe_port}" + printf 'fail - %s\n' "$(printf '%s' "${output}" | head -n 1)" failures=$((failures + 1)) fi done <"${spec_file}" @@ -6194,7 +6197,7 @@ Cluster Lifecycle rebuild-cluster Recreate the cluster through the guarded path. stop-cluster Stop Kubernetes and worker VMs without destroy. start-cluster Start Kubernetes and desired worker VMs. - workers + workers Manage Pimox/Kubernetes workers. move-prometheus-stack-workers Move monitoring workloads to worker nodes. doctor-versions Check Kubernetes/container tooling versions. diff --git a/scripts/workers b/scripts/workers index c2d386e..ebbae86 100755 --- a/scripts/workers +++ b/scripts/workers @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" PIMOX_HOST="${LAB_PIMOX_HOST:-192.168.100.80}" PIMOX_USER="${LAB_PIMOX_USER:-jv}" @@ -17,6 +18,7 @@ Usage: ./jeannie workers [args] Commands: list List desired Pimox workers, VM status, and node status + tailnet Install/join Tailscale on generated Pimox workers start [index|all] Start Pimox worker VM(s) stop [index|all] Gracefully stop Pimox worker VM(s) drain Drain a Kubernetes worker node @@ -202,10 +204,73 @@ rebalance() { echo "Restarting pending/failed pods is intentionally manual. Use kubectl delete pod for specific pods after checking events." } +tailnet_setup() { + local var_file="${LAB_CLUSTER_VAR_FILE:-${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json}" + local auth_key_file="${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-${HOME}/.config/homelab/tailscale-pimox-worker.authkey}" + local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" + local accept_routes="${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-true}" + local pod_egress_snat="${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-true}" + local keys=() + local key + local cmd + + if [ ! -s "$var_file" ]; then + echo "Missing generated worker var file: $var_file" >&2 + echo "Run ./jeannie up once, or set LAB_CLUSTER_VAR_FILE." >&2 + exit 1 + fi + if [ -z "${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}" ] && [ ! -s "$auth_key_file" ]; then + echo "Missing Pimox worker Tailscale auth key." >&2 + echo "Create $auth_key_file outside Git or set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY." >&2 + exit 1 + fi + + while IFS= read -r key; do + [ -n "$key" ] && keys+=("$key") + done < <(python3 - "$var_file" "$worker_key_prefix" <<'PY' +import json +import sys + +var_file, prefix = sys.argv[1:3] +with open(var_file, encoding="utf-8") as handle: + worker_nodes = (json.load(handle).get("worker_nodes") or {}) +for key in sorted(worker_nodes): + if key.startswith(prefix): + print(key) +PY +) + + if ((${#keys[@]} == 0)); then + echo "No Pimox worker keys found in $var_file with prefix '$worker_key_prefix'." >&2 + exit 1 + fi + + cmd=(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" apply -var-file="$var_file") + for key in "${keys[@]}"; do + cmd+=("-replace=null_resource.kubeadm_worker[\"${key}\"]") + done + + printf 'Re-running cluster worker bootstrap with Tailscale enabled for keys: %s\n' "${keys[*]}" + if [ -n "${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}" ]; then + printf 'Auth key source: LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY\n' + else + printf 'Auth key source: %s\n' "$auth_key_file" + fi + + env \ + TF_VAR_worker_tailscale_enabled=true \ + TF_VAR_worker_tailscale_accept_routes="$accept_routes" \ + TF_VAR_worker_tailscale_pod_egress_snat="$pod_egress_snat" \ + "${cmd[@]}" +} + case "${1:-}" in list) list_workers ;; + tailnet) + tailnet_setup + ;; start) start_workers "${2:-all}" ;;