Add Tailscale bootstrap for Pimox workers
This commit is contained in:
juvdiaz
parent
a2ef9b7627
commit
df5702de34
22
README.md
22
README.md
|
|
@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set
|
|||
must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID
|
||||
`9010` and VMID `9011`.
|
||||
|
||||
Pimox workers can join the tailnet during cluster worker bootstrap. Keep the
|
||||
ephemeral or pre-authorized auth key outside Git, either in
|
||||
`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with:
|
||||
|
||||
```bash
|
||||
LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./jeannie up
|
||||
```
|
||||
|
||||
Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT
|
||||
from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach
|
||||
tailnet-only services. Override with
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not
|
||||
wanted. `./jeannie doctor-cluster` checks Pimox worker tailnet egress
|
||||
against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and
|
||||
`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`.
|
||||
|
||||
OpenWrt firewall VM automation is available as a standalone command because it
|
||||
attaches to both WAN and LAN bridges. Run `./jeannie openwrt` after `vmbr1`
|
||||
already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM
|
||||
|
|
@ -456,6 +474,10 @@ next step:
|
|||
./jeannie doctor-cluster
|
||||
```
|
||||
|
||||
`doctor-cluster` also checks whether Pimox workers have Tailscale installed,
|
||||
`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet
|
||||
probe service.
|
||||
|
||||
## RPi Services
|
||||
|
||||
The Raspberry Pi runs lightweight always-on services outside Kubernetes from
|
||||
|
|
|
|||
|
|
@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set
|
|||
must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID
|
||||
`9010` and VMID `9011`.
|
||||
|
||||
Pimox workers can join the tailnet during cluster worker bootstrap. Keep the
|
||||
ephemeral or pre-authorized auth key outside Git, either in
|
||||
`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with:
|
||||
|
||||
```bash
|
||||
LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./{{ main_script }} up
|
||||
```
|
||||
|
||||
Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT
|
||||
from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach
|
||||
tailnet-only services. Override with
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or
|
||||
`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not
|
||||
wanted. `./{{ main_script }} doctor-cluster` checks Pimox worker tailnet egress
|
||||
against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and
|
||||
`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`.
|
||||
|
||||
OpenWrt firewall VM automation is available as a standalone command because it
|
||||
attaches to both WAN and LAN bridges. Run `./{{ main_script }} openwrt` after `vmbr1`
|
||||
already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM
|
||||
|
|
@ -456,6 +474,10 @@ next step:
|
|||
./{{ main_script }} doctor-cluster
|
||||
```
|
||||
|
||||
`doctor-cluster` also checks whether Pimox workers have Tailscale installed,
|
||||
`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet
|
||||
probe service.
|
||||
|
||||
## RPi Services
|
||||
|
||||
The Raspberry Pi runs lightweight always-on services outside Kubernetes from
|
||||
|
|
|
|||
|
|
@ -415,6 +415,10 @@ resource "null_resource" "kubeadm_worker" {
|
|||
worker_rejoin_reset_version = "2"
|
||||
node_dns_servers = join(" ", var.node_dns_servers)
|
||||
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
||||
worker_tailscale_version = "1"
|
||||
worker_tailscale_enabled = tostring(var.worker_tailscale_enabled)
|
||||
worker_tailscale_accept_routes = tostring(var.worker_tailscale_accept_routes)
|
||||
worker_tailscale_pod_egress_snat = tostring(var.worker_tailscale_pod_egress_snat)
|
||||
tailscale_nodeport_version = "3"
|
||||
tailscale_nodeport_enabled = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false"
|
||||
tailscale_nodeport_peer_ip = var.tailscale_nodeport_access.peer_ip
|
||||
|
|
@ -478,6 +482,84 @@ DOCKER_SOURCES
|
|||
sudo apt-mark hold containerd.io
|
||||
}
|
||||
|
||||
install_tailscale() {
|
||||
if [ "${self.triggers.worker_tailscale_enabled}" != "true" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v tailscale >/dev/null 2>&1 && systemctl list-unit-files tailscaled.service >/dev/null 2>&1; then
|
||||
sudo systemctl enable --now tailscaled
|
||||
return 0
|
||||
fi
|
||||
|
||||
codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
|
||||
sudo install -d -m 0755 /usr/share/keyrings
|
||||
curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.noarmor.gpg" |
|
||||
sudo tee /usr/share/keyrings/tailscale-archive-keyring.gpg >/dev/null
|
||||
curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.tailscale-keyring.list" |
|
||||
sudo tee /etc/apt/sources.list.d/tailscale.list >/dev/null
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --no-install-recommends tailscale
|
||||
sudo systemctl enable --now tailscaled
|
||||
}
|
||||
|
||||
configure_tailscale_pod_egress_snat() {
|
||||
local enabled="$1"
|
||||
local pod_cidr="$2"
|
||||
|
||||
if [ "${self.triggers.worker_tailscale_enabled}" != "true" ] || [ "$enabled" != "true" ]; then
|
||||
sudo systemctl disable --now homelab-tailscale-pod-egress.service >/dev/null 2>&1 || true
|
||||
return 0
|
||||
fi
|
||||
|
||||
sudo mkdir -p /usr/local/sbin /etc/sysctl.d
|
||||
sudo tee /etc/sysctl.d/98-homelab-tailscale-pod-egress.conf >/dev/null <<POD_EGRESS_SYSCTL_EOT
|
||||
net.ipv4.ip_forward = 1
|
||||
net.ipv4.conf.all.rp_filter = 0
|
||||
net.ipv4.conf.tailscale0.rp_filter = 0
|
||||
POD_EGRESS_SYSCTL_EOT
|
||||
|
||||
sudo tee /usr/local/sbin/homelab-tailscale-pod-egress.sh >/dev/null <<POD_EGRESS_SCRIPT_EOT
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
||||
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
|
||||
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true
|
||||
|
||||
if ! ip link show tailscale0 >/dev/null 2>&1; then
|
||||
echo "tailscale0 is not present; skipping Tailscale pod egress SNAT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -j ACCEPT 2>/dev/null ||
|
||||
iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -j ACCEPT
|
||||
iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
|
||||
iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
||||
iptables -t nat -C POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE 2>/dev/null ||
|
||||
iptables -t nat -A POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE
|
||||
POD_EGRESS_SCRIPT_EOT
|
||||
sudo chmod 0755 /usr/local/sbin/homelab-tailscale-pod-egress.sh
|
||||
|
||||
sudo tee /etc/systemd/system/homelab-tailscale-pod-egress.service >/dev/null <<'POD_EGRESS_SERVICE_EOT'
|
||||
[Unit]
|
||||
Description=Homelab Tailscale pod egress SNAT
|
||||
After=network-online.target tailscaled.service kubelet.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/sbin/homelab-tailscale-pod-egress.sh
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
POD_EGRESS_SERVICE_EOT
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable homelab-tailscale-pod-egress.service >/dev/null
|
||||
sudo systemctl restart homelab-tailscale-pod-egress.service
|
||||
}
|
||||
|
||||
configure_node_dns() {
|
||||
dns_servers="${self.triggers.node_dns_servers}"
|
||||
if [ -z "$dns_servers" ]; then
|
||||
|
|
@ -705,6 +787,7 @@ REGISTRY_EOT
|
|||
configure_node_dns
|
||||
install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
|
||||
install_containerd_runtime
|
||||
install_tailscale
|
||||
install_cni_plugins
|
||||
sudo systemctl enable --now iscsid
|
||||
sudo systemctl enable kubelet || true
|
||||
|
|
@ -739,6 +822,10 @@ if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|||
printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
||||
fi
|
||||
|
||||
configure_tailscale_pod_egress_snat \
|
||||
"${self.triggers.worker_tailscale_pod_egress_snat}" \
|
||||
"${self.triggers.tailscale_nodeport_pod_cidr}"
|
||||
|
||||
configure_tailscale_nodeport_access() {
|
||||
local enabled="$1"
|
||||
local peer_ip="$2"
|
||||
|
|
@ -887,6 +974,52 @@ EOT
|
|||
command = <<EOT
|
||||
set -euo pipefail
|
||||
|
||||
if [ '${self.triggers.worker_tailscale_enabled}' != 'true' ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
ssh_target='${self.triggers.user}@${self.triggers.host}'
|
||||
ssh_key='${self.triggers.ssh_key_path}'
|
||||
node_name='${self.triggers.node_name}'
|
||||
accept_routes='${self.triggers.worker_tailscale_accept_routes}'
|
||||
|
||||
if ! ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
|
||||
'command -v tailscale >/dev/null 2>&1 && sudo systemctl is-active --quiet tailscaled'; then
|
||||
echo "tailscale is not installed or tailscaled is not active on $ssh_target" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
|
||||
'tailscale status --json >/dev/null 2>&1'; then
|
||||
ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
|
||||
"sudo tailscale set --accept-routes=$accept_routes"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
auth_key="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}"
|
||||
auth_key_file="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-$${HOME}/.config/homelab/tailscale-pimox-worker.authkey}"
|
||||
if [ -z "$auth_key" ] && [ -s "$auth_key_file" ]; then
|
||||
auth_key="$(tr -d '\n\r' <"$auth_key_file")"
|
||||
fi
|
||||
if [ -z "$auth_key" ]; then
|
||||
echo "Pimox worker Tailscale is enabled, but no auth key was provided." >&2
|
||||
echo "Set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY or create $auth_key_file outside Git." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "$auth_key" |
|
||||
ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
|
||||
"read -r auth_key && sudo tailscale up --auth-key \"\$auth_key\" --hostname '$node_name' --accept-routes=$accept_routes --reset"
|
||||
|
||||
ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \
|
||||
'sudo systemctl restart homelab-tailscale-pod-egress.service 2>/dev/null || true'
|
||||
EOT
|
||||
}
|
||||
|
||||
provisioner "local-exec" {
|
||||
command = <<EOT
|
||||
set -euo pipefail
|
||||
|
||||
ssh_target='${self.triggers.user}@${self.triggers.host}'
|
||||
ssh_key='${self.triggers.ssh_key_path}'
|
||||
node_name='${self.triggers.node_name}'
|
||||
|
|
|
|||
|
|
@ -79,6 +79,21 @@ variable "force_worker_rejoin" {
|
|||
default = false
|
||||
}
|
||||
|
||||
variable "worker_tailscale_enabled" {
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "worker_tailscale_accept_routes" {
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "worker_tailscale_pod_egress_snat" {
|
||||
type = bool
|
||||
default = true
|
||||
}
|
||||
|
||||
variable "tailscale_nodeport_access" {
|
||||
type = object({
|
||||
enabled = bool
|
||||
|
|
|
|||
|
|
@ -279,7 +279,8 @@ health.
|
|||
Uptime Kuma, and Docker root state.
|
||||
|
||||
`doctor-cluster`
|
||||
: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, and app health.
|
||||
: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, app health, and
|
||||
Pimox worker tailnet egress.
|
||||
|
||||
`inventory-check`
|
||||
: Validate `homelab.yml` with a real YAML parser when available.
|
||||
|
|
|
|||
52
jeannie
52
jeannie
|
|
@ -196,6 +196,9 @@ export_homelab_inventory_tf_vars() {
|
|||
export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}"
|
||||
export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}"
|
||||
export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}"
|
||||
export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}"
|
||||
export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}"
|
||||
export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}"
|
||||
}
|
||||
|
||||
export_homelab_inventory_tf_vars
|
||||
|
|
@ -4607,6 +4610,53 @@ status_run() {
|
|||
fi
|
||||
}
|
||||
|
||||
check_pimox_worker_tailnet_egress() {
|
||||
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
|
||||
local probe_host="${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}"
|
||||
local probe_port="${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}"
|
||||
local failures=0
|
||||
local found=0
|
||||
local worker_key
|
||||
local host
|
||||
local user
|
||||
local node_name
|
||||
local key_path
|
||||
|
||||
if [[ ! -s "${spec_file}" ]]; then
|
||||
echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE."
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${probe_host}" ]]; then
|
||||
echo "No tailnet probe host configured; set LAB_PIMOX_WORKER_TAILNET_PROBE_HOST or LAB_DEBIAN_TAILSCALE_IP."
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS=$'\t' read -r worker_key host user node_name key_path; do
|
||||
[[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue
|
||||
found=1
|
||||
printf '%-28s ' "${node_name}"
|
||||
if ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" "
|
||||
set -eu
|
||||
command -v tailscale >/dev/null
|
||||
sudo systemctl is-active --quiet tailscaled
|
||||
tailscale ip -4 >/dev/null
|
||||
timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}'
|
||||
" >/dev/null 2>&1; then
|
||||
printf 'ok - %s:%s reachable over tailnet\n' "${probe_host}" "${probe_port}"
|
||||
else
|
||||
printf 'fail - cannot reach %s:%s over tailnet\n' "${probe_host}" "${probe_port}"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
done <"${spec_file}"
|
||||
|
||||
if ((found == 0)); then
|
||||
echo "No Pimox workers were listed in ${spec_file}."
|
||||
return 1
|
||||
fi
|
||||
|
||||
((failures == 0))
|
||||
}
|
||||
|
||||
status_http() {
|
||||
local name="$1"
|
||||
local url="$2"
|
||||
|
|
@ -5455,6 +5505,7 @@ doctor_cluster() {
|
|||
status_systemd_units kubelet containerd docker
|
||||
status_kubernetes
|
||||
status_pimox_workers
|
||||
status_run "pimox worker tailnet egress" check_pimox_worker_tailnet_egress
|
||||
status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/"
|
||||
|
||||
printf '\n-- local CRI containers\n'
|
||||
|
|
@ -5480,6 +5531,7 @@ Kubeconfig path: ${KUBECONFIG_PATH}
|
|||
Traefik LoadBalancer IP: ${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}
|
||||
Pimox host: ${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}@${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}
|
||||
Worker storage: ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}
|
||||
Pimox worker tailnet probe: ${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-unset}}:${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}
|
||||
Known runbook: docs/runbooks/cluster-stop-start-failures.md
|
||||
EOF
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue