diff --git a/README.md b/README.md index 29e25bb..1a4df68 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID `9010` and VMID `9011`. +Pimox workers can join the tailnet during cluster worker bootstrap. Keep the +ephemeral or pre-authorized auth key outside Git, either in +`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in +`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with: + +```bash +LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./jeannie up +``` + +Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT +from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach +tailnet-only services. Override with +`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or +`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not +wanted. `./jeannie doctor-cluster` checks Pimox worker tailnet egress +against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and +`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`. + OpenWrt firewall VM automation is available as a standalone command because it attaches to both WAN and LAN bridges. Run `./jeannie openwrt` after `vmbr1` already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM @@ -456,6 +474,10 @@ next step: ./jeannie doctor-cluster ``` +`doctor-cluster` also checks whether Pimox workers have Tailscale installed, +`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet +probe service. + ## RPi Services The Raspberry Pi runs lightweight always-on services outside Kubernetes from diff --git a/README.md.tmpl b/README.md.tmpl index ff48975..83ab855 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -187,6 +187,24 @@ worker index `1` and VMID `9010` when `LAB_PIMOX_WORKER_COUNT=1`. Set must be left untouched, or set `LAB_PIMOX_WORKER_COUNT=2` to manage both VMID `9010` and VMID `9011`. +Pimox workers can join the tailnet during cluster worker bootstrap. Keep the +ephemeral or pre-authorized auth key outside Git, either in +`~/.config/homelab/tailscale-pimox-worker.authkey` on the Debian host or in +`LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY`. Then run the cluster pipeline with: + +```bash +LAB_PIMOX_WORKER_TAILSCALE_ENABLED=true ./{{ main_script }} up +``` + +Worker Tailscale defaults to `--accept-routes=true` and enables pod egress SNAT +from the Kubernetes pod CIDR to `tailscale0`, so pods on Pimox workers can reach +tailnet-only services. Override with +`LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES=false` or +`LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT=false` when that behavior is not +wanted. `./{{ main_script }} doctor-cluster` checks Pimox worker tailnet egress +against `${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-LAB_DEBIAN_TAILSCALE_IP}` and +`${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-LAB_GITEA_HTTP_PORT}`. + OpenWrt firewall VM automation is available as a standalone command because it attaches to both WAN and LAN bridges. Run `./{{ main_script }} openwrt` after `vmbr1` already exists on the Orange Pi. The pipeline downloads the OpenWrt ARM @@ -456,6 +474,10 @@ next step: ./{{ main_script }} doctor-cluster ``` +`doctor-cluster` also checks whether Pimox workers have Tailscale installed, +`tailscaled` active, a Tailscale IP, and TCP egress to the configured tailnet +probe service. + ## RPi Services The Raspberry Pi runs lightweight always-on services outside Kubernetes from diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index 2e0b9bc..af8287d 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -415,6 +415,10 @@ resource "null_resource" "kubeadm_worker" { worker_rejoin_reset_version = "2" node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) + worker_tailscale_version = "1" + worker_tailscale_enabled = tostring(var.worker_tailscale_enabled) + worker_tailscale_accept_routes = tostring(var.worker_tailscale_accept_routes) + worker_tailscale_pod_egress_snat = tostring(var.worker_tailscale_pod_egress_snat) tailscale_nodeport_version = "3" tailscale_nodeport_enabled = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false" tailscale_nodeport_peer_ip = var.tailscale_nodeport_access.peer_ip @@ -478,6 +482,84 @@ DOCKER_SOURCES sudo apt-mark hold containerd.io } +install_tailscale() { + if [ "${self.triggers.worker_tailscale_enabled}" != "true" ]; then + return 0 + fi + + if command -v tailscale >/dev/null 2>&1 && systemctl list-unit-files tailscaled.service >/dev/null 2>&1; then + sudo systemctl enable --now tailscaled + return 0 + fi + + codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")" + sudo install -d -m 0755 /usr/share/keyrings + curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.noarmor.gpg" | + sudo tee /usr/share/keyrings/tailscale-archive-keyring.gpg >/dev/null + curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.tailscale-keyring.list" | + sudo tee /etc/apt/sources.list.d/tailscale.list >/dev/null + sudo apt-get update + sudo apt-get install -y --no-install-recommends tailscale + sudo systemctl enable --now tailscaled +} + +configure_tailscale_pod_egress_snat() { + local enabled="$1" + local pod_cidr="$2" + + if [ "${self.triggers.worker_tailscale_enabled}" != "true" ] || [ "$enabled" != "true" ]; then + sudo systemctl disable --now homelab-tailscale-pod-egress.service >/dev/null 2>&1 || true + return 0 + fi + + sudo mkdir -p /usr/local/sbin /etc/sysctl.d + sudo tee /etc/sysctl.d/98-homelab-tailscale-pod-egress.conf >/dev/null </dev/null </dev/null +sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null +sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true + +if ! ip link show tailscale0 >/dev/null 2>&1; then + echo "tailscale0 is not present; skipping Tailscale pod egress SNAT" + exit 0 +fi + +iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -j ACCEPT 2>/dev/null || + iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -j ACCEPT +iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || + iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +iptables -t nat -C POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE 2>/dev/null || + iptables -t nat -A POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE +POD_EGRESS_SCRIPT_EOT + sudo chmod 0755 /usr/local/sbin/homelab-tailscale-pod-egress.sh + + sudo tee /etc/systemd/system/homelab-tailscale-pod-egress.service >/dev/null <<'POD_EGRESS_SERVICE_EOT' +[Unit] +Description=Homelab Tailscale pod egress SNAT +After=network-online.target tailscaled.service kubelet.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-tailscale-pod-egress.sh +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +POD_EGRESS_SERVICE_EOT + sudo systemctl daemon-reload + sudo systemctl enable homelab-tailscale-pod-egress.service >/dev/null + sudo systemctl restart homelab-tailscale-pod-egress.service +} + configure_node_dns() { dns_servers="${self.triggers.node_dns_servers}" if [ -z "$dns_servers" ]; then @@ -705,6 +787,7 @@ REGISTRY_EOT configure_node_dns install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common install_containerd_runtime +install_tailscale install_cni_plugins sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true @@ -739,6 +822,10 @@ if ! getent hosts "${self.triggers.node_name}" >/dev/null; then printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null fi +configure_tailscale_pod_egress_snat \ + "${self.triggers.worker_tailscale_pod_egress_snat}" \ + "${self.triggers.tailscale_nodeport_pod_cidr}" + configure_tailscale_nodeport_access() { local enabled="$1" local peer_ip="$2" @@ -887,6 +974,52 @@ EOT command = </dev/null 2>&1 && sudo systemctl is-active --quiet tailscaled'; then + echo "tailscale is not installed or tailscaled is not active on $ssh_target" >&2 + exit 1 +fi + +if ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \ + 'tailscale status --json >/dev/null 2>&1'; then + ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \ + "sudo tailscale set --accept-routes=$accept_routes" + exit 0 +fi + +auth_key="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}" +auth_key_file="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-$${HOME}/.config/homelab/tailscale-pimox-worker.authkey}" +if [ -z "$auth_key" ] && [ -s "$auth_key_file" ]; then + auth_key="$(tr -d '\n\r' <"$auth_key_file")" +fi +if [ -z "$auth_key" ]; then + echo "Pimox worker Tailscale is enabled, but no auth key was provided." >&2 + echo "Set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY or create $auth_key_file outside Git." >&2 + exit 1 +fi + +printf '%s\n' "$auth_key" | + ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \ + "read -r auth_key && sudo tailscale up --auth-key \"\$auth_key\" --hostname '$node_name' --accept-routes=$accept_routes --reset" + +ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$ssh_target" \ + 'sudo systemctl restart homelab-tailscale-pod-egress.service 2>/dev/null || true' +EOT + } + + provisioner "local-exec" { + command = </dev/null +sudo systemctl is-active --quiet tailscaled +tailscale ip -4 >/dev/null +timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' +" >/dev/null 2>&1; then + printf 'ok - %s:%s reachable over tailnet\n' "${probe_host}" "${probe_port}" + else + printf 'fail - cannot reach %s:%s over tailnet\n' "${probe_host}" "${probe_port}" + failures=$((failures + 1)) + fi + done <"${spec_file}" + + if ((found == 0)); then + echo "No Pimox workers were listed in ${spec_file}." + return 1 + fi + + ((failures == 0)) +} + status_http() { local name="$1" local url="$2" @@ -5455,6 +5505,7 @@ doctor_cluster() { status_systemd_units kubelet containerd docker status_kubernetes status_pimox_workers + status_run "pimox worker tailnet egress" check_pimox_worker_tailnet_egress status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" printf '\n-- local CRI containers\n' @@ -5480,6 +5531,7 @@ Kubeconfig path: ${KUBECONFIG_PATH} Traefik LoadBalancer IP: ${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml} Pimox host: ${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}@${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}} Worker storage: ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} +Pimox worker tailnet probe: ${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-unset}}:${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}} Known runbook: docs/runbooks/cluster-stop-start-failures.md EOF }