1151 lines
36 KiB
HCL
1151 lines
36 KiB
HCL
terraform {
|
|
required_version = ">= 1.0"
|
|
required_providers {
|
|
null = {
|
|
source = "hashicorp/null"
|
|
version = "~> 3.2"
|
|
}
|
|
external = {
|
|
source = "hashicorp/external"
|
|
version = "~> 2.3"
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_control_plane" {
|
|
triggers = {
|
|
node_name = var.control_plane_node_name
|
|
advertise_address = var.control_plane_advertise_address
|
|
pod_network_cidr = var.pod_network_cidr
|
|
kubeconfig_path = var.kubeconfig_path
|
|
kubeconfig_owner = var.kubeconfig_owner
|
|
registry_endpoint = var.registry_endpoint
|
|
registry_config_version = "8"
|
|
containerd_runtime = "docker-containerd-io"
|
|
cni_plugins_version = "2"
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
install_containerd_runtime() {
|
|
codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
|
|
arch="$(dpkg --print-architecture)"
|
|
|
|
sudo install -d -m 0755 /etc/apt/keyrings
|
|
sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
sudo chmod a+r /etc/apt/keyrings/docker.asc
|
|
sudo rm -f /etc/apt/sources.list.d/docker.list
|
|
sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
|
|
Types: deb
|
|
URIs: https://download.docker.com/linux/debian
|
|
Suites: $${codename}
|
|
Components: stable
|
|
Architectures: $${arch}
|
|
Signed-By: /etc/apt/keyrings/docker.asc
|
|
DOCKER_SOURCES
|
|
|
|
if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
|
|
sudo apt-get remove -y containerd runc || true
|
|
fi
|
|
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends containerd.io
|
|
sudo apt-mark hold containerd.io
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
remove_containerd_section() {
|
|
local section="$1"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v section="$section" '
|
|
$0 == section { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_registry_config_path() {
|
|
local plugin="$1"
|
|
local append_section="$2"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v plugin="$plugin" -v append_section="$append_section" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_target_registry(line) {
|
|
return is_table(line) &&
|
|
index(line, plugin) > 0 &&
|
|
line ~ /[.]registry[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_target = 0
|
|
found = 0
|
|
wrote = 0
|
|
}
|
|
is_target_registry($0) {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 1
|
|
found = 1
|
|
wrote = 0
|
|
print
|
|
next
|
|
}
|
|
in_target && is_table($0) {
|
|
if (!wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 0
|
|
wrote = 0
|
|
}
|
|
in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
wrote = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
if (!found) {
|
|
print ""
|
|
print append_section
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
containerd_config_version() {
|
|
sudo awk -F= '
|
|
/^[[:space:]]*version[[:space:]]*=/ {
|
|
gsub(/[[:space:]]/, "", $2)
|
|
print $2
|
|
exit
|
|
}
|
|
' /etc/containerd/config.toml
|
|
}
|
|
|
|
reset_containerd_registry_tables() {
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk '
|
|
function is_registry_table(line) {
|
|
return line ~ /^\[plugins\./ &&
|
|
line ~ /\.registry([.\]]|$)/ &&
|
|
(line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
|
|
line ~ /io[.]containerd[.]cri[.]v1[.]images/)
|
|
}
|
|
is_registry_table($0) { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_cni_bin_dir() {
|
|
local config_version
|
|
local tmp
|
|
|
|
config_version="$(containerd_config_version)"
|
|
tmp="$(mktemp)"
|
|
sudo awk -v config_version="$config_version" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_cni_table(line) {
|
|
return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_cni = 0
|
|
found = 0
|
|
}
|
|
is_cni_table($0) {
|
|
in_cni = 1
|
|
print
|
|
next
|
|
}
|
|
in_cni && is_table($0) {
|
|
in_cni = 0
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
|
|
sub(/=.*/, "= \"/opt/cni/bin\"")
|
|
found = 1
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
|
|
sub(/=.*/, "= [\"/opt/cni/bin\"]")
|
|
found = 1
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!found) {
|
|
print ""
|
|
if (config_version == "3") {
|
|
print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
|
|
print " bin_dirs = [\"/opt/cni/bin\"]"
|
|
} else {
|
|
print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
|
|
print " bin_dir = \"/opt/cni/bin\""
|
|
}
|
|
print " conf_dir = \"/etc/cni/net.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
install_cni_plugins() {
|
|
local plugin
|
|
|
|
sudo mkdir -p /opt/cni/bin
|
|
sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
|
|
sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
|
|
if [ -d /usr/lib/cni ]; then
|
|
for plugin in /usr/lib/cni/*; do
|
|
[ -f "$plugin" ] && [ -x "$plugin" ] || continue
|
|
sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
|
|
done
|
|
fi
|
|
}
|
|
|
|
configure_containerd_registry() {
|
|
local registry_endpoint="$1"
|
|
local config_version
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
|
|
ensure_containerd_cni_bin_dir
|
|
config_version="$(containerd_config_version)"
|
|
if [ "$config_version" = "3" ]; then
|
|
ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
|
|
else
|
|
ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
|
|
fi
|
|
|
|
sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
|
|
sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
|
|
server = "http://$registry_endpoint"
|
|
|
|
[host."http://$registry_endpoint"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
if ! sudo containerd config dump >/dev/null; then
|
|
sudo containerd config dump || true
|
|
exit 1
|
|
fi
|
|
if ! sudo systemctl restart containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
|
|
install_containerd_runtime
|
|
install_cni_plugins
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
configure_containerd_registry "${self.triggers.registry_endpoint}"
|
|
|
|
reset_control_plane_join_state() {
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo systemctl stop containerd 2>/dev/null || true
|
|
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
sudo mkdir -p /var/lib/kubelet /var/lib/cni /etc/cni/net.d
|
|
|
|
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
|
|
if ! sudo systemctl start containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}"
|
|
for path in "$${pv_dirs[@]}"; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ]; then
|
|
reset_control_plane_join_state
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ]; then
|
|
if ! sudo kubeadm init \
|
|
--pod-network-cidr=${self.triggers.pod_network_cidr} \
|
|
--node-name=${self.triggers.node_name} \
|
|
--apiserver-advertise-address=${self.triggers.advertise_address}; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")"
|
|
sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}"
|
|
sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}"
|
|
|
|
kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true
|
|
EOT
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "worker_nodes_required" {
|
|
triggers = {
|
|
worker_count = tostring(length(var.worker_nodes))
|
|
}
|
|
|
|
lifecycle {
|
|
precondition {
|
|
condition = var.allow_empty_worker_nodes || length(var.worker_nodes) > 0
|
|
error_message = "worker_nodes is empty. Pass -var-file=/path/to/.lab/cluster-workers.auto.tfvars.json or set allow_empty_worker_nodes=true for an intentional control-plane-only cluster."
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_worker" {
|
|
for_each = var.worker_nodes
|
|
|
|
depends_on = [
|
|
null_resource.kubeadm_control_plane,
|
|
null_resource.worker_nodes_required,
|
|
]
|
|
|
|
triggers = {
|
|
node_name = each.value.node_name
|
|
host = each.value.host
|
|
user = each.value.user
|
|
ssh_key_path = each.value.ssh_key_path
|
|
registry_endpoint = var.registry_endpoint
|
|
registry_config_version = "8"
|
|
containerd_runtime = "docker-containerd-io"
|
|
cni_plugins_version = "2"
|
|
force_rejoin = tostring(var.force_worker_rejoin)
|
|
worker_rejoin_reset_version = "2"
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
worker_tailscale_version = "1"
|
|
worker_tailscale_enabled = tostring(var.worker_tailscale_enabled)
|
|
worker_tailscale_accept_routes = tostring(var.worker_tailscale_accept_routes)
|
|
worker_tailscale_pod_egress_snat = tostring(var.worker_tailscale_pod_egress_snat)
|
|
tailscale_nodeport_version = "3"
|
|
tailscale_nodeport_enabled = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false"
|
|
tailscale_nodeport_peer_ip = var.tailscale_nodeport_access.peer_ip
|
|
tailscale_nodeport_node_tailscale_ip = var.tailscale_nodeport_access.node_tailscale_ip
|
|
tailscale_nodeport_pod_cidr = var.tailscale_nodeport_access.pod_cidr
|
|
tailscale_nodeport_node_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.node_port], var.tailscale_nodeport_extra_ports)))
|
|
tailscale_nodeport_target_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.target_port], var.tailscale_nodeport_extra_target_ports)))
|
|
tailscale_subnet_routes_version = "1"
|
|
tailscale_subnet_routes_enabled = var.tailscale_subnet_routes.enabled && each.key == var.tailscale_subnet_routes.worker_key ? "true" : "false"
|
|
tailscale_subnet_routes = join(",", var.tailscale_subnet_routes.routes)
|
|
}
|
|
|
|
connection {
|
|
type = "ssh"
|
|
user = self.triggers.user
|
|
private_key = file(self.triggers.ssh_key_path)
|
|
host = self.triggers.host
|
|
timeout = "15m"
|
|
}
|
|
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<EOT
|
|
set -eu
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
install_containerd_runtime() {
|
|
codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
|
|
arch="$(dpkg --print-architecture)"
|
|
|
|
sudo install -d -m 0755 /etc/apt/keyrings
|
|
sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
|
|
sudo chmod a+r /etc/apt/keyrings/docker.asc
|
|
sudo rm -f /etc/apt/sources.list.d/docker.list
|
|
sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
|
|
Types: deb
|
|
URIs: https://download.docker.com/linux/debian
|
|
Suites: $${codename}
|
|
Components: stable
|
|
Architectures: $${arch}
|
|
Signed-By: /etc/apt/keyrings/docker.asc
|
|
DOCKER_SOURCES
|
|
|
|
if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
|
|
sudo apt-get remove -y containerd runc || true
|
|
fi
|
|
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends containerd.io
|
|
sudo apt-mark hold containerd.io
|
|
}
|
|
|
|
install_tailscale() {
|
|
if [ "${self.triggers.worker_tailscale_enabled}" != "true" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if command -v tailscale >/dev/null 2>&1 && systemctl list-unit-files tailscaled.service >/dev/null 2>&1; then
|
|
sudo systemctl enable --now tailscaled
|
|
return 0
|
|
fi
|
|
|
|
codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
|
|
sudo install -d -m 0755 /usr/share/keyrings
|
|
curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.noarmor.gpg" |
|
|
sudo tee /usr/share/keyrings/tailscale-archive-keyring.gpg >/dev/null
|
|
curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.tailscale-keyring.list" |
|
|
sudo tee /etc/apt/sources.list.d/tailscale.list >/dev/null
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends tailscale
|
|
sudo systemctl enable --now tailscaled
|
|
}
|
|
|
|
configure_tailscale_pod_egress_snat() {
|
|
local enabled="$1"
|
|
local pod_cidr="$2"
|
|
|
|
if [ "${self.triggers.worker_tailscale_enabled}" != "true" ] || [ "$enabled" != "true" ]; then
|
|
sudo systemctl disable --now homelab-tailscale-pod-egress.service >/dev/null 2>&1 || true
|
|
return 0
|
|
fi
|
|
|
|
sudo mkdir -p /usr/local/sbin /etc/sysctl.d
|
|
sudo tee /etc/sysctl.d/98-homelab-tailscale-pod-egress.conf >/dev/null <<POD_EGRESS_SYSCTL_EOT
|
|
net.ipv4.ip_forward = 1
|
|
net.ipv4.conf.all.rp_filter = 0
|
|
net.ipv4.conf.tailscale0.rp_filter = 0
|
|
POD_EGRESS_SYSCTL_EOT
|
|
|
|
sudo tee /usr/local/sbin/homelab-tailscale-pod-egress.sh >/dev/null <<POD_EGRESS_SCRIPT_EOT
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
|
|
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true
|
|
|
|
if ! ip link show tailscale0 >/dev/null 2>&1; then
|
|
echo "tailscale0 is not present; skipping Tailscale pod egress SNAT"
|
|
exit 0
|
|
fi
|
|
|
|
iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -j ACCEPT
|
|
iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
|
iptables -t nat -C POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE 2>/dev/null ||
|
|
iptables -t nat -A POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE
|
|
POD_EGRESS_SCRIPT_EOT
|
|
sudo chmod 0755 /usr/local/sbin/homelab-tailscale-pod-egress.sh
|
|
|
|
sudo tee /etc/systemd/system/homelab-tailscale-pod-egress.service >/dev/null <<'POD_EGRESS_SERVICE_EOT'
|
|
[Unit]
|
|
Description=Homelab Tailscale pod egress SNAT
|
|
After=network-online.target tailscaled.service kubelet.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/sbin/homelab-tailscale-pod-egress.sh
|
|
RemainAfterExit=yes
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
POD_EGRESS_SERVICE_EOT
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable homelab-tailscale-pod-egress.service >/dev/null
|
|
sudo systemctl restart homelab-tailscale-pod-egress.service
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
remove_containerd_section() {
|
|
local section="$1"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v section="$section" '
|
|
$0 == section { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_registry_config_path() {
|
|
local plugin="$1"
|
|
local append_section="$2"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v plugin="$plugin" -v append_section="$append_section" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_target_registry(line) {
|
|
return is_table(line) &&
|
|
index(line, plugin) > 0 &&
|
|
line ~ /[.]registry[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_target = 0
|
|
found = 0
|
|
wrote = 0
|
|
}
|
|
is_target_registry($0) {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 1
|
|
found = 1
|
|
wrote = 0
|
|
print
|
|
next
|
|
}
|
|
in_target && is_table($0) {
|
|
if (!wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 0
|
|
wrote = 0
|
|
}
|
|
in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
wrote = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
if (!found) {
|
|
print ""
|
|
print append_section
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
containerd_config_version() {
|
|
sudo awk -F= '
|
|
/^[[:space:]]*version[[:space:]]*=/ {
|
|
gsub(/[[:space:]]/, "", $2)
|
|
print $2
|
|
exit
|
|
}
|
|
' /etc/containerd/config.toml
|
|
}
|
|
|
|
reset_containerd_registry_tables() {
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk '
|
|
function is_registry_table(line) {
|
|
return line ~ /^\[plugins\./ &&
|
|
line ~ /\.registry([.\]]|$)/ &&
|
|
(line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
|
|
line ~ /io[.]containerd[.]cri[.]v1[.]images/)
|
|
}
|
|
is_registry_table($0) { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_cni_bin_dir() {
|
|
local config_version
|
|
local tmp
|
|
|
|
config_version="$(containerd_config_version)"
|
|
tmp="$(mktemp)"
|
|
sudo awk -v config_version="$config_version" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_cni_table(line) {
|
|
return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_cni = 0
|
|
found = 0
|
|
}
|
|
is_cni_table($0) {
|
|
in_cni = 1
|
|
print
|
|
next
|
|
}
|
|
in_cni && is_table($0) {
|
|
in_cni = 0
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
|
|
sub(/=.*/, "= \"/opt/cni/bin\"")
|
|
found = 1
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
|
|
sub(/=.*/, "= [\"/opt/cni/bin\"]")
|
|
found = 1
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!found) {
|
|
print ""
|
|
if (config_version == "3") {
|
|
print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
|
|
print " bin_dirs = [\"/opt/cni/bin\"]"
|
|
} else {
|
|
print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
|
|
print " bin_dir = \"/opt/cni/bin\""
|
|
}
|
|
print " conf_dir = \"/etc/cni/net.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
install_cni_plugins() {
|
|
local plugin
|
|
|
|
sudo mkdir -p /opt/cni/bin
|
|
sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
|
|
sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
|
|
if [ -d /usr/lib/cni ]; then
|
|
for plugin in /usr/lib/cni/*; do
|
|
[ -f "$plugin" ] && [ -x "$plugin" ] || continue
|
|
sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
|
|
done
|
|
fi
|
|
}
|
|
|
|
configure_containerd_registry() {
|
|
local registry_endpoint="$1"
|
|
local config_version
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
|
|
ensure_containerd_cni_bin_dir
|
|
config_version="$(containerd_config_version)"
|
|
if [ "$config_version" = "3" ]; then
|
|
ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
|
|
else
|
|
ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
|
|
fi
|
|
|
|
sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
|
|
sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
|
|
server = "http://$registry_endpoint"
|
|
|
|
[host."http://$registry_endpoint"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
if ! sudo containerd config dump >/dev/null; then
|
|
sudo containerd config dump || true
|
|
exit 1
|
|
fi
|
|
if ! sudo systemctl restart containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
|
|
install_containerd_runtime
|
|
install_tailscale
|
|
install_cni_plugins
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
configure_tailscale_pod_egress_snat \
|
|
"${self.triggers.worker_tailscale_pod_egress_snat}" \
|
|
"${self.triggers.tailscale_nodeport_pod_cidr}"
|
|
|
|
configure_tailscale_nodeport_access() {
|
|
local enabled="$1"
|
|
local peer_ip="$2"
|
|
local node_tailscale_ip="$3"
|
|
local pod_cidr="$4"
|
|
local node_ports="$5"
|
|
local target_ports="$6"
|
|
|
|
if [ "$enabled" != "true" ]; then
|
|
return 0
|
|
fi
|
|
|
|
sudo mkdir -p /usr/local/sbin /etc/sysctl.d
|
|
sudo tee /etc/sysctl.d/98-homelab-tailscale-nodeport.conf >/dev/null <<NODEPORT_SYSCTL_EOT
|
|
net.ipv4.conf.all.rp_filter = 0
|
|
net.ipv4.conf.tailscale0.rp_filter = 0
|
|
NODEPORT_SYSCTL_EOT
|
|
|
|
sudo tee /usr/local/sbin/homelab-tailscale-nodeport.sh >/dev/null <<NODEPORT_SCRIPT_EOT
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
|
|
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true
|
|
|
|
if ! ip link show tailscale0 >/dev/null 2>&1; then
|
|
echo "tailscale0 is not present; skipping Tailscale NodePort routing"
|
|
exit 0
|
|
fi
|
|
|
|
ip route replace "$peer_ip/32" dev tailscale0 src "$node_tailscale_ip"
|
|
|
|
for node_port in $node_ports; do
|
|
iptables -C INPUT -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT 2>/dev/null ||
|
|
iptables -I INPUT 1 -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT
|
|
done
|
|
for target_port in $target_ports; do
|
|
iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT
|
|
done
|
|
iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
|
for target_port in $target_ports; do
|
|
iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null ||
|
|
iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE
|
|
done
|
|
NODEPORT_SCRIPT_EOT
|
|
sudo chmod 0755 /usr/local/sbin/homelab-tailscale-nodeport.sh
|
|
|
|
sudo tee /etc/systemd/system/homelab-tailscale-nodeport.service >/dev/null <<'NODEPORT_SERVICE_EOT'
|
|
[Unit]
|
|
Description=Homelab Tailscale NodePort routing
|
|
After=network-online.target tailscaled.service kubelet.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/sbin/homelab-tailscale-nodeport.sh
|
|
RemainAfterExit=yes
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
NODEPORT_SERVICE_EOT
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable homelab-tailscale-nodeport.service >/dev/null
|
|
sudo systemctl restart homelab-tailscale-nodeport.service
|
|
}
|
|
|
|
configure_tailscale_nodeport_access \
|
|
"${self.triggers.tailscale_nodeport_enabled}" \
|
|
"${self.triggers.tailscale_nodeport_peer_ip}" \
|
|
"${self.triggers.tailscale_nodeport_node_tailscale_ip}" \
|
|
"${self.triggers.tailscale_nodeport_pod_cidr}" \
|
|
"${self.triggers.tailscale_nodeport_node_ports}" \
|
|
"${self.triggers.tailscale_nodeport_target_ports}"
|
|
|
|
configure_tailscale_subnet_routes() {
|
|
local enabled="$1"
|
|
local routes="$2"
|
|
|
|
if [ "$enabled" != "true" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo "tailscale is required to advertise subnet routes but is not installed on this worker." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$routes" ]; then
|
|
echo "tailscale subnet route advertisement is enabled but no routes were configured." >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo tailscale set --advertise-routes="$routes"
|
|
}
|
|
|
|
configure_tailscale_subnet_routes \
|
|
"${self.triggers.tailscale_subnet_routes_enabled}" \
|
|
"${self.triggers.tailscale_subnet_routes}"
|
|
|
|
configure_containerd_registry "${self.triggers.registry_endpoint}"
|
|
|
|
reset_worker_join_state() {
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo systemctl stop containerd 2>/dev/null || true
|
|
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/
|
|
sudo mkdir -p /etc/cni/net.d
|
|
sudo rm -rf /etc/cni/net.d/*
|
|
sudo mkdir -p /var/lib/kubelet /var/lib/cni
|
|
|
|
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
|
|
if ! sudo systemctl start containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
pv_dirs="${self.triggers.persistent_volume_dirs}"
|
|
IFS=','
|
|
for path in $pv_dirs; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ "${self.triggers.force_rejoin}" = "true" ]; then
|
|
reset_worker_join_state
|
|
fi
|
|
|
|
if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then
|
|
reset_worker_join_state
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
|
|
reset_worker_join_state
|
|
fi
|
|
EOT
|
|
]
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/usr/bin/env", "bash", "-c"]
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
if [ '${self.triggers.worker_tailscale_enabled}' != 'true' ]; then
|
|
exit 0
|
|
fi
|
|
|
|
ssh_target='${self.triggers.user}@${self.triggers.host}'
|
|
ssh_key='${self.triggers.ssh_key_path}'
|
|
node_name='${self.triggers.node_name}'
|
|
accept_routes='${self.triggers.worker_tailscale_accept_routes}'
|
|
ssh_host="$${ssh_target#*@}"
|
|
known_hosts_file="$${TF_DATA_DIR:-.terraform}/homelab-worker-known_hosts"
|
|
|
|
mkdir -p "$(dirname "$known_hosts_file")"
|
|
touch "$known_hosts_file"
|
|
chmod 0600 "$known_hosts_file"
|
|
ssh-keygen -R "$ssh_host" -f "$known_hosts_file" >/dev/null 2>&1 || true
|
|
|
|
worker_ssh() {
|
|
ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$ssh_target" "$@"
|
|
}
|
|
|
|
if ! worker_ssh \
|
|
'command -v tailscale >/dev/null 2>&1 && sudo systemctl is-active --quiet tailscaled'; then
|
|
echo "tailscale is not installed or tailscaled is not active on $ssh_target" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if worker_ssh \
|
|
'test -n "$(tailscale ip -4 2>/dev/null | head -n 1)"'; then
|
|
worker_ssh "sudo tailscale set --accept-routes=$accept_routes"
|
|
exit 0
|
|
fi
|
|
|
|
auth_key="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}"
|
|
auth_key_file="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-$${HOME}/.config/homelab/tailscale-pimox-worker.authkey}"
|
|
if [ -z "$auth_key" ] && [ -s "$auth_key_file" ]; then
|
|
auth_key="$(tr -d '\n\r' <"$auth_key_file")"
|
|
fi
|
|
if [ -z "$auth_key" ]; then
|
|
echo "Pimox worker Tailscale is enabled, but no auth key was provided." >&2
|
|
echo "Set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY or create $auth_key_file outside Git." >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\n' "$auth_key" |
|
|
worker_ssh \
|
|
"read -r auth_key && sudo tailscale up --auth-key \"\$auth_key\" --hostname '$node_name' --accept-routes=$accept_routes --reset"
|
|
|
|
worker_ssh 'sudo systemctl restart homelab-tailscale-pod-egress.service 2>/dev/null || true'
|
|
EOT
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/usr/bin/env", "bash", "-c"]
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
ssh_target='${self.triggers.user}@${self.triggers.host}'
|
|
ssh_key='${self.triggers.ssh_key_path}'
|
|
node_name='${self.triggers.node_name}'
|
|
ssh_host="$${ssh_target#*@}"
|
|
known_hosts_file="$${TF_DATA_DIR:-.terraform}/homelab-worker-known_hosts"
|
|
|
|
mkdir -p "$(dirname "$known_hosts_file")"
|
|
touch "$known_hosts_file"
|
|
chmod 0600 "$known_hosts_file"
|
|
ssh-keygen -R "$ssh_host" -f "$known_hosts_file" >/dev/null 2>&1 || true
|
|
|
|
worker_ssh() {
|
|
ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$ssh_target" "$@"
|
|
}
|
|
|
|
if worker_ssh \
|
|
'test -f /etc/kubernetes/kubelet.conf && test -f /var/lib/kubelet/config.yaml'; then
|
|
exit 0
|
|
fi
|
|
|
|
join_cmd="$(sudo env KUBECONFIG=/etc/kubernetes/admin.conf kubeadm token create --print-join-command)"
|
|
remote_join_script="$(mktemp)"
|
|
trap 'rm -f "$remote_join_script"' EXIT
|
|
|
|
cat >"$remote_join_script" <<JOIN_EOT
|
|
set -eu
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
if ! sudo $join_cmd --node-name=$node_name; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ] || [ ! -f /var/lib/kubelet/config.yaml ]; then
|
|
echo "kubeadm join completed without creating kubelet configuration" >&2
|
|
sudo ls -la /etc/kubernetes /var/lib/kubelet 2>/dev/null || true
|
|
sudo systemctl status kubelet containerd --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
JOIN_EOT
|
|
|
|
worker_ssh 'bash -s' <"$remote_join_script"
|
|
EOT
|
|
}
|
|
}
|
|
|
|
locals {
|
|
control_plane_node_label_pairs = [
|
|
for label, value in var.control_plane_node_labels : {
|
|
node_name = var.control_plane_node_name
|
|
label = label
|
|
value = value
|
|
}
|
|
]
|
|
|
|
worker_node_label_pairs = flatten([
|
|
for worker_key, worker in var.worker_nodes : [
|
|
for label, value in lookup(var.worker_node_labels, worker_key, {}) : {
|
|
node_name = worker.node_name
|
|
label = label
|
|
value = value
|
|
}
|
|
]
|
|
])
|
|
|
|
node_label_pairs = concat(local.control_plane_node_label_pairs, local.worker_node_label_pairs)
|
|
}
|
|
|
|
resource "null_resource" "node_labels" {
|
|
for_each = {
|
|
for pair in local.node_label_pairs : "${pair.node_name}/${pair.label}" => pair
|
|
}
|
|
|
|
depends_on = [
|
|
null_resource.kubeadm_control_plane,
|
|
null_resource.kubeadm_worker,
|
|
]
|
|
|
|
triggers = {
|
|
kubeconfig_path = var.kubeconfig_path
|
|
node_name = each.value.node_name
|
|
label = each.value.label
|
|
value = each.value.value
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
environment = {
|
|
KUBECONFIG_PATH = self.triggers.kubeconfig_path
|
|
NODE_NAME = self.triggers.node_name
|
|
NODE_LABEL = self.triggers.label
|
|
NODE_LABEL_VALUE = self.triggers.value
|
|
}
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
for _ in $(seq 1 60); do
|
|
if kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null
|
|
kubectl --kubeconfig "$${KUBECONFIG_PATH}" label node "$${NODE_NAME}" "$${NODE_LABEL}=$${NODE_LABEL_VALUE}" --overwrite
|
|
EOT
|
|
}
|
|
}
|
|
|
|
output "kubeconfig_path" {
|
|
value = var.kubeconfig_path
|
|
}
|
|
|
|
output "pod_network_cidr" {
|
|
value = var.pod_network_cidr
|
|
}
|