terraform { required_version = ">= 1.0" required_providers { null = { source = "hashicorp/null" version = "~> 3.2" } external = { source = "hashicorp/external" version = "~> 2.3" } } } resource "null_resource" "kubeadm_control_plane" { triggers = { node_name = var.control_plane_node_name advertise_address = var.control_plane_advertise_address pod_network_cidr = var.pod_network_cidr kubeconfig_path = var.kubeconfig_path kubeconfig_owner = var.kubeconfig_owner registry_endpoint = var.registry_endpoint registry_config_version = "6" node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) } provisioner "local-exec" { interpreter = ["/bin/bash", "-lc"] command = </dev/null | grep -q "install ok installed"; then missing_packages="$missing_packages $package" fi done if [ -n "$missing_packages" ]; then sudo apt-get update sudo apt-get install -y --no-install-recommends $missing_packages fi } configure_node_dns() { dns_servers="${self.triggers.node_dns_servers}" if [ -z "$dns_servers" ]; then return 0 fi if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then sudo mkdir -p /etc/systemd/resolved.conf.d { echo "[Resolve]" printf 'DNS=%s\n' "$dns_servers" printf 'FallbackDNS=%s\n' "$dns_servers" echo "DNSSEC=no" } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null sudo systemctl restart systemd-resolved 2>/dev/null || true fi if ! getent hosts quay.io >/dev/null 2>&1; then sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true sudo rm -f /etc/resolv.conf for server in $dns_servers; do printf 'nameserver %s\n' "$server" done | sudo tee /etc/resolv.conf >/dev/null fi } remove_containerd_section() { local section="$1" local tmp tmp="$(mktemp)" sudo awk -v section="$section" ' $0 == section { skip = 1; next } skip && /^\[/ { skip = 0 } !skip { print } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } ensure_containerd_registry_config_path() { local plugin="$1" local append_section="$2" local tmp tmp="$(mktemp)" sudo awk -v plugin="$plugin" -v append_section="$append_section" ' function is_table(line) { return line ~ /^[[:space:]]*\[/ } function is_target_registry(line) { return is_table(line) && index(line, plugin) > 0 && line ~ /[.]registry[[:space:]]*\]/ } BEGIN { in_target = 0 found = 0 wrote = 0 } is_target_registry($0) { if (in_target && !wrote) { print " config_path = \"/etc/containerd/certs.d\"" } in_target = 1 found = 1 wrote = 0 print next } in_target && is_table($0) { if (!wrote) { print " config_path = \"/etc/containerd/certs.d\"" } in_target = 0 wrote = 0 } in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ { print " config_path = \"/etc/containerd/certs.d\"" wrote = 1 next } { print } END { if (in_target && !wrote) { print " config_path = \"/etc/containerd/certs.d\"" } if (!found) { print "" print append_section print " config_path = \"/etc/containerd/certs.d\"" } } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } containerd_config_version() { sudo awk -F= ' /^[[:space:]]*version[[:space:]]*=/ { gsub(/[[:space:]]/, "", $2) print $2 exit } ' /etc/containerd/config.toml } reset_containerd_registry_tables() { local tmp tmp="$(mktemp)" sudo awk ' function is_registry_table(line) { return line ~ /^\[plugins\./ && line ~ /\.registry([.\]]|$)/ && (line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ || line ~ /io[.]containerd[.]cri[.]v1[.]images/) } is_registry_table($0) { skip = 1; next } skip && /^\[/ { skip = 0 } !skip { print } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } configure_containerd_registry() { local registry_endpoint="$1" local config_version sudo mkdir -p /etc/containerd sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml config_version="$(containerd_config_version)" if [ "$config_version" = "3" ]; then ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]' else ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]' fi sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint" sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null </dev/null; then sudo containerd config dump || true exit 1 fi if ! sudo systemctl restart containerd; then sudo systemctl status containerd --no-pager -l || true sudo journalctl -u containerd --no-pager -n 160 || true exit 1 fi } configure_node_dns install_missing_packages open-iscsi nfs-common sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true sudo swapoff -a || true sudo awk ' /^[[:space:]]*#/ { print; next } $3 == "swap" { print "# kubeadm-disabled " $0; next } { print } ' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null sudo mv /etc/fstab.kubeadm /etc/fstab sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT' overlay br_netfilter MODULES_EOT sudo modprobe overlay || true sudo modprobe br_netfilter || true sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT' net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 SYSCTL_EOT sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null fi if ! getent hosts "${self.triggers.node_name}" >/dev/null; then printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null fi configure_containerd_registry "${self.triggers.registry_endpoint}" IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}" for path in "$${pv_dirs[@]}"; do sudo mkdir -p "$path" sudo chmod 0775 "$path" done if [ ! -f /etc/kubernetes/admin.conf ] && [ -d /etc/kubernetes ]; then sudo kubeadm reset --force || true sudo systemctl stop kubelet 2>/dev/null || true sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d fi if [ ! -f /etc/kubernetes/admin.conf ]; then sudo systemctl stop kubelet 2>/dev/null || true if ! sudo kubeadm init \ --pod-network-cidr=${self.triggers.pod_network_cidr} \ --node-name=${self.triggers.node_name} \ --apiserver-advertise-address=${self.triggers.advertise_address}; then sudo systemctl status kubelet --no-pager -l || true sudo journalctl -u kubelet --no-pager -n 160 || true exit 1 fi fi mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")" sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}" sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}" kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true EOT } } data "external" "kubeadm_join_command" { depends_on = [null_resource.kubeadm_control_plane] program = [ "bash", "-lc", </dev/null | grep -q "install ok installed"; then missing_packages="$missing_packages $package" fi done if [ -n "$missing_packages" ]; then sudo apt-get update sudo apt-get install -y --no-install-recommends $missing_packages fi } configure_node_dns() { dns_servers="${self.triggers.node_dns_servers}" if [ -z "$dns_servers" ]; then return 0 fi if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then sudo mkdir -p /etc/systemd/resolved.conf.d { echo "[Resolve]" printf 'DNS=%s\n' "$dns_servers" printf 'FallbackDNS=%s\n' "$dns_servers" echo "DNSSEC=no" } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null sudo systemctl restart systemd-resolved 2>/dev/null || true fi if ! getent hosts quay.io >/dev/null 2>&1; then sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true sudo rm -f /etc/resolv.conf for server in $dns_servers; do printf 'nameserver %s\n' "$server" done | sudo tee /etc/resolv.conf >/dev/null fi } remove_containerd_section() { local section="$1" local tmp tmp="$(mktemp)" sudo awk -v section="$section" ' $0 == section { skip = 1; next } skip && /^\[/ { skip = 0 } !skip { print } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } ensure_containerd_registry_config_path() { local plugin="$1" local append_section="$2" local tmp tmp="$(mktemp)" sudo awk -v plugin="$plugin" -v append_section="$append_section" ' function is_table(line) { return line ~ /^[[:space:]]*\[/ } function is_target_registry(line) { return is_table(line) && index(line, plugin) > 0 && line ~ /[.]registry[[:space:]]*\]/ } BEGIN { in_target = 0 found = 0 wrote = 0 } is_target_registry($0) { if (in_target && !wrote) { print " config_path = \"/etc/containerd/certs.d\"" } in_target = 1 found = 1 wrote = 0 print next } in_target && is_table($0) { if (!wrote) { print " config_path = \"/etc/containerd/certs.d\"" } in_target = 0 wrote = 0 } in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ { print " config_path = \"/etc/containerd/certs.d\"" wrote = 1 next } { print } END { if (in_target && !wrote) { print " config_path = \"/etc/containerd/certs.d\"" } if (!found) { print "" print append_section print " config_path = \"/etc/containerd/certs.d\"" } } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } containerd_config_version() { sudo awk -F= ' /^[[:space:]]*version[[:space:]]*=/ { gsub(/[[:space:]]/, "", $2) print $2 exit } ' /etc/containerd/config.toml } reset_containerd_registry_tables() { local tmp tmp="$(mktemp)" sudo awk ' function is_registry_table(line) { return line ~ /^\[plugins\./ && line ~ /\.registry([.\]]|$)/ && (line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ || line ~ /io[.]containerd[.]cri[.]v1[.]images/) } is_registry_table($0) { skip = 1; next } skip && /^\[/ { skip = 0 } !skip { print } ' /etc/containerd/config.toml > "$tmp" sudo mv "$tmp" /etc/containerd/config.toml } configure_containerd_registry() { local registry_endpoint="$1" local config_version sudo mkdir -p /etc/containerd sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml config_version="$(containerd_config_version)" if [ "$config_version" = "3" ]; then ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]' else ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]' fi sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint" sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null </dev/null; then sudo containerd config dump || true exit 1 fi if ! sudo systemctl restart containerd; then sudo systemctl status containerd --no-pager -l || true sudo journalctl -u containerd --no-pager -n 160 || true exit 1 fi } configure_node_dns install_missing_packages open-iscsi nfs-common sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true sudo swapoff -a || true sudo awk ' /^[[:space:]]*#/ { print; next } $3 == "swap" { print "# kubeadm-disabled " $0; next } { print } ' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null sudo mv /etc/fstab.kubeadm /etc/fstab sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT' overlay br_netfilter MODULES_EOT sudo modprobe overlay || true sudo modprobe br_netfilter || true sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT' net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 net.ipv4.ip_forward = 1 SYSCTL_EOT sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null fi if ! getent hosts "${self.triggers.node_name}" >/dev/null; then printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null fi configure_tailscale_nodeport_access() { local enabled="$1" local peer_ip="$2" local node_tailscale_ip="$3" local pod_cidr="$4" local node_ports="$5" local target_ports="$6" if [ "$enabled" != "true" ]; then return 0 fi sudo mkdir -p /usr/local/sbin /etc/sysctl.d sudo tee /etc/sysctl.d/98-homelab-tailscale-nodeport.conf >/dev/null </dev/null </dev/null sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true if ! ip link show tailscale0 >/dev/null 2>&1; then echo "tailscale0 is not present; skipping Tailscale NodePort routing" exit 0 fi ip route replace "$peer_ip/32" dev tailscale0 src "$node_tailscale_ip" for node_port in $node_ports; do iptables -C INPUT -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT 2>/dev/null || iptables -I INPUT 1 -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT done for target_port in $target_ports; do iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT done iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT for target_port in $target_ports; do iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null || iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE done NODEPORT_SCRIPT_EOT sudo chmod 0755 /usr/local/sbin/homelab-tailscale-nodeport.sh sudo tee /etc/systemd/system/homelab-tailscale-nodeport.service >/dev/null <<'NODEPORT_SERVICE_EOT' [Unit] Description=Homelab Tailscale NodePort routing After=network-online.target tailscaled.service kubelet.service Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/homelab-tailscale-nodeport.sh RemainAfterExit=yes [Install] WantedBy=multi-user.target NODEPORT_SERVICE_EOT sudo systemctl daemon-reload sudo systemctl enable homelab-tailscale-nodeport.service >/dev/null sudo systemctl restart homelab-tailscale-nodeport.service } configure_tailscale_nodeport_access \ "${self.triggers.tailscale_nodeport_enabled}" \ "${self.triggers.tailscale_nodeport_peer_ip}" \ "${self.triggers.tailscale_nodeport_node_tailscale_ip}" \ "${self.triggers.tailscale_nodeport_pod_cidr}" \ "${self.triggers.tailscale_nodeport_node_ports}" \ "${self.triggers.tailscale_nodeport_target_ports}" configure_containerd_registry "${self.triggers.registry_endpoint}" pv_dirs="${self.triggers.persistent_volume_dirs}" IFS=',' for path in $pv_dirs; do sudo mkdir -p "$path" sudo chmod 0775 "$path" done if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then sudo kubeadm reset --force || true sudo systemctl stop kubelet 2>/dev/null || true sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d fi if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then sudo kubeadm reset --force || true sudo systemctl stop kubelet 2>/dev/null || true sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d fi if [ ! -f /etc/kubernetes/kubelet.conf ]; then sudo systemctl stop kubelet 2>/dev/null || true if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then sudo systemctl status kubelet --no-pager -l || true sudo journalctl -u kubelet --no-pager -n 160 || true exit 1 fi fi EOT ] } } locals { control_plane_node_label_pairs = [ for label, value in var.control_plane_node_labels : { node_name = var.control_plane_node_name label = label value = value } ] worker_node_label_pairs = flatten([ for worker_key, worker in var.worker_nodes : [ for label, value in lookup(var.worker_node_labels, worker_key, {}) : { node_name = worker.node_name label = label value = value } ] ]) node_label_pairs = concat(local.control_plane_node_label_pairs, local.worker_node_label_pairs) } resource "null_resource" "node_labels" { for_each = { for pair in local.node_label_pairs : "${pair.node_name}/${pair.label}" => pair } depends_on = [ null_resource.kubeadm_control_plane, null_resource.kubeadm_worker, ] triggers = { kubeconfig_path = var.kubeconfig_path node_name = each.value.node_name label = each.value.label value = each.value.value } provisioner "local-exec" { interpreter = ["/bin/bash", "-lc"] environment = { KUBECONFIG_PATH = self.triggers.kubeconfig_path NODE_NAME = self.triggers.node_name NODE_LABEL = self.triggers.label NODE_LABEL_VALUE = self.triggers.value } command = </dev/null 2>&1; then break fi sleep 5 done kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null kubectl --kubeconfig "$${KUBECONFIG_PATH}" label node "$${NODE_NAME}" "$${NODE_LABEL}=$${NODE_LABEL_VALUE}" --overwrite EOT } } output "kubeconfig_path" { value = var.kubeconfig_path } output "pod_network_cidr" { value = var.pod_network_cidr }