my-homelab-configs/bootstrap/cluster/main.tf

terraform {
  required_version = ">= 1.0"
  required_providers {
    null = {
      source  = "hashicorp/null"
      version = "~> 3.2"
    }
    external = {
      source  = "hashicorp/external"
      version = "~> 2.3"
    }
  }
}

resource "null_resource" "kubeadm_control_plane" {
  triggers = {
    node_name              = var.control_plane_node_name
    advertise_address      = var.control_plane_advertise_address
    pod_network_cidr       = var.pod_network_cidr
    kubeconfig_path        = var.kubeconfig_path
    kubeconfig_owner       = var.kubeconfig_owner
    registry_endpoint      = var.registry_endpoint
    persistent_volume_dirs = join(",", var.persistent_volume_dirs)
  }

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    command     = <<EOT
set -euo pipefail

sudo apt-get update
sudo apt-get install -y curl open-iscsi nfs-common
sudo systemctl enable --now iscsid
sudo systemctl enable kubelet || true

sudo swapoff -a || true
sudo awk '
  /^[[:space:]]*#/ { print; next }
  $3 == "swap" { print "# kubeadm-disabled " $0; next }
  { print }
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
sudo mv /etc/fstab.kubeadm /etc/fstab

sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
overlay
br_netfilter
MODULES_EOT
sudo modprobe overlay || true
sudo modprobe br_netfilter || true

sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
SYSCTL_EOT
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
  sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
fi

if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
  printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
fi

sudo mkdir -p /etc/containerd
if [ ! -f /etc/containerd/config.toml ]; then
  sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
fi
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml
sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint}
sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null <<REGISTRY_EOT
server = "http://${self.triggers.registry_endpoint}"

[host."http://${self.triggers.registry_endpoint}"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
REGISTRY_EOT
sudo systemctl restart containerd

IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}"
for path in "$${pv_dirs[@]}"; do
  sudo mkdir -p "$path"
  sudo chmod 0775 "$path"
done

if [ ! -f /etc/kubernetes/admin.conf ] && [ -d /etc/kubernetes ]; then
  sudo kubeadm reset --force || true
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
fi

if [ ! -f /etc/kubernetes/admin.conf ]; then
  sudo systemctl stop kubelet 2>/dev/null || true
  if ! sudo kubeadm init \
    --pod-network-cidr=${self.triggers.pod_network_cidr} \
    --node-name=${self.triggers.node_name} \
    --apiserver-advertise-address=${self.triggers.advertise_address}; then
    sudo systemctl status kubelet --no-pager -l || true
    sudo journalctl -u kubelet --no-pager -n 160 || true
    exit 1
  fi
fi

mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")"
sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}"
sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}"

kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true
EOT
  }
}

data "external" "kubeadm_join_command" {
  depends_on = [null_resource.kubeadm_control_plane]
  program = [
    "bash",
    "-lc",
    <<EOT
set -euo pipefail
cmd="$(sudo kubeadm token create --print-join-command)"
printf '{"cmd":"%s"}\n' "$(printf '%s' "$cmd" | sed 's/\\/\\\\/g; s/"/\\"/g')"
EOT
  ]
}

resource "null_resource" "kubeadm_worker" {
  for_each = var.worker_nodes

  depends_on = [data.external.kubeadm_join_command]

  triggers = {
    node_name              = each.value.node_name
    host                   = each.value.host
    user                   = each.value.user
    ssh_key_path           = each.value.ssh_key_path
    registry_endpoint      = var.registry_endpoint
    persistent_volume_dirs = join(",", var.persistent_volume_dirs)
  }

  connection {
    type        = "ssh"
    user        = self.triggers.user
    private_key = file(self.triggers.ssh_key_path)
    host        = self.triggers.host
  }

  provisioner "remote-exec" {
    inline = [
      <<EOT
set -eu

sudo apt-get update
sudo apt-get install -y curl open-iscsi nfs-common
sudo systemctl enable --now iscsid
sudo systemctl enable kubelet || true

sudo swapoff -a || true
sudo awk '
  /^[[:space:]]*#/ { print; next }
  $3 == "swap" { print "# kubeadm-disabled " $0; next }
  { print }
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
sudo mv /etc/fstab.kubeadm /etc/fstab

sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
overlay
br_netfilter
MODULES_EOT
sudo modprobe overlay || true
sudo modprobe br_netfilter || true

sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
SYSCTL_EOT
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
  sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
fi

if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
  printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
fi

sudo mkdir -p /etc/containerd
if [ ! -f /etc/containerd/config.toml ]; then
  sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
fi
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml
sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint}
sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null <<REGISTRY_EOT
server = "http://${self.triggers.registry_endpoint}"

[host."http://${self.triggers.registry_endpoint}"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
REGISTRY_EOT
sudo systemctl restart containerd

pv_dirs="${self.triggers.persistent_volume_dirs}"
IFS=','
for path in $pv_dirs; do
  sudo mkdir -p "$path"
  sudo chmod 0775 "$path"
done

if [ -f /etc/kubernetes/kubelet.conf ] && ! curl -fsS --max-time 5 http://127.0.0.1:10248/healthz >/dev/null 2>&1; then
  sudo kubeadm reset --force || true
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
fi

if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
  sudo kubeadm reset --force || true
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
fi

if [ ! -f /etc/kubernetes/kubelet.conf ]; then
  sudo systemctl stop kubelet 2>/dev/null || true
  if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then
    sudo systemctl status kubelet --no-pager -l || true
    sudo journalctl -u kubelet --no-pager -n 160 || true
    exit 1
  fi
fi
EOT
    ]
  }
}

output "kubeconfig_path" {
  value = var.kubeconfig_path
}

output "pod_network_cidr" {
  value = var.pod_network_cidr
}