terraform {
  required_version = ">= 1.0"
  required_providers {
    null = {
      source  = "hashicorp/null"
      version = "~> 3.2"
    }
    external = {
      source  = "hashicorp/external"
      version = "~> 2.3"
    }
  }
}

resource "null_resource" "kubeadm_control_plane" {
  triggers = {
    node_name               = var.control_plane_node_name
    advertise_address       = var.control_plane_advertise_address
    pod_network_cidr        = var.pod_network_cidr
    kubeconfig_path         = var.kubeconfig_path
    kubeconfig_owner        = var.kubeconfig_owner
    registry_endpoint       = var.registry_endpoint
    registry_config_version = "8"
    containerd_runtime      = "docker-containerd-io"
    cni_plugins_version     = "2"
    node_dns_servers        = join(" ", var.node_dns_servers)
    persistent_volume_dirs  = join(",", var.persistent_volume_dirs)
  }

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    command     = <<EOT
set -euo pipefail

apt_get() {
  sudo apt-get \
    -o Acquire::Retries=4 \
    -o Acquire::http::Timeout=20 \
    -o Acquire::https::Timeout=20 \
    "$@"
}

configure_debian_apt_https() {
  for source_file in /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources; do
    if sudo test -f "$source_file"; then
      sudo sed -i \
        -e 's|http://deb.debian.org/debian|https://deb.debian.org/debian|g' \
        -e 's|http://security.debian.org/debian-security|https://security.debian.org/debian-security|g' \
        "$source_file"
    fi
  done
}

diagnose_apt_network() {
  echo "APT package install failed; checking Debian mirror reachability..." >&2
  ip route >&2 || true
  getent hosts deb.debian.org >&2 || true
  curl -I --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/ >&2 || true
}

install_missing_packages() {
  missing_packages=""
  for package in "$@"; do
    if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
      missing_packages="$missing_packages $package"
    fi
  done
  if [ -n "$missing_packages" ]; then
    configure_debian_apt_https
    apt_get update
    apt_get install -y --no-install-recommends $missing_packages || {
      diagnose_apt_network
      exit 1
    }
  fi
}

install_containerd_runtime() {
  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
  arch="$(dpkg --print-architecture)"

  sudo install -d -m 0755 /etc/apt/keyrings
  sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
  sudo chmod a+r /etc/apt/keyrings/docker.asc
  sudo rm -f /etc/apt/sources.list.d/docker.list
  sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
Types: deb
URIs: https://download.docker.com/linux/debian
Suites: $${codename}
Components: stable
Architectures: $${arch}
Signed-By: /etc/apt/keyrings/docker.asc
DOCKER_SOURCES

  if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
    sudo apt-get remove -y containerd runc || true
  fi

  apt_get update
  apt_get install -y --no-install-recommends containerd.io
  sudo apt-mark hold containerd.io
}

configure_node_dns() {
  dns_servers="${self.triggers.node_dns_servers}"
  if [ -z "$dns_servers" ]; then
    return 0
  fi

  if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
    sudo mkdir -p /etc/systemd/resolved.conf.d
    {
      echo "[Resolve]"
      printf 'DNS=%s\n' "$dns_servers"
      printf 'FallbackDNS=%s\n' "$dns_servers"
      echo "DNSSEC=no"
    } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
    sudo systemctl restart systemd-resolved 2>/dev/null || true
  fi

  if ! getent hosts quay.io >/dev/null 2>&1; then
    sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
    sudo rm -f /etc/resolv.conf
    for server in $dns_servers; do
      printf 'nameserver %s\n' "$server"
    done | sudo tee /etc/resolv.conf >/dev/null
  fi
}

remove_containerd_section() {
  local section="$1"
  local tmp

  tmp="$(mktemp)"
  sudo awk -v section="$section" '
    $0 == section { skip = 1; next }
    skip && /^\[/ { skip = 0 }
    !skip { print }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

ensure_containerd_registry_config_path() {
  local plugin="$1"
  local append_section="$2"
  local tmp

  tmp="$(mktemp)"
  sudo awk -v plugin="$plugin" -v append_section="$append_section" '
    function is_table(line) {
      return line ~ /^[[:space:]]*\[/
    }
    function is_target_registry(line) {
      return is_table(line) &&
        index(line, plugin) > 0 &&
        line ~ /[.]registry[[:space:]]*\]/
    }
    BEGIN {
      in_target = 0
      found = 0
      wrote = 0
    }
    is_target_registry($0) {
      if (in_target && !wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      in_target = 1
      found = 1
      wrote = 0
      print
      next
    }
    in_target && is_table($0) {
      if (!wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      in_target = 0
      wrote = 0
    }
    in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
      print "  config_path = \"/etc/containerd/certs.d\""
      wrote = 1
      next
    }
    { print }
    END {
      if (in_target && !wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      if (!found) {
        print ""
        print append_section
        print "  config_path = \"/etc/containerd/certs.d\""
      }
    }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

containerd_config_version() {
  sudo awk -F= '
    /^[[:space:]]*version[[:space:]]*=/ {
      gsub(/[[:space:]]/, "", $2)
      print $2
      exit
    }
  ' /etc/containerd/config.toml
}

reset_containerd_registry_tables() {
  local tmp

  tmp="$(mktemp)"
  sudo awk '
    function is_registry_table(line) {
      return line ~ /^\[plugins\./ &&
        line ~ /\.registry([.\]]|$)/ &&
        (line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
         line ~ /io[.]containerd[.]cri[.]v1[.]images/)
    }
    is_registry_table($0) { skip = 1; next }
    skip && /^\[/ { skip = 0 }
    !skip { print }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

ensure_containerd_cni_bin_dir() {
  local config_version
  local tmp

  config_version="$(containerd_config_version)"
  tmp="$(mktemp)"
  sudo awk -v config_version="$config_version" '
    function is_table(line) {
      return line ~ /^[[:space:]]*\[/
    }
    function is_cni_table(line) {
      return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
    }
    BEGIN {
      in_cni = 0
      found = 0
    }
    is_cni_table($0) {
      in_cni = 1
      print
      next
    }
    in_cni && is_table($0) {
      in_cni = 0
    }
    in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
      sub(/=.*/, "= \"/opt/cni/bin\"")
      found = 1
    }
    in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
      sub(/=.*/, "= [\"/opt/cni/bin\"]")
      found = 1
    }
    { print }
    END {
      if (!found) {
        print ""
        if (config_version == "3") {
          print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
          print "  bin_dirs = [\"/opt/cni/bin\"]"
        } else {
          print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
          print "  bin_dir = \"/opt/cni/bin\""
        }
        print "  conf_dir = \"/etc/cni/net.d\""
      }
    }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

install_cni_plugins() {
  local plugin

  sudo mkdir -p /opt/cni/bin
  sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
  sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
  if [ -d /usr/lib/cni ]; then
    for plugin in /usr/lib/cni/*; do
      [ -f "$plugin" ] && [ -x "$plugin" ] || continue
      sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
    done
  fi
}

configure_containerd_registry() {
  local registry_endpoint="$1"
  local config_version

  sudo mkdir -p /etc/containerd
  sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null

  sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
  ensure_containerd_cni_bin_dir
  config_version="$(containerd_config_version)"
  if [ "$config_version" = "3" ]; then
    ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
  else
    ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
  fi

  sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
  sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
server = "http://$registry_endpoint"

[host."http://$registry_endpoint"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
REGISTRY_EOT
  if ! sudo containerd config dump >/dev/null; then
    sudo containerd config dump || true
    exit 1
  fi
  if ! sudo systemctl restart containerd; then
    sudo systemctl status containerd --no-pager -l || true
    sudo journalctl -u containerd --no-pager -n 160 || true
    exit 1
  fi
}

configure_node_dns
install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
install_containerd_runtime
install_cni_plugins
sudo systemctl enable --now iscsid
sudo systemctl enable kubelet || true

sudo swapoff -a || true
sudo awk '
  /^[[:space:]]*#/ { print; next }
  $3 == "swap" { print "# kubeadm-disabled " $0; next }
  { print }
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
sudo mv /etc/fstab.kubeadm /etc/fstab

sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
overlay
br_netfilter
MODULES_EOT
sudo modprobe overlay || true
sudo modprobe br_netfilter || true

sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
SYSCTL_EOT
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
  sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
fi

if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
  printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
fi

configure_containerd_registry "${self.triggers.registry_endpoint}"

reset_control_plane_join_state() {
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo kubeadm reset --force || true
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo systemctl stop containerd 2>/dev/null || true

  sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
  sudo mkdir -p /var/lib/kubelet /var/lib/cni /etc/cni/net.d

  sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
  if ! sudo systemctl start containerd; then
    sudo systemctl status containerd --no-pager -l || true
    sudo journalctl -u containerd --no-pager -n 160 || true
    exit 1
  fi
}

IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}"
for path in "$${pv_dirs[@]}"; do
  sudo mkdir -p "$path"
  sudo chmod 0775 "$path"
done

if [ ! -f /etc/kubernetes/admin.conf ]; then
  reset_control_plane_join_state
fi

if [ ! -f /etc/kubernetes/admin.conf ]; then
  if ! sudo kubeadm init \
    --pod-network-cidr=${self.triggers.pod_network_cidr} \
    --node-name=${self.triggers.node_name} \
    --apiserver-advertise-address=${self.triggers.advertise_address}; then
    sudo systemctl status kubelet --no-pager -l || true
    sudo journalctl -u kubelet --no-pager -n 160 || true
    exit 1
  fi
fi

mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")"
sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}"
sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}"

kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true
EOT
  }
}

resource "null_resource" "worker_nodes_required" {
  triggers = {
    worker_count = tostring(length(var.worker_nodes))
  }

  lifecycle {
    precondition {
      condition     = var.allow_empty_worker_nodes || length(var.worker_nodes) > 0
      error_message = "worker_nodes is empty. Pass -var-file=/path/to/.lab/cluster-workers.auto.tfvars.json or set allow_empty_worker_nodes=true for an intentional control-plane-only cluster."
    }
  }
}

resource "null_resource" "kubeadm_worker" {
  for_each = var.worker_nodes

  depends_on = [
    null_resource.kubeadm_control_plane,
    null_resource.worker_nodes_required,
  ]

  triggers = {
    node_name                            = each.value.node_name
    host                                 = each.value.host
    user                                 = each.value.user
    ssh_key_path                         = each.value.ssh_key_path
    kubeconfig_path                      = var.kubeconfig_path
    registry_endpoint                    = var.registry_endpoint
    registry_config_version              = "8"
    containerd_runtime                   = "docker-containerd-io"
    cni_plugins_version                  = "2"
    force_rejoin                         = tostring(var.force_worker_rejoin)
    worker_rejoin_reset_version          = "2"
    node_dns_servers                     = join(" ", var.node_dns_servers)
    persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
    worker_tailscale_version             = "1"
    worker_tailscale_enabled             = tostring(var.worker_tailscale_enabled)
    worker_tailscale_accept_routes       = tostring(var.worker_tailscale_accept_routes)
    worker_tailscale_pod_egress_snat     = tostring(var.worker_tailscale_pod_egress_snat)
    tailscale_nodeport_version           = "3"
    tailscale_nodeport_enabled           = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false"
    tailscale_nodeport_peer_ip           = var.tailscale_nodeport_access.peer_ip
    tailscale_nodeport_node_tailscale_ip = var.tailscale_nodeport_access.node_tailscale_ip
    tailscale_nodeport_pod_cidr          = var.tailscale_nodeport_access.pod_cidr
    tailscale_nodeport_node_ports        = join(" ", distinct(concat([var.tailscale_nodeport_access.node_port], var.tailscale_nodeport_extra_ports)))
    tailscale_nodeport_target_ports      = join(" ", distinct(concat([var.tailscale_nodeport_access.target_port], var.tailscale_nodeport_extra_target_ports)))
    tailscale_subnet_routes_version      = "1"
    tailscale_subnet_routes_enabled      = var.tailscale_subnet_routes.enabled && each.key == var.tailscale_subnet_routes.worker_key ? "true" : "false"
    tailscale_subnet_routes              = join(",", var.tailscale_subnet_routes.routes)
  }

  connection {
    type        = "ssh"
    user        = self.triggers.user
    private_key = file(self.triggers.ssh_key_path)
    host        = self.triggers.host
    timeout     = "15m"
  }

  provisioner "remote-exec" {
    inline = [
      <<EOT
set -eu

apt_get() {
  sudo apt-get \
    -o Acquire::Retries=4 \
    -o Acquire::http::Timeout=20 \
    -o Acquire::https::Timeout=20 \
    "$@"
}

configure_debian_apt_https() {
  for source_file in /etc/apt/sources.list /etc/apt/sources.list.d/debian.sources; do
    if sudo test -f "$source_file"; then
      sudo sed -i \
        -e 's|http://deb.debian.org/debian|https://deb.debian.org/debian|g' \
        -e 's|http://security.debian.org/debian-security|https://security.debian.org/debian-security|g' \
        "$source_file"
    fi
  done
}

diagnose_apt_network() {
  echo "APT package install failed; checking Debian mirror reachability..." >&2
  ip route >&2 || true
  getent hosts deb.debian.org >&2 || true
  curl -I --connect-timeout 5 --max-time 10 https://deb.debian.org/debian/ >&2 || true
}

install_missing_packages() {
  missing_packages=""
  for package in "$@"; do
    if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
      missing_packages="$missing_packages $package"
    fi
  done
  if [ -n "$missing_packages" ]; then
    configure_debian_apt_https
    apt_get update
    apt_get install -y --no-install-recommends $missing_packages || {
      diagnose_apt_network
      exit 1
    }
  fi
}

install_containerd_runtime() {
  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
  arch="$(dpkg --print-architecture)"

  sudo install -d -m 0755 /etc/apt/keyrings
  sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
  sudo chmod a+r /etc/apt/keyrings/docker.asc
  sudo rm -f /etc/apt/sources.list.d/docker.list
  sudo tee /etc/apt/sources.list.d/docker.sources >/dev/null <<DOCKER_SOURCES
Types: deb
URIs: https://download.docker.com/linux/debian
Suites: $${codename}
Components: stable
Architectures: $${arch}
Signed-By: /etc/apt/keyrings/docker.asc
DOCKER_SOURCES

  if dpkg -s containerd >/dev/null 2>&1 && ! dpkg -s containerd.io >/dev/null 2>&1; then
    sudo apt-get remove -y containerd runc || true
  fi

  apt_get update
  apt_get install -y --no-install-recommends containerd.io
  sudo apt-mark hold containerd.io
}

install_tailscale() {
  if [ "${self.triggers.worker_tailscale_enabled}" != "true" ]; then
    return 0
  fi

  if command -v tailscale >/dev/null 2>&1 && systemctl list-unit-files tailscaled.service >/dev/null 2>&1; then
    sudo systemctl enable --now tailscaled
    return 0
  fi

  codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
  sudo install -d -m 0755 /usr/share/keyrings
  curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.noarmor.gpg" |
    sudo tee /usr/share/keyrings/tailscale-archive-keyring.gpg >/dev/null
  curl -fsSL "https://pkgs.tailscale.com/stable/debian/$${codename}.tailscale-keyring.list" |
    sudo tee /etc/apt/sources.list.d/tailscale.list >/dev/null
  apt_get update
  apt_get install -y --no-install-recommends tailscale
  sudo systemctl enable --now tailscaled
}

configure_tailscale_pod_egress_snat() {
  local enabled="$1"
  local pod_cidr="$2"

  if [ "${self.triggers.worker_tailscale_enabled}" != "true" ] || [ "$enabled" != "true" ]; then
    sudo systemctl disable --now homelab-tailscale-pod-egress.service >/dev/null 2>&1 || true
    return 0
  fi

  sudo mkdir -p /usr/local/sbin /etc/sysctl.d
  sudo tee /etc/sysctl.d/98-homelab-tailscale-pod-egress.conf >/dev/null <<POD_EGRESS_SYSCTL_EOT
net.ipv4.ip_forward = 1
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.tailscale0.rp_filter = 0
POD_EGRESS_SYSCTL_EOT

  sudo tee /usr/local/sbin/homelab-tailscale-pod-egress.sh >/dev/null <<POD_EGRESS_SCRIPT_EOT
#!/bin/sh
set -eu

sysctl -w net.ipv4.ip_forward=1 >/dev/null
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true

if ! ip link show tailscale0 >/dev/null 2>&1; then
  echo "tailscale0 is not present; skipping Tailscale pod egress SNAT"
  exit 0
fi

iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -j ACCEPT 2>/dev/null ||
  iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -j ACCEPT
iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
  iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
iptables -t nat -C POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE 2>/dev/null ||
  iptables -t nat -A POSTROUTING -s "$pod_cidr" -o tailscale0 -m comment --comment homelab-tailscale-pod-egress -j MASQUERADE
POD_EGRESS_SCRIPT_EOT
  sudo chmod 0755 /usr/local/sbin/homelab-tailscale-pod-egress.sh

  sudo tee /etc/systemd/system/homelab-tailscale-pod-egress.service >/dev/null <<'POD_EGRESS_SERVICE_EOT'
[Unit]
Description=Homelab Tailscale pod egress SNAT
After=network-online.target tailscaled.service kubelet.service
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-tailscale-pod-egress.sh
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
POD_EGRESS_SERVICE_EOT
  sudo systemctl daemon-reload
  sudo systemctl enable homelab-tailscale-pod-egress.service >/dev/null
  sudo systemctl restart homelab-tailscale-pod-egress.service
}

configure_node_dns() {
  dns_servers="${self.triggers.node_dns_servers}"
  if [ -z "$dns_servers" ]; then
    return 0
  fi

  if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
    sudo mkdir -p /etc/systemd/resolved.conf.d
    {
      echo "[Resolve]"
      printf 'DNS=%s\n' "$dns_servers"
      printf 'FallbackDNS=%s\n' "$dns_servers"
      echo "DNSSEC=no"
    } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
    sudo systemctl restart systemd-resolved 2>/dev/null || true
  fi

  if ! getent hosts quay.io >/dev/null 2>&1; then
    sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
    sudo rm -f /etc/resolv.conf
    for server in $dns_servers; do
      printf 'nameserver %s\n' "$server"
    done | sudo tee /etc/resolv.conf >/dev/null
  fi
}

remove_containerd_section() {
  local section="$1"
  local tmp

  tmp="$(mktemp)"
  sudo awk -v section="$section" '
    $0 == section { skip = 1; next }
    skip && /^\[/ { skip = 0 }
    !skip { print }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

ensure_containerd_registry_config_path() {
  local plugin="$1"
  local append_section="$2"
  local tmp

  tmp="$(mktemp)"
  sudo awk -v plugin="$plugin" -v append_section="$append_section" '
    function is_table(line) {
      return line ~ /^[[:space:]]*\[/
    }
    function is_target_registry(line) {
      return is_table(line) &&
        index(line, plugin) > 0 &&
        line ~ /[.]registry[[:space:]]*\]/
    }
    BEGIN {
      in_target = 0
      found = 0
      wrote = 0
    }
    is_target_registry($0) {
      if (in_target && !wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      in_target = 1
      found = 1
      wrote = 0
      print
      next
    }
    in_target && is_table($0) {
      if (!wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      in_target = 0
      wrote = 0
    }
    in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
      print "  config_path = \"/etc/containerd/certs.d\""
      wrote = 1
      next
    }
    { print }
    END {
      if (in_target && !wrote) {
        print "  config_path = \"/etc/containerd/certs.d\""
      }
      if (!found) {
        print ""
        print append_section
        print "  config_path = \"/etc/containerd/certs.d\""
      }
    }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

containerd_config_version() {
  sudo awk -F= '
    /^[[:space:]]*version[[:space:]]*=/ {
      gsub(/[[:space:]]/, "", $2)
      print $2
      exit
    }
  ' /etc/containerd/config.toml
}

reset_containerd_registry_tables() {
  local tmp

  tmp="$(mktemp)"
  sudo awk '
    function is_registry_table(line) {
      return line ~ /^\[plugins\./ &&
        line ~ /\.registry([.\]]|$)/ &&
        (line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
         line ~ /io[.]containerd[.]cri[.]v1[.]images/)
    }
    is_registry_table($0) { skip = 1; next }
    skip && /^\[/ { skip = 0 }
    !skip { print }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

ensure_containerd_cni_bin_dir() {
  local config_version
  local tmp

  config_version="$(containerd_config_version)"
  tmp="$(mktemp)"
  sudo awk -v config_version="$config_version" '
    function is_table(line) {
      return line ~ /^[[:space:]]*\[/
    }
    function is_cni_table(line) {
      return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
    }
    BEGIN {
      in_cni = 0
      found = 0
    }
    is_cni_table($0) {
      in_cni = 1
      print
      next
    }
    in_cni && is_table($0) {
      in_cni = 0
    }
    in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
      sub(/=.*/, "= \"/opt/cni/bin\"")
      found = 1
    }
    in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
      sub(/=.*/, "= [\"/opt/cni/bin\"]")
      found = 1
    }
    { print }
    END {
      if (!found) {
        print ""
        if (config_version == "3") {
          print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
          print "  bin_dirs = [\"/opt/cni/bin\"]"
        } else {
          print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
          print "  bin_dir = \"/opt/cni/bin\""
        }
        print "  conf_dir = \"/etc/cni/net.d\""
      }
    }
  ' /etc/containerd/config.toml > "$tmp"
  sudo mv "$tmp" /etc/containerd/config.toml
}

install_cni_plugins() {
  local plugin

  sudo mkdir -p /opt/cni/bin
  sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
  sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
  if [ -d /usr/lib/cni ]; then
    for plugin in /usr/lib/cni/*; do
      [ -f "$plugin" ] && [ -x "$plugin" ] || continue
      sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
    done
  fi
}

configure_containerd_registry() {
  local registry_endpoint="$1"
  local config_version

  sudo mkdir -p /etc/containerd
  sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null

  sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
  ensure_containerd_cni_bin_dir
  config_version="$(containerd_config_version)"
  if [ "$config_version" = "3" ]; then
    ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
  else
    ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
  fi

  sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
  sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
server = "http://$registry_endpoint"

[host."http://$registry_endpoint"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
REGISTRY_EOT
  if ! sudo containerd config dump >/dev/null; then
    sudo containerd config dump || true
    exit 1
  fi
  if ! sudo systemctl restart containerd; then
    sudo systemctl status containerd --no-pager -l || true
    sudo journalctl -u containerd --no-pager -n 160 || true
    exit 1
  fi
}

configure_node_dns
install_missing_packages ca-certificates curl gpg apt-transport-https containernetworking-plugins open-iscsi nfs-common
install_containerd_runtime
install_tailscale
install_cni_plugins
sudo systemctl enable --now iscsid
sudo systemctl enable kubelet || true

sudo swapoff -a || true
sudo awk '
  /^[[:space:]]*#/ { print; next }
  $3 == "swap" { print "# kubeadm-disabled " $0; next }
  { print }
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
sudo mv /etc/fstab.kubeadm /etc/fstab

sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
overlay
br_netfilter
MODULES_EOT
sudo modprobe overlay || true
sudo modprobe br_netfilter || true

sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
SYSCTL_EOT
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
  sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
  sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
fi

if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
  printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
fi

configure_tailscale_pod_egress_snat \
  "${self.triggers.worker_tailscale_pod_egress_snat}" \
  "${self.triggers.tailscale_nodeport_pod_cidr}"

configure_tailscale_nodeport_access() {
  local enabled="$1"
  local peer_ip="$2"
  local node_tailscale_ip="$3"
  local pod_cidr="$4"
  local node_ports="$5"
  local target_ports="$6"

  if [ "$enabled" != "true" ]; then
    return 0
  fi

  sudo mkdir -p /usr/local/sbin /etc/sysctl.d
  sudo tee /etc/sysctl.d/98-homelab-tailscale-nodeport.conf >/dev/null <<NODEPORT_SYSCTL_EOT
net.ipv4.conf.all.rp_filter = 0
net.ipv4.conf.tailscale0.rp_filter = 0
NODEPORT_SYSCTL_EOT

  sudo tee /usr/local/sbin/homelab-tailscale-nodeport.sh >/dev/null <<NODEPORT_SCRIPT_EOT
#!/bin/sh
set -eu

sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true

if ! ip link show tailscale0 >/dev/null 2>&1; then
  echo "tailscale0 is not present; skipping Tailscale NodePort routing"
  exit 0
fi

ip route replace "$peer_ip/32" dev tailscale0 src "$node_tailscale_ip"

for node_port in $node_ports; do
  iptables -C INPUT -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT 2>/dev/null ||
    iptables -I INPUT 1 -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT
done
for target_port in $target_ports; do
  iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT 2>/dev/null ||
    iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT
done
iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
  iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
for target_port in $target_ports; do
  iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null ||
    iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE
done
NODEPORT_SCRIPT_EOT
  sudo chmod 0755 /usr/local/sbin/homelab-tailscale-nodeport.sh

  sudo tee /etc/systemd/system/homelab-tailscale-nodeport.service >/dev/null <<'NODEPORT_SERVICE_EOT'
[Unit]
Description=Homelab Tailscale NodePort routing
After=network-online.target tailscaled.service kubelet.service
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-tailscale-nodeport.sh
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
NODEPORT_SERVICE_EOT
  sudo systemctl daemon-reload
  sudo systemctl enable homelab-tailscale-nodeport.service >/dev/null
  sudo systemctl restart homelab-tailscale-nodeport.service
}

configure_tailscale_nodeport_access \
  "${self.triggers.tailscale_nodeport_enabled}" \
  "${self.triggers.tailscale_nodeport_peer_ip}" \
  "${self.triggers.tailscale_nodeport_node_tailscale_ip}" \
  "${self.triggers.tailscale_nodeport_pod_cidr}" \
  "${self.triggers.tailscale_nodeport_node_ports}" \
  "${self.triggers.tailscale_nodeport_target_ports}"

configure_tailscale_subnet_routes() {
  local enabled="$1"
  local routes="$2"

  if [ "$enabled" != "true" ]; then
    return 0
  fi

  if ! command -v tailscale >/dev/null 2>&1; then
    echo "tailscale is required to advertise subnet routes but is not installed on this worker." >&2
    exit 1
  fi

  if [ -z "$routes" ]; then
    echo "tailscale subnet route advertisement is enabled but no routes were configured." >&2
    exit 1
  fi

  sudo tailscale set --advertise-routes="$routes"
}

configure_tailscale_subnet_routes \
  "${self.triggers.tailscale_subnet_routes_enabled}" \
  "${self.triggers.tailscale_subnet_routes}"

configure_containerd_registry "${self.triggers.registry_endpoint}"

reset_worker_join_state() {
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo kubeadm reset --force || true
  sudo systemctl stop kubelet 2>/dev/null || true
  sudo systemctl stop containerd 2>/dev/null || true

  sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/
  sudo mkdir -p /etc/cni/net.d
  sudo rm -rf /etc/cni/net.d/*
  sudo mkdir -p /var/lib/kubelet /var/lib/cni

  sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
  if ! sudo systemctl start containerd; then
    sudo systemctl status containerd --no-pager -l || true
    sudo journalctl -u containerd --no-pager -n 160 || true
    exit 1
  fi
}

pv_dirs="${self.triggers.persistent_volume_dirs}"
IFS=','
for path in $pv_dirs; do
  sudo mkdir -p "$path"
  sudo chmod 0775 "$path"
done

if [ "${self.triggers.force_rejoin}" = "true" ]; then
  reset_worker_join_state
fi

if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then
  reset_worker_join_state
fi

if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
  reset_worker_join_state
fi
EOT
    ]
  }

  provisioner "local-exec" {
    interpreter = ["/usr/bin/env", "bash", "-c"]
    command     = <<EOT
set -euo pipefail

if [ '${self.triggers.worker_tailscale_enabled}' != 'true' ]; then
  exit 0
fi

ssh_target='${self.triggers.user}@${self.triggers.host}'
ssh_key='${self.triggers.ssh_key_path}'
node_name='${self.triggers.node_name}'
accept_routes='${self.triggers.worker_tailscale_accept_routes}'
ssh_host="$${ssh_target#*@}"
known_hosts_file="$${TF_DATA_DIR:-.terraform}/homelab-worker-known_hosts"

mkdir -p "$(dirname "$known_hosts_file")"
touch "$known_hosts_file"
chmod 0600 "$known_hosts_file"
ssh-keygen -R "$ssh_host" -f "$known_hosts_file" >/dev/null 2>&1 || true

worker_ssh() {
  ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$ssh_target" "$@"
}

if ! worker_ssh \
  'command -v tailscale >/dev/null 2>&1 && sudo systemctl is-active --quiet tailscaled'; then
  echo "tailscale is not installed or tailscaled is not active on $ssh_target" >&2
  exit 1
fi

if worker_ssh \
  'test -n "$(tailscale ip -4 2>/dev/null | head -n 1)"'; then
  worker_ssh "sudo tailscale set --accept-routes=$accept_routes"
  exit 0
fi

auth_key="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY:-}"
auth_key_file="$${LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY_FILE:-$${HOME}/.config/homelab/tailscale-pimox-worker.authkey}"
if [ -z "$auth_key" ] && [ -s "$auth_key_file" ]; then
  auth_key="$(tr -d '\n\r' <"$auth_key_file")"
fi
if [ -z "$auth_key" ]; then
  echo "Pimox worker Tailscale is enabled, but no auth key was provided." >&2
  echo "Set LAB_PIMOX_WORKER_TAILSCALE_AUTH_KEY or create $auth_key_file outside Git." >&2
  exit 1
fi

printf '%s\n' "$auth_key" |
  worker_ssh \
    "read -r auth_key && sudo tailscale up --auth-key \"\$auth_key\" --hostname '$node_name' --accept-routes=$accept_routes --reset"

worker_ssh 'sudo systemctl restart homelab-tailscale-pod-egress.service 2>/dev/null || true'
EOT
  }

  provisioner "local-exec" {
    interpreter = ["/usr/bin/env", "bash", "-c"]
    command     = <<EOT
set -euo pipefail

ssh_target='${self.triggers.user}@${self.triggers.host}'
ssh_key='${self.triggers.ssh_key_path}'
node_name='${self.triggers.node_name}'
kubeconfig_path='${self.triggers.kubeconfig_path}'
ssh_host="$${ssh_target#*@}"
known_hosts_file="$${TF_DATA_DIR:-.terraform}/homelab-worker-known_hosts"

mkdir -p "$(dirname "$known_hosts_file")"
touch "$known_hosts_file"
chmod 0600 "$known_hosts_file"
ssh-keygen -R "$ssh_host" -f "$known_hosts_file" >/dev/null 2>&1 || true

worker_ssh() {
  ssh -i "$ssh_key" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$ssh_target" "$@"
}

if worker_ssh \
  'test -f /etc/kubernetes/kubelet.conf && test -f /var/lib/kubelet/config.yaml'; then
  exit 0
fi

token_kubeconfig=""
if [ -s "$kubeconfig_path" ]; then
  token_kubeconfig="$kubeconfig_path"
elif [ -s /etc/kubernetes/admin.conf ]; then
  token_kubeconfig=/etc/kubernetes/admin.conf
elif [ -n "$${KUBECONFIG:-}" ] && [ -s "$${KUBECONFIG}" ]; then
  token_kubeconfig="$${KUBECONFIG}"
fi
if [ -z "$token_kubeconfig" ]; then
  echo "No kubeconfig available to create a kubeadm join token." >&2
  echo "Checked: $kubeconfig_path, /etc/kubernetes/admin.conf, and KUBECONFIG." >&2
  exit 1
fi

join_cmd="$(sudo env KUBECONFIG="$token_kubeconfig" kubeadm token create --print-join-command)"
remote_join_script="$(mktemp)"
trap 'rm -f "$remote_join_script"' EXIT

cat >"$remote_join_script" <<JOIN_EOT
set -eu
sudo systemctl stop kubelet 2>/dev/null || true
if ! sudo $join_cmd --node-name=$node_name; then
  sudo systemctl status kubelet --no-pager -l || true
  sudo journalctl -u kubelet --no-pager -n 160 || true
  exit 1
fi
if [ ! -f /etc/kubernetes/kubelet.conf ] || [ ! -f /var/lib/kubelet/config.yaml ]; then
  echo "kubeadm join completed without creating kubelet configuration" >&2
  sudo ls -la /etc/kubernetes /var/lib/kubelet 2>/dev/null || true
  sudo systemctl status kubelet containerd --no-pager -l || true
  sudo journalctl -u kubelet --no-pager -n 160 || true
  exit 1
fi
JOIN_EOT

worker_ssh 'bash -s' <"$remote_join_script"
EOT
  }
}

locals {
  control_plane_node_label_pairs = [
    for label, value in var.control_plane_node_labels : {
      node_name = var.control_plane_node_name
      label     = label
      value     = value
    }
  ]

  worker_node_label_pairs = flatten([
    for worker_key, worker in var.worker_nodes : [
      for label, value in lookup(var.worker_node_labels, worker_key, {}) : {
        node_name = worker.node_name
        label     = label
        value     = value
      }
    ]
  ])

  node_label_pairs = concat(local.control_plane_node_label_pairs, local.worker_node_label_pairs)
}

resource "null_resource" "node_labels" {
  for_each = {
    for pair in local.node_label_pairs : "${pair.node_name}/${pair.label}" => pair
  }

  depends_on = [
    null_resource.kubeadm_control_plane,
    null_resource.kubeadm_worker,
  ]

  triggers = {
    kubeconfig_path = var.kubeconfig_path
    node_name       = each.value.node_name
    label           = each.value.label
    value           = each.value.value
  }

  provisioner "local-exec" {
    interpreter = ["/bin/bash", "-lc"]
    environment = {
      KUBECONFIG_PATH  = self.triggers.kubeconfig_path
      NODE_NAME        = self.triggers.node_name
      NODE_LABEL       = self.triggers.label
      NODE_LABEL_VALUE = self.triggers.value
    }
    command = <<EOT
set -euo pipefail

for _ in $(seq 1 60); do
  if kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null 2>&1; then
    break
  fi
  sleep 5
done

kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null
kubectl --kubeconfig "$${KUBECONFIG_PATH}" label node "$${NODE_NAME}" "$${NODE_LABEL}=$${NODE_LABEL_VALUE}" --overwrite
EOT
  }
}

output "kubeconfig_path" {
  value = var.kubeconfig_path
}

output "pod_network_cidr" {
  value = var.pod_network_cidr
}