324 lines
10 KiB
HCL
324 lines
10 KiB
HCL
terraform {
|
|
required_version = ">= 1.0"
|
|
required_providers {
|
|
null = {
|
|
source = "hashicorp/null"
|
|
version = "~> 3.2"
|
|
}
|
|
external = {
|
|
source = "hashicorp/external"
|
|
version = "~> 2.3"
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_control_plane" {
|
|
triggers = {
|
|
node_name = var.control_plane_node_name
|
|
advertise_address = var.control_plane_advertise_address
|
|
pod_network_cidr = var.pod_network_cidr
|
|
kubeconfig_path = var.kubeconfig_path
|
|
kubeconfig_owner = var.kubeconfig_owner
|
|
registry_endpoint = var.registry_endpoint
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages open-iscsi nfs-common
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
if [ ! -f /etc/containerd/config.toml ]; then
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
fi
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
|
|
sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml
|
|
sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint}
|
|
sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null <<REGISTRY_EOT
|
|
server = "http://${self.triggers.registry_endpoint}"
|
|
|
|
[host."http://${self.triggers.registry_endpoint}"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
sudo systemctl restart containerd
|
|
|
|
IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}"
|
|
for path in "$${pv_dirs[@]}"; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ] && [ -d /etc/kubernetes ]; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ]; then
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
if ! sudo kubeadm init \
|
|
--pod-network-cidr=${self.triggers.pod_network_cidr} \
|
|
--node-name=${self.triggers.node_name} \
|
|
--apiserver-advertise-address=${self.triggers.advertise_address}; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")"
|
|
sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}"
|
|
sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}"
|
|
|
|
kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true
|
|
EOT
|
|
}
|
|
}
|
|
|
|
data "external" "kubeadm_join_command" {
|
|
depends_on = [null_resource.kubeadm_control_plane]
|
|
program = [
|
|
"bash",
|
|
"-lc",
|
|
<<EOT
|
|
set -euo pipefail
|
|
cmd="$(sudo kubeadm token create --print-join-command)"
|
|
printf '{"cmd":"%s"}\n' "$(printf '%s' "$cmd" | sed 's/\\/\\\\/g; s/"/\\"/g')"
|
|
EOT
|
|
]
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_worker" {
|
|
for_each = var.worker_nodes
|
|
|
|
depends_on = [data.external.kubeadm_join_command]
|
|
|
|
triggers = {
|
|
node_name = each.value.node_name
|
|
host = each.value.host
|
|
user = each.value.user
|
|
ssh_key_path = each.value.ssh_key_path
|
|
registry_endpoint = var.registry_endpoint
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
}
|
|
|
|
connection {
|
|
type = "ssh"
|
|
user = self.triggers.user
|
|
private_key = file(self.triggers.ssh_key_path)
|
|
host = self.triggers.host
|
|
}
|
|
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<EOT
|
|
set -eu
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages open-iscsi nfs-common
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
if [ ! -f /etc/containerd/config.toml ]; then
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
fi
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
|
|
sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml
|
|
sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint}
|
|
sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null <<REGISTRY_EOT
|
|
server = "http://${self.triggers.registry_endpoint}"
|
|
|
|
[host."http://${self.triggers.registry_endpoint}"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
sudo systemctl restart containerd
|
|
|
|
pv_dirs="${self.triggers.persistent_volume_dirs}"
|
|
IFS=','
|
|
for path in $pv_dirs; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ]; then
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
EOT
|
|
]
|
|
}
|
|
}
|
|
|
|
output "kubeconfig_path" {
|
|
value = var.kubeconfig_path
|
|
}
|
|
|
|
output "pod_network_cidr" {
|
|
value = var.pod_network_cidr
|
|
}
|