872 lines
26 KiB
HCL
872 lines
26 KiB
HCL
terraform {
|
|
required_version = ">= 1.0"
|
|
required_providers {
|
|
null = {
|
|
source = "hashicorp/null"
|
|
version = "~> 3.2"
|
|
}
|
|
external = {
|
|
source = "hashicorp/external"
|
|
version = "~> 2.3"
|
|
}
|
|
}
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_control_plane" {
|
|
triggers = {
|
|
node_name = var.control_plane_node_name
|
|
advertise_address = var.control_plane_advertise_address
|
|
pod_network_cidr = var.pod_network_cidr
|
|
kubeconfig_path = var.kubeconfig_path
|
|
kubeconfig_owner = var.kubeconfig_owner
|
|
registry_endpoint = var.registry_endpoint
|
|
registry_config_version = "8"
|
|
cni_plugins_version = "2"
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
remove_containerd_section() {
|
|
local section="$1"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v section="$section" '
|
|
$0 == section { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_registry_config_path() {
|
|
local plugin="$1"
|
|
local append_section="$2"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v plugin="$plugin" -v append_section="$append_section" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_target_registry(line) {
|
|
return is_table(line) &&
|
|
index(line, plugin) > 0 &&
|
|
line ~ /[.]registry[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_target = 0
|
|
found = 0
|
|
wrote = 0
|
|
}
|
|
is_target_registry($0) {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 1
|
|
found = 1
|
|
wrote = 0
|
|
print
|
|
next
|
|
}
|
|
in_target && is_table($0) {
|
|
if (!wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 0
|
|
wrote = 0
|
|
}
|
|
in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
wrote = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
if (!found) {
|
|
print ""
|
|
print append_section
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
containerd_config_version() {
|
|
sudo awk -F= '
|
|
/^[[:space:]]*version[[:space:]]*=/ {
|
|
gsub(/[[:space:]]/, "", $2)
|
|
print $2
|
|
exit
|
|
}
|
|
' /etc/containerd/config.toml
|
|
}
|
|
|
|
reset_containerd_registry_tables() {
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk '
|
|
function is_registry_table(line) {
|
|
return line ~ /^\[plugins\./ &&
|
|
line ~ /\.registry([.\]]|$)/ &&
|
|
(line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
|
|
line ~ /io[.]containerd[.]cri[.]v1[.]images/)
|
|
}
|
|
is_registry_table($0) { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_cni_bin_dir() {
|
|
local config_version
|
|
local tmp
|
|
|
|
config_version="$(containerd_config_version)"
|
|
tmp="$(mktemp)"
|
|
sudo awk -v config_version="$config_version" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_cni_table(line) {
|
|
return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_cni = 0
|
|
found = 0
|
|
}
|
|
is_cni_table($0) {
|
|
in_cni = 1
|
|
print
|
|
next
|
|
}
|
|
in_cni && is_table($0) {
|
|
in_cni = 0
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
|
|
sub(/=.*/, "= \"/opt/cni/bin\"")
|
|
found = 1
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
|
|
sub(/=.*/, "= [\"/opt/cni/bin\"]")
|
|
found = 1
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!found) {
|
|
print ""
|
|
if (config_version == "3") {
|
|
print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
|
|
print " bin_dirs = [\"/opt/cni/bin\"]"
|
|
} else {
|
|
print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
|
|
print " bin_dir = \"/opt/cni/bin\""
|
|
}
|
|
print " conf_dir = \"/etc/cni/net.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
install_cni_plugins() {
|
|
local plugin
|
|
|
|
sudo mkdir -p /opt/cni/bin
|
|
sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
|
|
sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
|
|
if [ -d /usr/lib/cni ]; then
|
|
for plugin in /usr/lib/cni/*; do
|
|
[ -f "$plugin" ] && [ -x "$plugin" ] || continue
|
|
sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
|
|
done
|
|
fi
|
|
}
|
|
|
|
configure_containerd_registry() {
|
|
local registry_endpoint="$1"
|
|
local config_version
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
|
|
ensure_containerd_cni_bin_dir
|
|
config_version="$(containerd_config_version)"
|
|
if [ "$config_version" = "3" ]; then
|
|
ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
|
|
else
|
|
ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
|
|
fi
|
|
|
|
sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
|
|
sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
|
|
server = "http://$registry_endpoint"
|
|
|
|
[host."http://$registry_endpoint"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
if ! sudo containerd config dump >/dev/null; then
|
|
sudo containerd config dump || true
|
|
exit 1
|
|
fi
|
|
if ! sudo systemctl restart containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages containernetworking-plugins open-iscsi nfs-common
|
|
install_cni_plugins
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
configure_containerd_registry "${self.triggers.registry_endpoint}"
|
|
|
|
IFS=',' read -r -a pv_dirs <<< "${self.triggers.persistent_volume_dirs}"
|
|
for path in "$${pv_dirs[@]}"; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ] && [ -d /etc/kubernetes ]; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/admin.conf ]; then
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
if ! sudo kubeadm init \
|
|
--pod-network-cidr=${self.triggers.pod_network_cidr} \
|
|
--node-name=${self.triggers.node_name} \
|
|
--apiserver-advertise-address=${self.triggers.advertise_address}; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")"
|
|
sudo cp -f /etc/kubernetes/admin.conf "${self.triggers.kubeconfig_path}"
|
|
sudo chown ${self.triggers.kubeconfig_owner} "${self.triggers.kubeconfig_path}"
|
|
|
|
kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint nodes "${self.triggers.node_name}" node-role.kubernetes.io/control-plane- || true
|
|
EOT
|
|
}
|
|
}
|
|
|
|
data "external" "kubeadm_join_command" {
|
|
depends_on = [null_resource.kubeadm_control_plane]
|
|
program = [
|
|
"bash",
|
|
"-lc",
|
|
<<EOT
|
|
set -euo pipefail
|
|
cmd="$(sudo kubeadm token create --print-join-command)"
|
|
printf '{"cmd":"%s"}\n' "$(printf '%s' "$cmd" | sed 's/\\/\\\\/g; s/"/\\"/g')"
|
|
EOT
|
|
]
|
|
}
|
|
|
|
resource "null_resource" "kubeadm_worker" {
|
|
for_each = var.worker_nodes
|
|
|
|
depends_on = [data.external.kubeadm_join_command]
|
|
|
|
triggers = {
|
|
node_name = each.value.node_name
|
|
host = each.value.host
|
|
user = each.value.user
|
|
ssh_key_path = each.value.ssh_key_path
|
|
registry_endpoint = var.registry_endpoint
|
|
registry_config_version = "8"
|
|
cni_plugins_version = "2"
|
|
node_dns_servers = join(" ", var.node_dns_servers)
|
|
persistent_volume_dirs = join(",", var.persistent_volume_dirs)
|
|
tailscale_nodeport_version = "3"
|
|
tailscale_nodeport_enabled = var.tailscale_nodeport_access.enabled && each.key == var.tailscale_nodeport_access.worker_key ? "true" : "false"
|
|
tailscale_nodeport_peer_ip = var.tailscale_nodeport_access.peer_ip
|
|
tailscale_nodeport_node_tailscale_ip = var.tailscale_nodeport_access.node_tailscale_ip
|
|
tailscale_nodeport_pod_cidr = var.tailscale_nodeport_access.pod_cidr
|
|
tailscale_nodeport_node_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.node_port], var.tailscale_nodeport_extra_ports)))
|
|
tailscale_nodeport_target_ports = join(" ", distinct(concat([var.tailscale_nodeport_access.target_port], var.tailscale_nodeport_extra_target_ports)))
|
|
tailscale_subnet_routes_version = "1"
|
|
tailscale_subnet_routes_enabled = var.tailscale_subnet_routes.enabled && each.key == var.tailscale_subnet_routes.worker_key ? "true" : "false"
|
|
tailscale_subnet_routes = join(",", var.tailscale_subnet_routes.routes)
|
|
}
|
|
|
|
connection {
|
|
type = "ssh"
|
|
user = self.triggers.user
|
|
private_key = file(self.triggers.ssh_key_path)
|
|
host = self.triggers.host
|
|
}
|
|
|
|
provisioner "remote-exec" {
|
|
inline = [
|
|
<<EOT
|
|
set -eu
|
|
|
|
install_missing_packages() {
|
|
missing_packages=""
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='$${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages="$missing_packages $package"
|
|
fi
|
|
done
|
|
if [ -n "$missing_packages" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends $missing_packages
|
|
fi
|
|
}
|
|
|
|
configure_node_dns() {
|
|
dns_servers="${self.triggers.node_dns_servers}"
|
|
if [ -z "$dns_servers" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then
|
|
sudo mkdir -p /etc/systemd/resolved.conf.d
|
|
{
|
|
echo "[Resolve]"
|
|
printf 'DNS=%s\n' "$dns_servers"
|
|
printf 'FallbackDNS=%s\n' "$dns_servers"
|
|
echo "DNSSEC=no"
|
|
} | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
fi
|
|
|
|
if ! getent hosts quay.io >/dev/null 2>&1; then
|
|
sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true
|
|
sudo rm -f /etc/resolv.conf
|
|
for server in $dns_servers; do
|
|
printf 'nameserver %s\n' "$server"
|
|
done | sudo tee /etc/resolv.conf >/dev/null
|
|
fi
|
|
}
|
|
|
|
remove_containerd_section() {
|
|
local section="$1"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v section="$section" '
|
|
$0 == section { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_registry_config_path() {
|
|
local plugin="$1"
|
|
local append_section="$2"
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk -v plugin="$plugin" -v append_section="$append_section" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_target_registry(line) {
|
|
return is_table(line) &&
|
|
index(line, plugin) > 0 &&
|
|
line ~ /[.]registry[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_target = 0
|
|
found = 0
|
|
wrote = 0
|
|
}
|
|
is_target_registry($0) {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 1
|
|
found = 1
|
|
wrote = 0
|
|
print
|
|
next
|
|
}
|
|
in_target && is_table($0) {
|
|
if (!wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
in_target = 0
|
|
wrote = 0
|
|
}
|
|
in_target && $0 ~ /^[[:space:]]*config_path[[:space:]]*=/ {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
wrote = 1
|
|
next
|
|
}
|
|
{ print }
|
|
END {
|
|
if (in_target && !wrote) {
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
if (!found) {
|
|
print ""
|
|
print append_section
|
|
print " config_path = \"/etc/containerd/certs.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
containerd_config_version() {
|
|
sudo awk -F= '
|
|
/^[[:space:]]*version[[:space:]]*=/ {
|
|
gsub(/[[:space:]]/, "", $2)
|
|
print $2
|
|
exit
|
|
}
|
|
' /etc/containerd/config.toml
|
|
}
|
|
|
|
reset_containerd_registry_tables() {
|
|
local tmp
|
|
|
|
tmp="$(mktemp)"
|
|
sudo awk '
|
|
function is_registry_table(line) {
|
|
return line ~ /^\[plugins\./ &&
|
|
line ~ /\.registry([.\]]|$)/ &&
|
|
(line ~ /io[.]containerd[.]grpc[.]v1[.]cri/ ||
|
|
line ~ /io[.]containerd[.]cri[.]v1[.]images/)
|
|
}
|
|
is_registry_table($0) { skip = 1; next }
|
|
skip && /^\[/ { skip = 0 }
|
|
!skip { print }
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
ensure_containerd_cni_bin_dir() {
|
|
local config_version
|
|
local tmp
|
|
|
|
config_version="$(containerd_config_version)"
|
|
tmp="$(mktemp)"
|
|
sudo awk -v config_version="$config_version" '
|
|
function is_table(line) {
|
|
return line ~ /^[[:space:]]*\[/
|
|
}
|
|
function is_cni_table(line) {
|
|
return is_table(line) && line ~ /[.]cni[[:space:]]*\]/
|
|
}
|
|
BEGIN {
|
|
in_cni = 0
|
|
found = 0
|
|
}
|
|
is_cni_table($0) {
|
|
in_cni = 1
|
|
print
|
|
next
|
|
}
|
|
in_cni && is_table($0) {
|
|
in_cni = 0
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dir[[:space:]]*=/ {
|
|
sub(/=.*/, "= \"/opt/cni/bin\"")
|
|
found = 1
|
|
}
|
|
in_cni && /^[[:space:]]*bin_dirs[[:space:]]*=/ {
|
|
sub(/=.*/, "= [\"/opt/cni/bin\"]")
|
|
found = 1
|
|
}
|
|
{ print }
|
|
END {
|
|
if (!found) {
|
|
print ""
|
|
if (config_version == "3") {
|
|
print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
|
|
print " bin_dirs = [\"/opt/cni/bin\"]"
|
|
} else {
|
|
print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
|
|
print " bin_dir = \"/opt/cni/bin\""
|
|
}
|
|
print " conf_dir = \"/etc/cni/net.d\""
|
|
}
|
|
}
|
|
' /etc/containerd/config.toml > "$tmp"
|
|
sudo mv "$tmp" /etc/containerd/config.toml
|
|
}
|
|
|
|
install_cni_plugins() {
|
|
local plugin
|
|
|
|
sudo mkdir -p /opt/cni/bin
|
|
sudo find /opt/cni/bin -maxdepth 1 -type f ! -perm -111 -delete
|
|
sudo find /opt/cni/bin -maxdepth 1 -type l ! -exec test -x {} \; -delete
|
|
if [ -d /usr/lib/cni ]; then
|
|
for plugin in /usr/lib/cni/*; do
|
|
[ -f "$plugin" ] && [ -x "$plugin" ] || continue
|
|
sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
|
|
done
|
|
fi
|
|
}
|
|
|
|
configure_containerd_registry() {
|
|
local registry_endpoint="$1"
|
|
local config_version
|
|
|
|
sudo mkdir -p /etc/containerd
|
|
sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
|
|
|
|
sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
|
|
ensure_containerd_cni_bin_dir
|
|
config_version="$(containerd_config_version)"
|
|
if [ "$config_version" = "3" ]; then
|
|
ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
|
|
else
|
|
ensure_containerd_registry_config_path "io.containerd.grpc.v1.cri" '[plugins."io.containerd.grpc.v1.cri".registry]'
|
|
fi
|
|
|
|
sudo mkdir -p "/etc/containerd/certs.d/$registry_endpoint"
|
|
sudo tee "/etc/containerd/certs.d/$registry_endpoint/hosts.toml" >/dev/null <<REGISTRY_EOT
|
|
server = "http://$registry_endpoint"
|
|
|
|
[host."http://$registry_endpoint"]
|
|
capabilities = ["pull", "resolve", "push"]
|
|
skip_verify = true
|
|
REGISTRY_EOT
|
|
if ! sudo containerd config dump >/dev/null; then
|
|
sudo containerd config dump || true
|
|
exit 1
|
|
fi
|
|
if ! sudo systemctl restart containerd; then
|
|
sudo systemctl status containerd --no-pager -l || true
|
|
sudo journalctl -u containerd --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
configure_node_dns
|
|
install_missing_packages containernetworking-plugins open-iscsi nfs-common
|
|
install_cni_plugins
|
|
sudo systemctl enable --now iscsid
|
|
sudo systemctl enable kubelet || true
|
|
|
|
sudo swapoff -a || true
|
|
sudo awk '
|
|
/^[[:space:]]*#/ { print; next }
|
|
$3 == "swap" { print "# kubeadm-disabled " $0; next }
|
|
{ print }
|
|
' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null
|
|
sudo mv /etc/fstab.kubeadm /etc/fstab
|
|
|
|
sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT'
|
|
overlay
|
|
br_netfilter
|
|
MODULES_EOT
|
|
sudo modprobe overlay || true
|
|
sudo modprobe br_netfilter || true
|
|
|
|
sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT'
|
|
net.bridge.bridge-nf-call-iptables = 1
|
|
net.bridge.bridge-nf-call-ip6tables = 1
|
|
net.ipv4.ip_forward = 1
|
|
SYSCTL_EOT
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then
|
|
sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null
|
|
sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null
|
|
fi
|
|
|
|
if ! getent hosts "${self.triggers.node_name}" >/dev/null; then
|
|
printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null
|
|
fi
|
|
|
|
configure_tailscale_nodeport_access() {
|
|
local enabled="$1"
|
|
local peer_ip="$2"
|
|
local node_tailscale_ip="$3"
|
|
local pod_cidr="$4"
|
|
local node_ports="$5"
|
|
local target_ports="$6"
|
|
|
|
if [ "$enabled" != "true" ]; then
|
|
return 0
|
|
fi
|
|
|
|
sudo mkdir -p /usr/local/sbin /etc/sysctl.d
|
|
sudo tee /etc/sysctl.d/98-homelab-tailscale-nodeport.conf >/dev/null <<NODEPORT_SYSCTL_EOT
|
|
net.ipv4.conf.all.rp_filter = 0
|
|
net.ipv4.conf.tailscale0.rp_filter = 0
|
|
NODEPORT_SYSCTL_EOT
|
|
|
|
sudo tee /usr/local/sbin/homelab-tailscale-nodeport.sh >/dev/null <<NODEPORT_SCRIPT_EOT
|
|
#!/bin/sh
|
|
set -eu
|
|
|
|
sysctl -w net.ipv4.conf.all.rp_filter=0 >/dev/null
|
|
sysctl -w net.ipv4.conf.tailscale0.rp_filter=0 >/dev/null 2>&1 || true
|
|
|
|
if ! ip link show tailscale0 >/dev/null 2>&1; then
|
|
echo "tailscale0 is not present; skipping Tailscale NodePort routing"
|
|
exit 0
|
|
fi
|
|
|
|
ip route replace "$peer_ip/32" dev tailscale0 src "$node_tailscale_ip"
|
|
|
|
for node_port in $node_ports; do
|
|
iptables -C INPUT -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT 2>/dev/null ||
|
|
iptables -I INPUT 1 -i tailscale0 -p tcp --dport "\$node_port" -j ACCEPT
|
|
done
|
|
for target_port in $target_ports; do
|
|
iptables -C FORWARD -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -i tailscale0 -d "$pod_cidr" -p tcp --dport "\$target_port" -j ACCEPT
|
|
done
|
|
iptables -C FORWARD -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null ||
|
|
iptables -I FORWARD 1 -s "$pod_cidr" -o tailscale0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
|
|
for target_port in $target_ports; do
|
|
iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE 2>/dev/null ||
|
|
iptables -t nat -I POSTROUTING 1 -s 100.64.0.0/10 -d "$pod_cidr" -p tcp --dport "\$target_port" -m comment --comment tailscale-nodeport-to-pods -j MASQUERADE
|
|
done
|
|
NODEPORT_SCRIPT_EOT
|
|
sudo chmod 0755 /usr/local/sbin/homelab-tailscale-nodeport.sh
|
|
|
|
sudo tee /etc/systemd/system/homelab-tailscale-nodeport.service >/dev/null <<'NODEPORT_SERVICE_EOT'
|
|
[Unit]
|
|
Description=Homelab Tailscale NodePort routing
|
|
After=network-online.target tailscaled.service kubelet.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/sbin/homelab-tailscale-nodeport.sh
|
|
RemainAfterExit=yes
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
NODEPORT_SERVICE_EOT
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable homelab-tailscale-nodeport.service >/dev/null
|
|
sudo systemctl restart homelab-tailscale-nodeport.service
|
|
}
|
|
|
|
configure_tailscale_nodeport_access \
|
|
"${self.triggers.tailscale_nodeport_enabled}" \
|
|
"${self.triggers.tailscale_nodeport_peer_ip}" \
|
|
"${self.triggers.tailscale_nodeport_node_tailscale_ip}" \
|
|
"${self.triggers.tailscale_nodeport_pod_cidr}" \
|
|
"${self.triggers.tailscale_nodeport_node_ports}" \
|
|
"${self.triggers.tailscale_nodeport_target_ports}"
|
|
|
|
configure_tailscale_subnet_routes() {
|
|
local enabled="$1"
|
|
local routes="$2"
|
|
|
|
if [ "$enabled" != "true" ]; then
|
|
return 0
|
|
fi
|
|
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo "tailscale is required to advertise subnet routes but is not installed on this worker." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$routes" ]; then
|
|
echo "tailscale subnet route advertisement is enabled but no routes were configured." >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo tailscale set --advertise-routes="$routes"
|
|
}
|
|
|
|
configure_tailscale_subnet_routes \
|
|
"${self.triggers.tailscale_subnet_routes_enabled}" \
|
|
"${self.triggers.tailscale_subnet_routes}"
|
|
|
|
configure_containerd_registry "${self.triggers.registry_endpoint}"
|
|
|
|
pv_dirs="${self.triggers.persistent_volume_dirs}"
|
|
IFS=','
|
|
for path in $pv_dirs; do
|
|
sudo mkdir -p "$path"
|
|
sudo chmod 0775 "$path"
|
|
done
|
|
|
|
if [ -f /etc/kubernetes/kubelet.conf ] && ! timeout 5 bash -c 'exec 3<>/dev/tcp/127.0.0.1/10248; printf "GET /healthz HTTP/1.0\r\n\r\n" >&3; grep -q ok <&3' >/dev/null 2>&1; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d
|
|
fi
|
|
|
|
if [ ! -f /etc/kubernetes/kubelet.conf ]; then
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then
|
|
sudo systemctl status kubelet --no-pager -l || true
|
|
sudo journalctl -u kubelet --no-pager -n 160 || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
EOT
|
|
]
|
|
}
|
|
}
|
|
|
|
locals {
|
|
control_plane_node_label_pairs = [
|
|
for label, value in var.control_plane_node_labels : {
|
|
node_name = var.control_plane_node_name
|
|
label = label
|
|
value = value
|
|
}
|
|
]
|
|
|
|
worker_node_label_pairs = flatten([
|
|
for worker_key, worker in var.worker_nodes : [
|
|
for label, value in lookup(var.worker_node_labels, worker_key, {}) : {
|
|
node_name = worker.node_name
|
|
label = label
|
|
value = value
|
|
}
|
|
]
|
|
])
|
|
|
|
node_label_pairs = concat(local.control_plane_node_label_pairs, local.worker_node_label_pairs)
|
|
}
|
|
|
|
resource "null_resource" "node_labels" {
|
|
for_each = {
|
|
for pair in local.node_label_pairs : "${pair.node_name}/${pair.label}" => pair
|
|
}
|
|
|
|
depends_on = [
|
|
null_resource.kubeadm_control_plane,
|
|
null_resource.kubeadm_worker,
|
|
]
|
|
|
|
triggers = {
|
|
kubeconfig_path = var.kubeconfig_path
|
|
node_name = each.value.node_name
|
|
label = each.value.label
|
|
value = each.value.value
|
|
}
|
|
|
|
provisioner "local-exec" {
|
|
interpreter = ["/bin/bash", "-lc"]
|
|
environment = {
|
|
KUBECONFIG_PATH = self.triggers.kubeconfig_path
|
|
NODE_NAME = self.triggers.node_name
|
|
NODE_LABEL = self.triggers.label
|
|
NODE_LABEL_VALUE = self.triggers.value
|
|
}
|
|
command = <<EOT
|
|
set -euo pipefail
|
|
|
|
for _ in $(seq 1 60); do
|
|
if kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
kubectl --kubeconfig "$${KUBECONFIG_PATH}" get node "$${NODE_NAME}" >/dev/null
|
|
kubectl --kubeconfig "$${KUBECONFIG_PATH}" label node "$${NODE_NAME}" "$${NODE_LABEL}=$${NODE_LABEL_VALUE}" --overwrite
|
|
EOT
|
|
}
|
|
}
|
|
|
|
output "kubeconfig_path" {
|
|
value = var.kubeconfig_path
|
|
}
|
|
|
|
output "pod_network_cidr" {
|
|
value = var.pod_network_cidr
|
|
}
|