diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index ae45875..3eac5fc 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -20,6 +20,7 @@ resource "null_resource" "kubeadm_control_plane" { kubeconfig_path = var.kubeconfig_path kubeconfig_owner = var.kubeconfig_owner registry_endpoint = var.registry_endpoint + node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) } @@ -41,6 +42,33 @@ install_missing_packages() { fi } +configure_node_dns() { + dns_servers="${self.triggers.node_dns_servers}" + if [ -z "$dns_servers" ]; then + return 0 + fi + + if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then + sudo mkdir -p /etc/systemd/resolved.conf.d + { + echo "[Resolve]" + printf 'DNS=%s\n' "$dns_servers" + printf 'FallbackDNS=%s\n' "$dns_servers" + echo "DNSSEC=no" + } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null + sudo systemctl restart systemd-resolved 2>/dev/null || true + fi + + if ! getent hosts quay.io >/dev/null 2>&1; then + sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true + sudo rm -f /etc/resolv.conf + for server in $dns_servers; do + printf 'nameserver %s\n' "$server" + done | sudo tee /etc/resolv.conf >/dev/null + fi +} + +configure_node_dns install_missing_packages open-iscsi nfs-common sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true @@ -148,6 +176,7 @@ resource "null_resource" "kubeadm_worker" { user = each.value.user ssh_key_path = each.value.ssh_key_path registry_endpoint = var.registry_endpoint + node_dns_servers = join(" ", var.node_dns_servers) persistent_volume_dirs = join(",", var.persistent_volume_dirs) } @@ -176,6 +205,33 @@ install_missing_packages() { fi } +configure_node_dns() { + dns_servers="${self.triggers.node_dns_servers}" + if [ -z "$dns_servers" ]; then + return 0 + fi + + if systemctl list-unit-files systemd-resolved.service >/dev/null 2>&1; then + sudo mkdir -p /etc/systemd/resolved.conf.d + { + echo "[Resolve]" + printf 'DNS=%s\n' "$dns_servers" + printf 'FallbackDNS=%s\n' "$dns_servers" + echo "DNSSEC=no" + } | sudo tee /etc/systemd/resolved.conf.d/homelab-k8s.conf >/dev/null + sudo systemctl restart systemd-resolved 2>/dev/null || true + fi + + if ! getent hosts quay.io >/dev/null 2>&1; then + sudo cp -a /etc/resolv.conf /etc/resolv.conf.homelab-k8s-backup 2>/dev/null || true + sudo rm -f /etc/resolv.conf + for server in $dns_servers; do + printf 'nameserver %s\n' "$server" + done | sudo tee /etc/resolv.conf >/dev/null + fi +} + +configure_node_dns install_missing_packages open-iscsi nfs-common sudo systemctl enable --now iscsid sudo systemctl enable kubelet || true diff --git a/bootstrap/cluster/variables.tf b/bootstrap/cluster/variables.tf index 31c377e..c38572a 100644 --- a/bootstrap/cluster/variables.tf +++ b/bootstrap/cluster/variables.tf @@ -28,6 +28,14 @@ variable "registry_endpoint" { default = "192.168.100.68:30500" } +variable "node_dns_servers" { + type = list(string) + default = [ + "1.1.1.1", + "8.8.8.8", + ] +} + variable "persistent_volume_dirs" { type = list(string) default = [ diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf index 9870629..0c5267a 100644 --- a/bootstrap/platform/main.tf +++ b/bootstrap/platform/main.tf @@ -79,6 +79,10 @@ resource "helm_release" "calico" { values = [ yamlencode({ manageCRDs = false + nodeSelector = { + "kubernetes.io/os" = "linux" + "kubernetes.io/hostname" = var.calico_operator_node_name + } apiServer = { enabled = false } diff --git a/bootstrap/platform/variables.tf b/bootstrap/platform/variables.tf index 245961b..0a0d0e6 100644 --- a/bootstrap/platform/variables.tf +++ b/bootstrap/platform/variables.tf @@ -8,6 +8,11 @@ variable "pod_network_cidr" { default = "10.244.0.0/16" } +variable "calico_operator_node_name" { + type = string + default = "debian" +} + variable "gitops_repo_url" { type = string default = "ssh://jv@192.168.100.68/home/jv/git-server/my-homelab-configs.git" diff --git a/lab.sh b/lab.sh index dc8571a..21b573f 100755 --- a/lab.sh +++ b/lab.sh @@ -37,6 +37,15 @@ cleanup_iptables() { fi } +restore_node_dns() { + sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf + if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then + sudo rm -f /etc/resolv.conf + sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf + fi + sudo systemctl restart systemd-resolved 2>/dev/null || true +} + cleanup_mounts() { if command -v findmnt >/dev/null 2>&1; then while IFS= read -r mountpoint; do @@ -76,6 +85,7 @@ cleanup_node() { cleanup_iptables cleanup_calico_links + restore_node_dns sudo mkdir -p /etc/containerd/certs.d sudo systemctl reset-failed kubelet containerd 2>/dev/null || true @@ -190,6 +200,15 @@ cleanup_iptables() { fi } +restore_node_dns() { + sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf + if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then + sudo rm -f /etc/resolv.conf + sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf + fi + sudo systemctl restart systemd-resolved 2>/dev/null || true +} + cleanup_mounts() { if command -v findmnt >/dev/null 2>&1; then while IFS= read -r mountpoint; do @@ -228,6 +247,7 @@ sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam cleanup_iptables cleanup_calico_links +restore_node_dns sudo mkdir -p /etc/containerd/certs.d sudo systemctl reset-failed kubelet containerd 2>/dev/null || true