diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index 8ae9678..da49e72 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -29,9 +29,39 @@ resource "null_resource" "kubeadm_control_plane" { set -euo pipefail sudo apt-get update -sudo apt-get install -y open-iscsi nfs-common +sudo apt-get install -y curl open-iscsi nfs-common sudo systemctl enable --now iscsid -sudo systemctl enable --now kubelet || true +sudo systemctl enable kubelet || true + +sudo swapoff -a || true +sudo awk ' + /^[[:space:]]*#/ { print; next } + $3 == "swap" { print "# kubeadm-disabled " $0; next } + { print } +' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null +sudo mv /etc/fstab.kubeadm /etc/fstab + +sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT' +overlay +br_netfilter +MODULES_EOT +sudo modprobe overlay || true +sudo modprobe br_netfilter || true + +sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT' +net.bridge.bridge-nf-call-iptables = 1 +net.bridge.bridge-nf-call-ip6tables = 1 +net.ipv4.ip_forward = 1 +SYSCTL_EOT +sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null +if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null + sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null +fi + +if ! getent hosts "${self.triggers.node_name}" >/dev/null; then + printf '%s %s\n' "${self.triggers.advertise_address}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null +fi sudo mkdir -p /etc/containerd if [ ! -f /etc/containerd/config.toml ]; then @@ -55,11 +85,22 @@ for path in "$${pv_dirs[@]}"; do sudo chmod 0775 "$path" done +if [ ! -f /etc/kubernetes/admin.conf ] && [ -d /etc/kubernetes ]; then + sudo kubeadm reset --force || true + sudo systemctl stop kubelet 2>/dev/null || true + sudo rm -rf /etc/kubernetes/ /var/lib/etcd/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d +fi + if [ ! -f /etc/kubernetes/admin.conf ]; then - sudo kubeadm init \ + sudo systemctl stop kubelet 2>/dev/null || true + if ! sudo kubeadm init \ --pod-network-cidr=${self.triggers.pod_network_cidr} \ --node-name=${self.triggers.node_name} \ - --apiserver-advertise-address=${self.triggers.advertise_address} + --apiserver-advertise-address=${self.triggers.advertise_address}; then + sudo systemctl status kubelet --no-pager -l || true + sudo journalctl -u kubelet --no-pager -n 160 || true + exit 1 + fi fi mkdir -p "$(dirname "${self.triggers.kubeconfig_path}")" @@ -111,9 +152,39 @@ resource "null_resource" "kubeadm_worker" { set -eu sudo apt-get update -sudo apt-get install -y open-iscsi nfs-common +sudo apt-get install -y curl open-iscsi nfs-common sudo systemctl enable --now iscsid -sudo systemctl enable --now kubelet || true +sudo systemctl enable kubelet || true + +sudo swapoff -a || true +sudo awk ' + /^[[:space:]]*#/ { print; next } + $3 == "swap" { print "# kubeadm-disabled " $0; next } + { print } +' /etc/fstab | sudo tee /etc/fstab.kubeadm >/dev/null +sudo mv /etc/fstab.kubeadm /etc/fstab + +sudo tee /etc/modules-load.d/k8s.conf >/dev/null <<'MODULES_EOT' +overlay +br_netfilter +MODULES_EOT +sudo modprobe overlay || true +sudo modprobe br_netfilter || true + +sudo tee /etc/sysctl.d/99-kubernetes-cri.conf >/dev/null <<'SYSCTL_EOT' +net.bridge.bridge-nf-call-iptables = 1 +net.bridge.bridge-nf-call-ip6tables = 1 +net.ipv4.ip_forward = 1 +SYSCTL_EOT +sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null +if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + sudo sysctl -w net.bridge.bridge-nf-call-iptables=1 >/dev/null + sudo sysctl -w net.bridge.bridge-nf-call-ip6tables=1 >/dev/null +fi + +if ! getent hosts "${self.triggers.node_name}" >/dev/null; then + printf '%s %s\n' "${self.triggers.host}" "${self.triggers.node_name}" | sudo tee -a /etc/hosts >/dev/null +fi sudo mkdir -p /etc/containerd if [ ! -f /etc/containerd/config.toml ]; then @@ -138,8 +209,25 @@ for path in $pv_dirs; do sudo chmod 0775 "$path" done +if [ -f /etc/kubernetes/kubelet.conf ] && ! curl -fsS --max-time 5 http://127.0.0.1:10248/healthz >/dev/null 2>&1; then + sudo kubeadm reset --force || true + sudo systemctl stop kubelet 2>/dev/null || true + sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d +fi + +if [ ! -f /etc/kubernetes/kubelet.conf ] && [ -e /var/lib/kubelet/kubeadm-flags.env ]; then + sudo kubeadm reset --force || true + sudo systemctl stop kubelet 2>/dev/null || true + sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d +fi + if [ ! -f /etc/kubernetes/kubelet.conf ]; then - sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name} + sudo systemctl stop kubelet 2>/dev/null || true + if ! sudo ${data.external.kubeadm_join_command.result.cmd} --node-name=${self.triggers.node_name}; then + sudo systemctl status kubelet --no-pager -l || true + sudo journalctl -u kubelet --no-pager -n 160 || true + exit 1 + fi fi EOT ]