Tune kubelet disk pressure thresholds

bootstrap/cluster/main.tf

@@ -23,6 +23,7 @@ resource "null_resource" "kubeadm_control_plane" {
     registry_config_version = "8"
     containerd_runtime      = "docker-containerd-io"
     cni_plugins_version     = "2"
+    kubelet_eviction_version = "1"
     node_dns_servers        = join(" ", var.node_dns_servers)
     persistent_volume_dirs  = join(",", var.persistent_volume_dirs)
   }
@@ -93,6 +94,13 @@ install_missing_packages() {
   fi
 }
 
+configure_kubelet_eviction_thresholds() {
+  sudo tee /etc/default/kubelet >/dev/null <<'KUBELET_DEFAULTS_EOT'
+KUBELET_EXTRA_ARGS="--eviction-hard=memory.available<100Mi,nodefs.available<50Gi,nodefs.inodesFree<5%,imagefs.available<50Gi,imagefs.inodesFree<5%"
+KUBELET_DEFAULTS_EOT
+  sudo systemctl daemon-reload
+}
+
 install_containerd_runtime() {
   codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
   arch="$(dpkg --print-architecture)"
@@ -359,6 +367,7 @@ install_missing_packages ca-certificates curl gpg apt-transport-https containern
 install_containerd_runtime
 install_cni_plugins
 sudo systemctl enable --now iscsid
+configure_kubelet_eviction_thresholds
 sudo systemctl enable kubelet || true
 
 sudo swapoff -a || true
@@ -572,6 +581,7 @@ resource "null_resource" "kubeadm_worker" {
     cni_plugins_version                  = "2"
     force_rejoin                         = tostring(var.force_worker_rejoin)
     worker_rejoin_reset_version          = "2"
+    kubelet_eviction_version             = "1"
     node_dns_servers                     = join(" ", var.node_dns_servers)
     persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
     worker_tailscale_version             = "1"
@@ -664,6 +674,13 @@ install_missing_packages() {
   fi
 }
 
+configure_kubelet_eviction_thresholds() {
+  sudo tee /etc/default/kubelet >/dev/null <<'KUBELET_DEFAULTS_EOT'
+KUBELET_EXTRA_ARGS="--eviction-hard=memory.available<100Mi,nodefs.available<50Gi,nodefs.inodesFree<5%,imagefs.available<50Gi,imagefs.inodesFree<5%"
+KUBELET_DEFAULTS_EOT
+  sudo systemctl daemon-reload
+}
+
 install_containerd_runtime() {
   codename="$(. /etc/os-release && printf '%s' "$${VERSION_CODENAME:-trixie}")"
   arch="$(dpkg --print-architecture)"
@@ -1015,6 +1032,7 @@ install_containerd_runtime
 install_tailscale
 install_cni_plugins
 sudo systemctl enable --now iscsid
+configure_kubelet_eviction_thresholds
 sudo systemctl enable kubelet || true
 
 sudo swapoff -a || true

bootstrap/platform/main.tf

@@ -449,7 +449,7 @@ resource "null_resource" "calico_ready" {
   triggers = {
     kubeconfig_path                         = var.kubeconfig_path
     calico_version                          = var.calico.version
-    calico_ready_version                    = "2"
+    calico_ready_version                    = "3"
     pod_network_cidr                        = var.pod_network_cidr
     calico_node_address_autodetection_cidrs = join(",", var.calico_node_address_autodetection_cidrs)
     calico_operator_node_name               = var.calico_operator_node_name
@@ -499,6 +499,14 @@ kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint node "${self.trigg
 kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint node "${self.triggers.calico_operator_node_name}" node-role.kubernetes.io/master:NoSchedule- || true
 kubectl --kubeconfig "${self.triggers.kubeconfig_path}" taint node "${self.triggers.calico_operator_node_name}" node-role.kubernetes.io/master- || true
 
+if kubectl --kubeconfig "${self.triggers.kubeconfig_path}" get node "${self.triggers.calico_operator_node_name}" \
+  -o jsonpath='{.status.conditions[?(@.type=="DiskPressure")].status}' | grep -qx True; then
+  echo "Node ${self.triggers.calico_operator_node_name} has DiskPressure; Calico bootstrap pods will be evicted until disk pressure is cleared." >&2
+  df -h >&2 || true
+  dump_calico_debug
+  exit 1
+fi
+
 wait_for_resource deployment tigera-operator tigera-operator 300
 kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n tigera-operator patch deployment tigera-operator --type merge -p '{"spec":{"template":{"spec":{"tolerations":[{"operator":"Exists","effect":"NoSchedule"},{"operator":"Exists","effect":"NoExecute"}]}}}}'
 kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n tigera-operator rollout status deployment/tigera-operator --timeout=300s