Fixing k8s api timing

bootstrap/cluster/main.tf

@@ -416,7 +416,7 @@ resource "null_resource" "kube_system_resource_policy" {
 
   triggers = {
     kubeconfig_path         = var.kubeconfig_path
-    resource_policy_version = "1"
+    resource_policy_version = "2"
   }
 
   provisioner "local-exec" {
@@ -468,21 +468,35 @@ patch_cluster_workload_resources() {
     --limits="cpu=$${limit_cpu},memory=$${limit_memory}"
 }
 
-patch_static_pod_resources etcd 100m 256Mi 500m 1Gi
+wait_for_apiserver_ready() {
-patch_static_pod_resources kube-apiserver 100m 256Mi 500m 1Gi
-patch_static_pod_resources kube-controller-manager 50m 128Mi 250m 512Mi
-patch_static_pod_resources kube-scheduler 50m 128Mi 200m 256Mi
-
   for _ in $(seq 1 60); do
     if kubectl --kubeconfig "${self.triggers.kubeconfig_path}" get --raw=/readyz >/dev/null 2>&1; then
-    break
+      return 0
     fi
     sleep 5
   done
   kubectl --kubeconfig "${self.triggers.kubeconfig_path}" get --raw=/readyz >/dev/null
+}
+
+wait_for_apiserver_stable() {
+  wait_for_apiserver_ready
+  for _ in $(seq 1 6); do
+    sleep 5
+    kubectl --kubeconfig "${self.triggers.kubeconfig_path}" get --raw=/readyz >/dev/null
+  done
+}
+
+patch_static_pod_resources etcd 100m 256Mi 1000m 2Gi
+patch_static_pod_resources kube-apiserver 200m 512Mi 1000m 2Gi
+patch_static_pod_resources kube-controller-manager 100m 128Mi 500m 768Mi
+patch_static_pod_resources kube-scheduler 50m 128Mi 300m 512Mi
+
+wait_for_apiserver_stable
 
 patch_cluster_workload_resources kube-system deployment/coredns coredns 50m 128Mi 200m 256Mi
 patch_cluster_workload_resources kube-system daemonset/kube-proxy kube-proxy 50m 64Mi 200m 256Mi
+
+wait_for_apiserver_stable
 EOT
   }
 }

jeannie

@@ -503,12 +503,14 @@ run_tofu_stack() {
     fi
 
     tofu -chdir="${REPO_ROOT}/${stack}" init
+    ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
     if [[ "${stack}" == "bootstrap/platform" ]]; then
         adopt_platform_existing_resources
     fi
     if [[ "${stack}" == "bootstrap/apps" ]]; then
         adopt_apps_existing_resources
     fi
+    ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
     tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}"
 }
 
@@ -5864,6 +5866,47 @@ kubernetes_api_reachable() {
     kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1
 }
 
+wait_for_kubernetes_api_stable() {
+    local timeout_seconds="${1:-300}"
+    local stable_seconds="${2:-30}"
+    local interval_seconds="${3:-5}"
+    local elapsed=0
+    local stable_elapsed=0
+
+    echo "Waiting for Kubernetes API to stay ready for ${stable_seconds}s..."
+    until ((stable_elapsed >= stable_seconds)); do
+        if kubernetes_api_reachable; then
+            stable_elapsed=$((stable_elapsed + interval_seconds))
+        else
+            stable_elapsed=0
+        fi
+
+        if ((stable_elapsed >= stable_seconds)); then
+            return 0
+        fi
+        if ((elapsed >= timeout_seconds)); then
+            echo "Kubernetes API did not stay ready for ${stable_seconds}s within ${timeout_seconds}s." >&2
+            return 1
+        fi
+
+        sleep "${interval_seconds}"
+        elapsed=$((elapsed + interval_seconds))
+    done
+}
+
+ensure_kubernetes_api_ready_for_tofu_stack() {
+    local stack="$1"
+
+    case "${stack}" in
+        bootstrap/platform|bootstrap/apps)
+            wait_for_kubernetes_api_stable \
+                "${LAB_KUBERNETES_API_WAIT_TIMEOUT_SECONDS:-300}" \
+                "${LAB_KUBERNETES_API_STABLE_SECONDS:-30}" \
+                "${LAB_KUBERNETES_API_STABLE_INTERVAL_SECONDS:-5}"
+            ;;
+    esac
+}
+
 cluster_control_plane_tracked() {
     tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane"
 }