diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf index 1a6a350..acee37f 100644 --- a/bootstrap/platform/main.tf +++ b/bootstrap/platform/main.tf @@ -1221,7 +1221,7 @@ resource "null_resource" "argocd_ready" { namespace = var.argocd.namespace version = var.argocd.version node_selector = jsonencode(local.argocd_node_selector) - readiness_check = "2" + readiness_check = "3" } provisioner "local-exec" { @@ -1233,40 +1233,64 @@ namespace="${self.triggers.namespace}" kubeconfig="${self.triggers.kubeconfig_path}" dump_argocd_readiness_debug() { + target="$1" + selector="$2" + echo - echo "Argo CD readiness diagnostics:" - kubectl --kubeconfig "$kubeconfig" -n "$namespace" get deploy,statefulset,replicaset,pods,svc,endpoints -o wide || true + echo "ARGOCD_READY_FAILED_TARGET=$target" + echo "ARGOCD_READY_FAILED_SELECTOR=$selector" echo - kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -80 || true + kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o wide || true echo - kubectl --kubeconfig "$kubeconfig" -n "$namespace" describe pods || true + kubectl --kubeconfig "$kubeconfig" -n "$namespace" get deploy,statefulset,pods,svc,endpoints -o wide || true echo - kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs deployment/argocd-repo-server --tail=120 || true - kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs deployment/argocd-server --tail=120 || true - kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs statefulset/argocd-application-controller --tail=120 || true + kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -40 || true + echo + kubectl --kubeconfig "$kubeconfig" -n "$namespace" describe pods -l "$selector" || true + echo + kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs -l "$selector" --all-containers --tail=40 --prefix || true + echo + echo "ARGOCD_READY_FAILED_TARGET=$target" } -rollout_if_present() { +wait_for_argocd_component() { target="$1" + selector="$2" + if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" get "$target" >/dev/null 2>&1; then echo "Argo CD $target is not present yet; skipping rollout wait." return 0 fi - if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" rollout status "$target" --timeout=600s; then - echo "Timed out waiting for Argo CD $target." >&2 - dump_argocd_readiness_debug + + echo "Waiting for Argo CD $target pods with selector $selector..." + for _ in $(seq 1 120); do + pod_count="$(kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o name 2>/dev/null | wc -l | tr -d ' ')" + if [ "$pod_count" -gt 0 ]; then + break + fi + sleep 5 + done + if [ "$pod_count" -eq 0 ]; then + echo "Timed out waiting for pods for Argo CD $target." >&2 + dump_argocd_readiness_debug "$target" "$selector" + exit 1 + fi + + if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" wait --for=condition=Ready pod -l "$selector" --timeout=600s; then + echo "Timed out waiting for ready pods for Argo CD $target." >&2 + dump_argocd_readiness_debug "$target" "$selector" exit 1 fi } kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Established --timeout=180s crd/applications.argoproj.io -rollout_if_present deployment/argocd-applicationset-controller -rollout_if_present statefulset/argocd-application-controller -rollout_if_present deployment/argocd-dex-server -rollout_if_present deployment/argocd-notifications-controller -rollout_if_present deployment/argocd-redis -rollout_if_present deployment/argocd-repo-server -rollout_if_present deployment/argocd-server +wait_for_argocd_component deployment/argocd-applicationset-controller app.kubernetes.io/name=argocd-applicationset-controller +wait_for_argocd_component statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller +wait_for_argocd_component deployment/argocd-dex-server app.kubernetes.io/name=argocd-dex-server +wait_for_argocd_component deployment/argocd-notifications-controller app.kubernetes.io/name=argocd-notifications-controller +wait_for_argocd_component deployment/argocd-redis app.kubernetes.io/name=argocd-redis +wait_for_argocd_component deployment/argocd-repo-server app.kubernetes.io/name=argocd-repo-server +wait_for_argocd_component deployment/argocd-server app.kubernetes.io/name=argocd-server EOT } }