jv
/
my-homelab-configs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Limit diagnostic to failed component

This commit is contained in:
juvdiaz 2026-07-02 20:21:31 -06:00
parent 1e63d0cbc8
commit d5458a3482
1 changed files with 43 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
bootstrap/platform/main.tf
View File

@ -1221,7 +1221,7 @@ resource "null_resource" "argocd_ready" {
namespace = var.argocd.namespace namespace = var.argocd.namespace
version = var.argocd.version version = var.argocd.version
node_selector = jsonencode(local.argocd_node_selector) node_selector = jsonencode(local.argocd_node_selector)
readiness_check = "2" readiness_check = "3"
} }
provisioner "local-exec" { provisioner "local-exec" {
@ -1233,40 +1233,64 @@ namespace="${self.triggers.namespace}"
kubeconfig="${self.triggers.kubeconfig_path}" kubeconfig="${self.triggers.kubeconfig_path}"
dump_argocd_readiness_debug() { dump_argocd_readiness_debug() {
target="$1"
selector="$2"
echo echo
echo "Argo CD readiness diagnostics:" echo "ARGOCD_READY_FAILED_TARGET=$target"
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get deploy,statefulset,replicaset,pods,svc,endpoints -o wide || true echo "ARGOCD_READY_FAILED_SELECTOR=$selector"
echo echo
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -80 || true kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o wide || true
echo echo
kubectl --kubeconfig "$kubeconfig" -n "$namespace" describe pods || true kubectl --kubeconfig "$kubeconfig" -n "$namespace" get deploy,statefulset,pods,svc,endpoints -o wide || true
echo echo
kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs deployment/argocd-repo-server --tail=120 || true kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -40 || true
kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs deployment/argocd-server --tail=120 || true echo
kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs statefulset/argocd-application-controller --tail=120 || true kubectl --kubeconfig "$kubeconfig" -n "$namespace" describe pods -l "$selector" || true
echo
kubectl --kubeconfig "$kubeconfig" -n "$namespace" logs -l "$selector" --all-containers --tail=40 --prefix || true
echo
echo "ARGOCD_READY_FAILED_TARGET=$target"
} }
rollout_if_present() { wait_for_argocd_component() {
target="$1" target="$1"
selector="$2"
if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" get "$target" >/dev/null 2>&1; then if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" get "$target" >/dev/null 2>&1; then
echo "Argo CD $target is not present yet; skipping rollout wait." echo "Argo CD $target is not present yet; skipping rollout wait."
return 0 return 0
fi fi
if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" rollout status "$target" --timeout=600s; then
echo "Timed out waiting for Argo CD $target." >&2 echo "Waiting for Argo CD $target pods with selector $selector..."
dump_argocd_readiness_debug for _ in $(seq 1 120); do
pod_count="$(kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o name 2>/dev/null | wc -l | tr -d ' ')"
if [ "$pod_count" -gt 0 ]; then
break
fi
sleep 5
done
if [ "$pod_count" -eq 0 ]; then
echo "Timed out waiting for pods for Argo CD $target." >&2
dump_argocd_readiness_debug "$target" "$selector"
exit 1
fi
if ! kubectl --kubeconfig "$kubeconfig" -n "$namespace" wait --for=condition=Ready pod -l "$selector" --timeout=600s; then
echo "Timed out waiting for ready pods for Argo CD $target." >&2
dump_argocd_readiness_debug "$target" "$selector"
exit 1 exit 1
fi fi
} }
kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Established --timeout=180s crd/applications.argoproj.io kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Established --timeout=180s crd/applications.argoproj.io
rollout_if_present deployment/argocd-applicationset-controller wait_for_argocd_component deployment/argocd-applicationset-controller app.kubernetes.io/name=argocd-applicationset-controller
rollout_if_present statefulset/argocd-application-controller wait_for_argocd_component statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller
rollout_if_present deployment/argocd-dex-server wait_for_argocd_component deployment/argocd-dex-server app.kubernetes.io/name=argocd-dex-server
rollout_if_present deployment/argocd-notifications-controller wait_for_argocd_component deployment/argocd-notifications-controller app.kubernetes.io/name=argocd-notifications-controller
rollout_if_present deployment/argocd-redis wait_for_argocd_component deployment/argocd-redis app.kubernetes.io/name=argocd-redis
rollout_if_present deployment/argocd-repo-server wait_for_argocd_component deployment/argocd-repo-server app.kubernetes.io/name=argocd-repo-server
rollout_if_present deployment/argocd-server wait_for_argocd_component deployment/argocd-server app.kubernetes.io/name=argocd-server
EOT EOT
} }
} }