Make helm do not block

This commit is contained in:
juvenal.diaz 2026-07-22 23:10:11 -06:00
parent 101668fbc9
commit 005a3c4b84
1 changed files with 48 additions and 3 deletions

View File

@ -1770,8 +1770,10 @@ resource "helm_release" "prometheus_stack" {
version = var.observability.prometheus.version version = var.observability.prometheus.version
namespace = var.observability.namespace namespace = var.observability.namespace
create_namespace = false create_namespace = false
timeout = 1200 # Readiness is checked by the following resource so a blocked PVC or pod
wait = true # produces actionable Kubernetes diagnostics instead of a provider timeout.
timeout = 600
wait = false
cleanup_on_fail = true cleanup_on_fail = true
values = [ values = [
@ -1973,6 +1975,49 @@ resource "helm_release" "prometheus_stack" {
] ]
} }
resource "null_resource" "prometheus_stack_readiness" {
depends_on = [helm_release.prometheus_stack]
triggers = {
kubeconfig_path = var.kubeconfig_path
namespace = var.observability.namespace
release_version = var.observability.prometheus.version
}
provisioner "local-exec" {
interpreter = ["/bin/bash", "-lc"]
command = <<EOT
set -euo pipefail
kubeconfig="${self.triggers.kubeconfig_path}"
namespace="${self.triggers.namespace}"
selector="app.kubernetes.io/instance=prometheus-stack"
timeout_seconds=600
elapsed=0
dump_prometheus_debug() {
echo "Prometheus stack readiness diagnostics:" >&2
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods,sts,deploy,pvc -l "$selector" -o wide >&2 || true
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -n 80 >&2 || true
}
trap dump_prometheus_debug ERR
until kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{range .status.conditions[*]}{.type}={.status}{" "}{end}{"\n"}{end}' 2>/dev/null |
awk 'NF > 0 { count++ } ($2 == "Succeeded") || ($2 == "Running" && $0 ~ /Ready=True/) { next } { failed = 1 } END { exit !(count > 0 && !failed) }'; do
if [ "$elapsed" -ge "$timeout_seconds" ]; then
echo "Timed out waiting for Prometheus stack workloads after $${timeout_seconds}s." >&2
exit 1
fi
sleep 10
elapsed=$((elapsed + 10))
done
echo "Prometheus stack workloads are ready."
EOT
}
}
resource "kubernetes_config_map" "grafana_dashboards" { resource "kubernetes_config_map" "grafana_dashboards" {
for_each = local.grafana_dashboard_files for_each = local.grafana_dashboard_files
@ -1991,7 +2036,7 @@ resource "kubernetes_config_map" "grafana_dashboards" {
(each.key) = file("${path.module}/grafana-dashboards/${each.key}") (each.key) = file("${path.module}/grafana-dashboards/${each.key}")
} }
depends_on = [helm_release.prometheus_stack] depends_on = [null_resource.prometheus_stack_readiness]
} }
resource "helm_release" "extra_tools" { resource "helm_release" "extra_tools" {