Make helm do not block
This commit is contained in:
parent
101668fbc9
commit
005a3c4b84
|
|
@ -1770,8 +1770,10 @@ resource "helm_release" "prometheus_stack" {
|
||||||
version = var.observability.prometheus.version
|
version = var.observability.prometheus.version
|
||||||
namespace = var.observability.namespace
|
namespace = var.observability.namespace
|
||||||
create_namespace = false
|
create_namespace = false
|
||||||
timeout = 1200
|
# Readiness is checked by the following resource so a blocked PVC or pod
|
||||||
wait = true
|
# produces actionable Kubernetes diagnostics instead of a provider timeout.
|
||||||
|
timeout = 600
|
||||||
|
wait = false
|
||||||
cleanup_on_fail = true
|
cleanup_on_fail = true
|
||||||
|
|
||||||
values = [
|
values = [
|
||||||
|
|
@ -1973,6 +1975,49 @@ resource "helm_release" "prometheus_stack" {
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "null_resource" "prometheus_stack_readiness" {
|
||||||
|
depends_on = [helm_release.prometheus_stack]
|
||||||
|
|
||||||
|
triggers = {
|
||||||
|
kubeconfig_path = var.kubeconfig_path
|
||||||
|
namespace = var.observability.namespace
|
||||||
|
release_version = var.observability.prometheus.version
|
||||||
|
}
|
||||||
|
|
||||||
|
provisioner "local-exec" {
|
||||||
|
interpreter = ["/bin/bash", "-lc"]
|
||||||
|
command = <<EOT
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
kubeconfig="${self.triggers.kubeconfig_path}"
|
||||||
|
namespace="${self.triggers.namespace}"
|
||||||
|
selector="app.kubernetes.io/instance=prometheus-stack"
|
||||||
|
timeout_seconds=600
|
||||||
|
elapsed=0
|
||||||
|
|
||||||
|
dump_prometheus_debug() {
|
||||||
|
echo "Prometheus stack readiness diagnostics:" >&2
|
||||||
|
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods,sts,deploy,pvc -l "$selector" -o wide >&2 || true
|
||||||
|
kubectl --kubeconfig "$kubeconfig" -n "$namespace" get events --sort-by=.lastTimestamp | tail -n 80 >&2 || true
|
||||||
|
}
|
||||||
|
|
||||||
|
trap dump_prometheus_debug ERR
|
||||||
|
|
||||||
|
until kubectl --kubeconfig "$kubeconfig" -n "$namespace" get pods -l "$selector" -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{range .status.conditions[*]}{.type}={.status}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
||||||
|
awk 'NF > 0 { count++ } ($2 == "Succeeded") || ($2 == "Running" && $0 ~ /Ready=True/) { next } { failed = 1 } END { exit !(count > 0 && !failed) }'; do
|
||||||
|
if [ "$elapsed" -ge "$timeout_seconds" ]; then
|
||||||
|
echo "Timed out waiting for Prometheus stack workloads after $${timeout_seconds}s." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 10
|
||||||
|
elapsed=$((elapsed + 10))
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "Prometheus stack workloads are ready."
|
||||||
|
EOT
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
resource "kubernetes_config_map" "grafana_dashboards" {
|
resource "kubernetes_config_map" "grafana_dashboards" {
|
||||||
for_each = local.grafana_dashboard_files
|
for_each = local.grafana_dashboard_files
|
||||||
|
|
||||||
|
|
@ -1991,7 +2036,7 @@ resource "kubernetes_config_map" "grafana_dashboards" {
|
||||||
(each.key) = file("${path.module}/grafana-dashboards/${each.key}")
|
(each.key) = file("${path.module}/grafana-dashboards/${each.key}")
|
||||||
}
|
}
|
||||||
|
|
||||||
depends_on = [helm_release.prometheus_stack]
|
depends_on = [null_resource.prometheus_stack_readiness]
|
||||||
}
|
}
|
||||||
|
|
||||||
resource "helm_release" "extra_tools" {
|
resource "helm_release" "extra_tools" {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue