diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf index fa3bd14..df96930 100644 --- a/bootstrap/platform/main.tf +++ b/bootstrap/platform/main.tf @@ -1733,8 +1733,37 @@ resource "helm_release" "promtail" { ] } +resource "null_resource" "prometheus_stack_helm_recovery" { + depends_on = [helm_release.loki, helm_release.mimir] + + triggers = { + kubeconfig_path = var.kubeconfig_path + namespace = var.observability.namespace + release_name = "prometheus-stack" + release_version = var.observability.prometheus.version + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-lc"] + command = </dev/null | +while IFS=$'\t' read -r secret status; do + case "$status" in + pending-install|pending-upgrade|pending-rollback|failed|uninstalling) + kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" delete secret "$secret" + ;; + esac +done +EOT + } +} + resource "helm_release" "prometheus_stack" { - depends_on = [helm_release.loki, helm_release.mimir] + depends_on = [null_resource.prometheus_stack_helm_recovery] name = "prometheus-stack" repository = var.observability.prometheus.repository chart = var.observability.prometheus.chart @@ -1743,6 +1772,7 @@ resource "helm_release" "prometheus_stack" { create_namespace = false timeout = 1200 wait = true + cleanup_on_fail = true values = [ yamlencode({