Fixes stale helm

bootstrap/platform/main.tf

@@ -1733,8 +1733,37 @@ resource "helm_release" "promtail" {
   ]
 }
 
+resource "null_resource" "prometheus_stack_helm_recovery" {
+  depends_on = [helm_release.loki, helm_release.mimir]
+
+  triggers = {
+    kubeconfig_path = var.kubeconfig_path
+    namespace       = var.observability.namespace
+    release_name    = "prometheus-stack"
+    release_version = var.observability.prometheus.version
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-lc"]
+    command     = <<EOT
+set -euo pipefail
+
+kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" get secrets \
+  -l "owner=helm,name=${self.triggers.release_name}" \
+  -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.metadata.labels.status}{"\n"}{end}' 2>/dev/null |
+while IFS=$'\t' read -r secret status; do
+  case "$status" in
+    pending-install|pending-upgrade|pending-rollback|failed|uninstalling)
+      kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" delete secret "$secret"
+      ;;
+  esac
+done
+EOT
+  }
+}
+
 resource "helm_release" "prometheus_stack" {
-  depends_on       = [helm_release.loki, helm_release.mimir]
+  depends_on       = [null_resource.prometheus_stack_helm_recovery]
   name             = "prometheus-stack"
   repository       = var.observability.prometheus.repository
   chart            = var.observability.prometheus.chart
@@ -1743,6 +1772,7 @@ resource "helm_release" "prometheus_stack" {
   create_namespace = false
   timeout          = 1200
   wait             = true
+  cleanup_on_fail  = true
 
   values = [
     yamlencode({