diff --git a/README.md b/README.md index 5d1c1cf..857cfc3 100644 --- a/README.md +++ b/README.md @@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals such as deployment readiness, pod restarts, node pressure, disk pressure, Traefik 5xx/404 log evidence, and recent Gitea errors. +High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node +readiness, pod restart storms, unavailable deployments, node/PVC storage +pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape +coverage. Keep this set small so alerts remain actionable. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/README.md.tmpl b/README.md.tmpl index 5e84f92..8fccc33 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals such as deployment readiness, pod restarts, node pressure, disk pressure, Traefik 5xx/404 log evidence, and recent Gitea errors. +High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node +readiness, pod restart storms, unavailable deployments, node/PVC storage +pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape +coverage. Keep this set small so alerts remain actionable. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/apps/homelab-alerts/kustomization.yaml b/apps/homelab-alerts/kustomization.yaml new file mode 100644 index 0000000..6cda811 --- /dev/null +++ b/apps/homelab-alerts/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - prometheus-rules.yaml diff --git a/apps/homelab-alerts/prometheus-rules.yaml b/apps/homelab-alerts/prometheus-rules.yaml new file mode 100644 index 0000000..807ab14 --- /dev/null +++ b/apps/homelab-alerts/prometheus-rules.yaml @@ -0,0 +1,90 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: homelab-high-signal-alerts + namespace: monitoring + labels: + release: prometheus-stack + role: alert-rules +spec: + groups: + - name: homelab.availability + rules: + - alert: HomelabKubernetesNodeNotReady + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Kubernetes node {{ $labels.node }} is not Ready" + description: "A node has been NotReady for 10 minutes. Run ./jeannie doctor-cluster." + - alert: HomelabPodRestartStorm + expr: increase(kube_pod_container_status_restarts_total[15m]) >= 3 + for: 5m + labels: + severity: warning + annotations: + summary: "Pod restart storm in {{ $labels.namespace }}/{{ $labels.pod }}" + description: "A container restarted at least 3 times in 15 minutes. Check pod logs and recent deployments." + - alert: HomelabDeploymentUnavailable + expr: kube_deployment_status_replicas_available < kube_deployment_spec_replicas + for: 10m + labels: + severity: warning + annotations: + summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} is unavailable" + description: "Available replicas are below desired replicas for 10 minutes." + - name: homelab.storage + rules: + - alert: HomelabNodeDiskPressure + expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Node {{ $labels.node }} reports DiskPressure" + description: "DiskPressure is active. Run ./jeannie capacity and inspect image/PVC/log usage." + - alert: HomelabPersistentVolumeAlmostFull + expr: (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85 + for: 15m + labels: + severity: warning + annotations: + summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full" + description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage." + - name: homelab.edge + rules: + - alert: HomelabTraefik5xxSpike + expr: sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "Traefik is returning elevated 5xx responses" + description: "Public edge or backend service may be unhealthy. Run ./jeannie doctor-edge." + - alert: HomelabWebsiteDown + expr: absent(kube_deployment_status_replicas_available{namespace="website-production",deployment="php-website-deployment"} > 0) + for: 5m + labels: + severity: critical + annotations: + summary: "Website deployment has no available replicas" + description: "The public website has no available Kubernetes replicas." + - alert: HomelabGiteaRouteDown + expr: absent(up{job=~".*gitea.*"} == 1) + for: 15m + labels: + severity: warning + annotations: + summary: "No Gitea scrape target is healthy" + description: "If Gitea is intentionally not scraped, keep Uptime Kuma as the source of truth; otherwise check ./jeannie doctor-gitea." + - name: homelab.dns + rules: + - alert: HomelabPiHoleDNSDown + expr: absent(up{job=~".*pihole.*"} == 1) + for: 15m + labels: + severity: warning + annotations: + summary: "No Pi-hole scrape target is healthy" + description: "If Pi-hole is intentionally not scraped, Uptime Kuma still monitors DNS. Run ./jeannie doctor-rpi." diff --git a/bootstrap/apps/variables.tf b/bootstrap/apps/variables.tf index 4029f0d..ac52cad 100644 --- a/bootstrap/apps/variables.tf +++ b/bootstrap/apps/variables.tf @@ -84,6 +84,15 @@ variable "applications" { self_heal = true create_namespace = false } + homelab-alerts = { + project = "default" + path = "apps/homelab-alerts" + namespace = "monitoring" + target_revision = "main" + prune = true + self_heal = true + create_namespace = false + } security-lab = { project = "default" path = "apps/security-lab" diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index cda1391..9569c95 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -52,6 +52,11 @@ the older detailed tables. The cascade includes "what broke" signals for recent deployment readiness, pod restarts, node pressure, disk pressure, Traefik 5xx/404 log evidence, and recent Gitea errors. +Prometheus alerts +: High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app. +They cover node readiness, restart storms, unavailable deployments, storage +pressure, Traefik 5xx, and core service visibility. + `validate` : Run non-mutating repository checks used by the GitOps promotion flow: inventory, generated docs, Bash syntax, shellcheck, YAML parsing, OpenTofu