Add high-signal homelab alerts

README.md

@@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
 such as deployment readiness, pod restarts, node pressure, disk pressure,
 Traefik 5xx/404 log evidence, and recent Gitea errors.
 
+High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
+readiness, pod restart storms, unavailable deployments, node/PVC storage
+pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape
+coverage. Keep this set small so alerts remain actionable.
+
 Focused doctor commands run narrower read-only checks and print the most likely
 next step:
 

README.md.tmpl

@@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
 such as deployment readiness, pod restarts, node pressure, disk pressure,
 Traefik 5xx/404 log evidence, and recent Gitea errors.
 
+High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
+readiness, pod restart storms, unavailable deployments, node/PVC storage
+pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape
+coverage. Keep this set small so alerts remain actionable.
+
 Focused doctor commands run narrower read-only checks and print the most likely
 next step:
 

apps/homelab-alerts/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - prometheus-rules.yaml

apps/homelab-alerts/prometheus-rules.yaml

@@ -0,0 +1,90 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: homelab-high-signal-alerts
+  namespace: monitoring
+  labels:
+    release: prometheus-stack
+    role: alert-rules
+spec:
+  groups:
+    - name: homelab.availability
+      rules:
+        - alert: HomelabKubernetesNodeNotReady
+          expr: kube_node_status_condition{condition="Ready",status="true"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Kubernetes node {{ $labels.node }} is not Ready"
+            description: "A node has been NotReady for 10 minutes. Run ./jeannie doctor-cluster."
+        - alert: HomelabPodRestartStorm
+          expr: increase(kube_pod_container_status_restarts_total[15m]) >= 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Pod restart storm in {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "A container restarted at least 3 times in 15 minutes. Check pod logs and recent deployments."
+        - alert: HomelabDeploymentUnavailable
+          expr: kube_deployment_status_replicas_available < kube_deployment_spec_replicas
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} is unavailable"
+            description: "Available replicas are below desired replicas for 10 minutes."
+    - name: homelab.storage
+      rules:
+        - alert: HomelabNodeDiskPressure
+          expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Node {{ $labels.node }} reports DiskPressure"
+            description: "DiskPressure is active. Run ./jeannie capacity and inspect image/PVC/log usage."
+        - alert: HomelabPersistentVolumeAlmostFull
+          expr: (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full"
+            description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage."
+    - name: homelab.edge
+      rules:
+        - alert: HomelabTraefik5xxSpike
+          expr: sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) > 0.05
+          for: 10m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Traefik is returning elevated 5xx responses"
+            description: "Public edge or backend service may be unhealthy. Run ./jeannie doctor-edge."
+        - alert: HomelabWebsiteDown
+          expr: absent(kube_deployment_status_replicas_available{namespace="website-production",deployment="php-website-deployment"} > 0)
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Website deployment has no available replicas"
+            description: "The public website has no available Kubernetes replicas."
+        - alert: HomelabGiteaRouteDown
+          expr: absent(up{job=~".*gitea.*"} == 1)
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "No Gitea scrape target is healthy"
+            description: "If Gitea is intentionally not scraped, keep Uptime Kuma as the source of truth; otherwise check ./jeannie doctor-gitea."
+    - name: homelab.dns
+      rules:
+        - alert: HomelabPiHoleDNSDown
+          expr: absent(up{job=~".*pihole.*"} == 1)
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "No Pi-hole scrape target is healthy"
+            description: "If Pi-hole is intentionally not scraped, Uptime Kuma still monitors DNS. Run ./jeannie doctor-rpi."

bootstrap/apps/variables.tf

@@ -84,6 +84,15 @@ variable "applications" {
       self_heal        = true
       create_namespace = false
     }
+    homelab-alerts = {
+      project          = "default"
+      path             = "apps/homelab-alerts"
+      namespace        = "monitoring"
+      target_revision  = "main"
+      prune            = true
+      self_heal        = true
+      create_namespace = false
+    }
     security-lab = {
       project          = "default"
       path             = "apps/security-lab"

docs/jeannie.1.md

@@ -52,6 +52,11 @@ the older detailed tables. The cascade includes "what broke" signals for recent
 deployment readiness, pod restarts, node pressure, disk pressure, Traefik
 5xx/404 log evidence, and recent Gitea errors.
 
+Prometheus alerts
+: High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app.
+They cover node readiness, restart storms, unavailable deployments, storage
+pressure, Traefik 5xx, and core service visibility.
+
 `validate`
 : Run non-mutating repository checks used by the GitOps promotion flow:
 inventory, generated docs, Bash syntax, shellcheck, YAML parsing, OpenTofu