91 lines
3.9 KiB
YAML
91 lines
3.9 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: homelab-high-signal-alerts
|
|
namespace: monitoring
|
|
labels:
|
|
release: prometheus-stack
|
|
role: alert-rules
|
|
spec:
|
|
groups:
|
|
- name: homelab.availability
|
|
rules:
|
|
- alert: HomelabKubernetesNodeNotReady
|
|
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Kubernetes node {{ $labels.node }} is not Ready"
|
|
description: "A node has been NotReady for 10 minutes. Run ./jeannie doctor-cluster."
|
|
- alert: HomelabPodRestartStorm
|
|
expr: increase(kube_pod_container_status_restarts_total[15m]) >= 3
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Pod restart storm in {{ $labels.namespace }}/{{ $labels.pod }}"
|
|
description: "A container restarted at least 3 times in 15 minutes. Check pod logs and recent deployments."
|
|
- alert: HomelabDeploymentUnavailable
|
|
expr: kube_deployment_status_replicas_available < kube_deployment_spec_replicas
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} is unavailable"
|
|
description: "Available replicas are below desired replicas for 10 minutes."
|
|
- name: homelab.storage
|
|
rules:
|
|
- alert: HomelabNodeDiskPressure
|
|
expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Node {{ $labels.node }} reports DiskPressure"
|
|
description: "DiskPressure is active. Run ./jeannie capacity and inspect image/PVC/log usage."
|
|
- alert: HomelabPersistentVolumeAlmostFull
|
|
expr: (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full"
|
|
description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage."
|
|
- name: homelab.edge
|
|
rules:
|
|
- alert: HomelabTraefik5xxSpike
|
|
expr: sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) > 0.05
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Traefik is returning elevated 5xx responses"
|
|
description: "Public edge or backend service may be unhealthy. Run ./jeannie doctor-edge."
|
|
- alert: HomelabWebsiteDown
|
|
expr: absent(kube_deployment_status_replicas_available{namespace="website-production",deployment="php-website-deployment"} > 0)
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Website deployment has no available replicas"
|
|
description: "The public website has no available Kubernetes replicas."
|
|
- alert: HomelabGiteaRouteDown
|
|
expr: absent(up{job=~".*gitea.*"} == 1)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "No Gitea scrape target is healthy"
|
|
description: "If Gitea is intentionally not scraped, keep Uptime Kuma as the source of truth; otherwise check ./jeannie doctor-gitea."
|
|
- name: homelab.dns
|
|
rules:
|
|
- alert: HomelabPiHoleDNSDown
|
|
expr: absent(up{job=~".*pihole.*"} == 1)
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "No Pi-hole scrape target is healthy"
|
|
description: "If Pi-hole is intentionally not scraped, Uptime Kuma still monitors DNS. Run ./jeannie doctor-rpi."
|