Add high-signal homelab alerts
This commit is contained in:
juvdiaz
parent
d2c959bebd
commit
bb5b618a22
|
|
@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
|
|||
such as deployment readiness, pod restarts, node pressure, disk pressure,
|
||||
Traefik 5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
|
||||
readiness, pod restart storms, unavailable deployments, node/PVC storage
|
||||
pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape
|
||||
coverage. Keep this set small so alerts remain actionable.
|
||||
|
||||
Focused doctor commands run narrower read-only checks and print the most likely
|
||||
next step:
|
||||
|
||||
|
|
|
|||
|
|
@ -278,6 +278,11 @@ services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
|
|||
such as deployment readiness, pod restarts, node pressure, disk pressure,
|
||||
Traefik 5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
|
||||
readiness, pod restart storms, unavailable deployments, node/PVC storage
|
||||
pressure, Traefik 5xx spikes, website availability, and Gitea/Pi-hole scrape
|
||||
coverage. Keep this set small so alerts remain actionable.
|
||||
|
||||
Focused doctor commands run narrower read-only checks and print the most likely
|
||||
next step:
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- prometheus-rules.yaml
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: homelab-high-signal-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: prometheus-stack
|
||||
role: alert-rules
|
||||
spec:
|
||||
groups:
|
||||
- name: homelab.availability
|
||||
rules:
|
||||
- alert: HomelabKubernetesNodeNotReady
|
||||
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Kubernetes node {{ $labels.node }} is not Ready"
|
||||
description: "A node has been NotReady for 10 minutes. Run ./jeannie doctor-cluster."
|
||||
- alert: HomelabPodRestartStorm
|
||||
expr: increase(kube_pod_container_status_restarts_total[15m]) >= 3
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Pod restart storm in {{ $labels.namespace }}/{{ $labels.pod }}"
|
||||
description: "A container restarted at least 3 times in 15 minutes. Check pod logs and recent deployments."
|
||||
- alert: HomelabDeploymentUnavailable
|
||||
expr: kube_deployment_status_replicas_available < kube_deployment_spec_replicas
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} is unavailable"
|
||||
description: "Available replicas are below desired replicas for 10 minutes."
|
||||
- name: homelab.storage
|
||||
rules:
|
||||
- alert: HomelabNodeDiskPressure
|
||||
expr: kube_node_status_condition{condition="DiskPressure",status="true"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Node {{ $labels.node }} reports DiskPressure"
|
||||
description: "DiskPressure is active. Run ./jeannie capacity and inspect image/PVC/log usage."
|
||||
- alert: HomelabPersistentVolumeAlmostFull
|
||||
expr: (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) * 100 > 85
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full"
|
||||
description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage."
|
||||
- name: homelab.edge
|
||||
rules:
|
||||
- alert: HomelabTraefik5xxSpike
|
||||
expr: sum(rate(traefik_service_requests_total{code=~"5.."}[5m])) > 0.05
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Traefik is returning elevated 5xx responses"
|
||||
description: "Public edge or backend service may be unhealthy. Run ./jeannie doctor-edge."
|
||||
- alert: HomelabWebsiteDown
|
||||
expr: absent(kube_deployment_status_replicas_available{namespace="website-production",deployment="php-website-deployment"} > 0)
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Website deployment has no available replicas"
|
||||
description: "The public website has no available Kubernetes replicas."
|
||||
- alert: HomelabGiteaRouteDown
|
||||
expr: absent(up{job=~".*gitea.*"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No Gitea scrape target is healthy"
|
||||
description: "If Gitea is intentionally not scraped, keep Uptime Kuma as the source of truth; otherwise check ./jeannie doctor-gitea."
|
||||
- name: homelab.dns
|
||||
rules:
|
||||
- alert: HomelabPiHoleDNSDown
|
||||
expr: absent(up{job=~".*pihole.*"} == 1)
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "No Pi-hole scrape target is healthy"
|
||||
description: "If Pi-hole is intentionally not scraped, Uptime Kuma still monitors DNS. Run ./jeannie doctor-rpi."
|
||||
|
|
@ -84,6 +84,15 @@ variable "applications" {
|
|||
self_heal = true
|
||||
create_namespace = false
|
||||
}
|
||||
homelab-alerts = {
|
||||
project = "default"
|
||||
path = "apps/homelab-alerts"
|
||||
namespace = "monitoring"
|
||||
target_revision = "main"
|
||||
prune = true
|
||||
self_heal = true
|
||||
create_namespace = false
|
||||
}
|
||||
security-lab = {
|
||||
project = "default"
|
||||
path = "apps/security-lab"
|
||||
|
|
|
|||
|
|
@ -52,6 +52,11 @@ the older detailed tables. The cascade includes "what broke" signals for recent
|
|||
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||
5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
Prometheus alerts
|
||||
: High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app.
|
||||
They cover node readiness, restart storms, unavailable deployments, storage
|
||||
pressure, Traefik 5xx, and core service visibility.
|
||||
|
||||
`validate`
|
||||
: Run non-mutating repository checks used by the GitOps promotion flow:
|
||||
inventory, generated docs, Bash syntax, shellcheck, YAML parsing, OpenTofu
|
||||
|
|
|
|||
Loading…
Reference in New Issue