Add Grafana dashboards for Jeannie checks
This commit is contained in:
juvdiaz
parent
4d7836475a
commit
dd15faabb0
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
./jeannie scorecard
|
./jeannie scorecard
|
||||||
./jeannie gitops-status
|
./jeannie gitops-status
|
||||||
./jeannie cert-check
|
./jeannie cert-check
|
||||||
|
./jeannie grafana-dashboards list
|
||||||
./jeannie release-snapshot
|
./jeannie release-snapshot
|
||||||
./jeannie backup-status
|
./jeannie backup-status
|
||||||
./jeannie synthetic-checks
|
./jeannie synthetic-checks
|
||||||
|
|
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
and recent repo-server/application-controller errors.
|
and recent repo-server/application-controller errors.
|
||||||
|
|
||||||
|
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
|
||||||
|
`bootstrap/platform/grafana-dashboards`. `./jeannie up` provisions
|
||||||
|
them through the platform stack; `./jeannie grafana-dashboards apply`
|
||||||
|
rebuilds just the dashboard ConfigMaps and restarts Grafana.
|
||||||
|
|
||||||
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
||||||
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
./{{ main_script }} scorecard
|
./{{ main_script }} scorecard
|
||||||
./{{ main_script }} gitops-status
|
./{{ main_script }} gitops-status
|
||||||
./{{ main_script }} cert-check
|
./{{ main_script }} cert-check
|
||||||
|
./{{ main_script }} grafana-dashboards list
|
||||||
./{{ main_script }} release-snapshot
|
./{{ main_script }} release-snapshot
|
||||||
./{{ main_script }} backup-status
|
./{{ main_script }} backup-status
|
||||||
./{{ main_script }} synthetic-checks
|
./{{ main_script }} synthetic-checks
|
||||||
|
|
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
and recent repo-server/application-controller errors.
|
and recent repo-server/application-controller errors.
|
||||||
|
|
||||||
|
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
|
||||||
|
`bootstrap/platform/grafana-dashboards`. `./{{ main_script }} up` provisions
|
||||||
|
them through the platform stack; `./{{ main_script }} grafana-dashboards apply`
|
||||||
|
rebuilds just the dashboard ConfigMaps and restarts Grafana.
|
||||||
|
|
||||||
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
||||||
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,295 @@
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "-- Grafana --"
|
||||||
|
},
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 8,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})",
|
||||||
|
"legendFormat": "ready nodes",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Ready Nodes",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 3
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 8,
|
||||||
|
"x": 8,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(kube_deployment_spec_replicas - kube_deployment_status_replicas_available)",
|
||||||
|
"legendFormat": "missing replicas",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Missing Deployment Replicas",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 1
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 5
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 8,
|
||||||
|
"x": 16,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto"
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(kube_pod_container_status_restarts_total[15m]))",
|
||||||
|
"legendFormat": "restarts",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Pod Restarts In 15m",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "reqps"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (code) (rate(traefik_service_requests_total{code=~\"4..|5..\"}[5m]))",
|
||||||
|
"legendFormat": "{{code}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Traefik Error Rate",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "up",
|
||||||
|
"legendFormat": "{{job}} {{instance}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Scrape Target Availability",
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": [
|
||||||
|
"homelab",
|
||||||
|
"jeannie",
|
||||||
|
"availability"
|
||||||
|
],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-6h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Homelab Availability",
|
||||||
|
"uid": "homelab-availability",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,303 @@
|
||||||
|
{
|
||||||
|
"annotations": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"builtIn": 1,
|
||||||
|
"datasource": {
|
||||||
|
"type": "grafana",
|
||||||
|
"uid": "-- Grafana --"
|
||||||
|
},
|
||||||
|
"enable": true,
|
||||||
|
"hide": true,
|
||||||
|
"iconColor": "rgba(0, 211, 255, 1)",
|
||||||
|
"name": "Annotations & Alerts",
|
||||||
|
"type": "dashboard"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"editable": true,
|
||||||
|
"fiscalYearStartMonth": 0,
|
||||||
|
"graphTooltip": 0,
|
||||||
|
"id": null,
|
||||||
|
"links": [],
|
||||||
|
"liveNow": false,
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"max": 100,
|
||||||
|
"min": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 90
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "percent"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 1,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||||
|
"legendFormat": "{{instance}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Node Memory Used",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"max": 100,
|
||||||
|
"min": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 90
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "percent"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 0
|
||||||
|
},
|
||||||
|
"id": 2,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 * (1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"}))",
|
||||||
|
"legendFormat": "{{instance}} {{mountpoint}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Important Filesystems Used",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 3,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"cpu\"})",
|
||||||
|
"legendFormat": "{{node}} cpu cores requested",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"cpu\"})",
|
||||||
|
"legendFormat": "{{node}} cpu cores limited",
|
||||||
|
"refId": "B"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "CPU Requests And Limits By Node",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "bytes"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 8
|
||||||
|
},
|
||||||
|
"id": 4,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"memory\"})",
|
||||||
|
"legendFormat": "{{node}} memory requested",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"memory\"})",
|
||||||
|
"legendFormat": "{{node}} memory limited",
|
||||||
|
"refId": "B"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Memory Requests And Limits By Node",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"max": 100,
|
||||||
|
"min": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "yellow",
|
||||||
|
"value": 75
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 90
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "percent"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 16
|
||||||
|
},
|
||||||
|
"id": 5,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom"
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "100 * (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes)",
|
||||||
|
"legendFormat": "{{namespace}}/{{persistentvolumeclaim}}",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "PVC Usage",
|
||||||
|
"type": "timeseries"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"refresh": "30s",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"tags": [
|
||||||
|
"homelab",
|
||||||
|
"jeannie",
|
||||||
|
"capacity"
|
||||||
|
],
|
||||||
|
"templating": {
|
||||||
|
"list": []
|
||||||
|
},
|
||||||
|
"time": {
|
||||||
|
"from": "now-6h",
|
||||||
|
"to": "now"
|
||||||
|
},
|
||||||
|
"timepicker": {},
|
||||||
|
"timezone": "browser",
|
||||||
|
"title": "Homelab Capacity",
|
||||||
|
"uid": "homelab-capacity",
|
||||||
|
"version": 1,
|
||||||
|
"weekStart": ""
|
||||||
|
}
|
||||||
|
|
@ -107,6 +107,7 @@ EOT
|
||||||
|
|
||||||
platform_topology_key = "kubernetes.io/hostname"
|
platform_topology_key = "kubernetes.io/hostname"
|
||||||
prometheus_stack_node_selector = var.prometheus_stack_node_selector
|
prometheus_stack_node_selector = var.prometheus_stack_node_selector
|
||||||
|
grafana_dashboard_files = fileset("${path.module}/grafana-dashboards", "*.json")
|
||||||
argocd_node_selector = {
|
argocd_node_selector = {
|
||||||
"kubernetes.io/os" = "linux"
|
"kubernetes.io/os" = "linux"
|
||||||
"homelab.dev/workload-class" = "platform"
|
"homelab.dev/workload-class" = "platform"
|
||||||
|
|
@ -1496,6 +1497,15 @@ resource "helm_release" "prometheus_stack" {
|
||||||
}
|
}
|
||||||
grafana = {
|
grafana = {
|
||||||
nodeSelector = local.prometheus_stack_node_selector
|
nodeSelector = local.prometheus_stack_node_selector
|
||||||
|
sidecar = {
|
||||||
|
dashboards = {
|
||||||
|
enabled = true
|
||||||
|
label = "grafana_dashboard"
|
||||||
|
labelValue = "1"
|
||||||
|
searchNamespace = var.observability.namespace
|
||||||
|
folderAnnotation = "grafana_folder"
|
||||||
|
}
|
||||||
|
}
|
||||||
persistence = {
|
persistence = {
|
||||||
enabled = true
|
enabled = true
|
||||||
type = "sts"
|
type = "sts"
|
||||||
|
|
@ -1542,6 +1552,27 @@ resource "helm_release" "prometheus_stack" {
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resource "kubernetes_config_map" "grafana_dashboards" {
|
||||||
|
for_each = local.grafana_dashboard_files
|
||||||
|
|
||||||
|
metadata {
|
||||||
|
name = "homelab-grafana-${trimsuffix(each.key, ".json")}"
|
||||||
|
namespace = var.observability.namespace
|
||||||
|
labels = {
|
||||||
|
grafana_dashboard = "1"
|
||||||
|
}
|
||||||
|
annotations = {
|
||||||
|
grafana_folder = "Homelab"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
data = {
|
||||||
|
(each.key) = file("${path.module}/grafana-dashboards/${each.key}")
|
||||||
|
}
|
||||||
|
|
||||||
|
depends_on = [helm_release.prometheus_stack]
|
||||||
|
}
|
||||||
|
|
||||||
resource "helm_release" "extra_tools" {
|
resource "helm_release" "extra_tools" {
|
||||||
for_each = var.extra_helm_releases
|
for_each = var.extra_helm_releases
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -80,6 +80,10 @@ TSV rows and prints compact grouped summaries, detailed output, or JSON.
|
||||||
or degraded apps, repository secret presence, recent events, and recent
|
or degraded apps, repository secret presence, recent events, and recent
|
||||||
repo-server/application-controller errors.
|
repo-server/application-controller errors.
|
||||||
|
|
||||||
|
`grafana-dashboards {list|apply}`
|
||||||
|
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
|
||||||
|
`bootstrap/platform/grafana-dashboards`.
|
||||||
|
|
||||||
`cert-check`
|
`cert-check`
|
||||||
: Verify public DNS, TLS certificate expiry, public website/Gitea HTTP status,
|
: Verify public DNS, TLS certificate expiry, public website/Gitea HTTP status,
|
||||||
and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||||
|
|
|
||||||
10
jeannie
10
jeannie
|
|
@ -6027,6 +6027,11 @@ scorecard() {
|
||||||
"${REPO_ROOT}/scripts/scorecard"
|
"${REPO_ROOT}/scripts/scorecard"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
grafana_dashboards() {
|
||||||
|
require_debian_server "grafana-dashboards"
|
||||||
|
"${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}"
|
||||||
|
}
|
||||||
|
|
||||||
workers_manage() {
|
workers_manage() {
|
||||||
require_debian_server "workers"
|
require_debian_server "workers"
|
||||||
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
||||||
|
|
@ -6110,6 +6115,9 @@ case "${1:-}" in
|
||||||
scorecard)
|
scorecard)
|
||||||
scorecard
|
scorecard
|
||||||
;;
|
;;
|
||||||
|
grafana-dashboards)
|
||||||
|
grafana_dashboards "$@"
|
||||||
|
;;
|
||||||
workers)
|
workers)
|
||||||
workers_manage "$@"
|
workers_manage "$@"
|
||||||
;;
|
;;
|
||||||
|
|
@ -6267,7 +6275,7 @@ case "${1:-}" in
|
||||||
echo "Log: ${JEANNIE_LOG_FILE}"
|
echo "Log: ${JEANNIE_LOG_FILE}"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|grafana-dashboards {list|apply}|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,8 @@ set -euo pipefail
|
||||||
|
|
||||||
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}"
|
||||||
|
GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1"
|
||||||
|
|
||||||
DETAIL_ARGS=()
|
DETAIL_ARGS=()
|
||||||
VERBOSE=false
|
VERBOSE=false
|
||||||
|
|
@ -175,9 +177,9 @@ compact_host_capacity() {
|
||||||
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
||||||
mem_pct=$((mem_used * 100 / mem_total))
|
mem_pct=$((mem_used * 100 / mem_total))
|
||||||
if ((mem_pct >= 85)); then
|
if ((mem_pct >= 85)); then
|
||||||
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers."
|
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
||||||
else
|
else
|
||||||
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB"
|
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
||||||
|
|
@ -195,11 +197,11 @@ compact_host_capacity() {
|
||||||
use_pct="${line%% *}"
|
use_pct="${line%% *}"
|
||||||
use_pct="${use_pct%%%}"
|
use_pct="${use_pct%%%}"
|
||||||
if ((use_pct >= 90)); then
|
if ((use_pct >= 90)); then
|
||||||
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services."
|
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||||
elif ((use_pct >= 80)); then
|
elif ((use_pct >= 80)); then
|
||||||
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage."
|
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||||
else
|
else
|
||||||
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }"
|
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
@ -254,7 +256,7 @@ compact_kubernetes_capacity() {
|
||||||
|
|
||||||
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
||||||
if [[ -n "${top_nodes}" ]]; then
|
if [[ -n "${top_nodes}" ]]; then
|
||||||
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}"
|
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=3"
|
||||||
else
|
else
|
||||||
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
||||||
fi
|
fi
|
||||||
|
|
@ -291,7 +293,7 @@ echo \"total_vms=\${total_vms:-0}\"
|
||||||
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
||||||
storage_pct="${storage_pct%%%}"
|
storage_pct="${storage_pct%%%}"
|
||||||
if ((${storage_pct%.*} >= 85)); then
|
if ((${storage_pct%.*} >= 85)); then
|
||||||
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
|
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Grafana: ${GRAFANA_CAPACITY_URL}"
|
||||||
else
|
else
|
||||||
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
DASHBOARD_DIR="${HOMELAB_GRAFANA_DASHBOARD_DIR:-${REPO_ROOT}/bootstrap/platform/grafana-dashboards}"
|
||||||
|
NAMESPACE="${HOMELAB_GRAFANA_NAMESPACE:-monitoring}"
|
||||||
|
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<EOF
|
||||||
|
Usage: ./jeannie grafana-dashboards {list|apply}
|
||||||
|
|
||||||
|
list Show repo-managed dashboard files.
|
||||||
|
apply Rebuild Grafana dashboard ConfigMaps and restart Grafana.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
list_dashboards() {
|
||||||
|
find "$DASHBOARD_DIR" -maxdepth 1 -type f -name '*.json' -print | sort
|
||||||
|
}
|
||||||
|
|
||||||
|
apply_dashboards() {
|
||||||
|
local file
|
||||||
|
local base
|
||||||
|
local name
|
||||||
|
|
||||||
|
if ! command -v kubectl >/dev/null 2>&1; then
|
||||||
|
echo "kubectl is required." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ ! -s "$KUBECONFIG_PATH" ]; then
|
||||||
|
echo "kubeconfig is missing: $KUBECONFIG_PATH" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS= read -r file; do
|
||||||
|
base="$(basename "$file")"
|
||||||
|
name="homelab-grafana-${base%.json}"
|
||||||
|
echo "Applying Grafana dashboard ConfigMap ${NAMESPACE}/${name} from ${base}..."
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" create configmap "$name" \
|
||||||
|
--from-file="$base=$file" \
|
||||||
|
--dry-run=client -o yaml |
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_PATH" apply -f -
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" label configmap "$name" grafana_dashboard=1 --overwrite
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" annotate configmap "$name" grafana_folder=Homelab --overwrite
|
||||||
|
done < <(list_dashboards)
|
||||||
|
|
||||||
|
echo "Restarting Grafana so the sidecar reloads dashboards..."
|
||||||
|
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" rollout restart deployment/prometheus-stack-grafana
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-list}" in
|
||||||
|
list)
|
||||||
|
list_dashboards
|
||||||
|
;;
|
||||||
|
apply|rebuild)
|
||||||
|
apply_dashboards
|
||||||
|
;;
|
||||||
|
-h|--help|help)
|
||||||
|
usage
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
usage >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
Loading…
Reference in New Issue