Add Grafana dashboards for Jeannie checks
This commit is contained in:
parent
4d7836475a
commit
dd15faabb0
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./jeannie scorecard
|
||||
./jeannie gitops-status
|
||||
./jeannie cert-check
|
||||
./jeannie grafana-dashboards list
|
||||
./jeannie release-snapshot
|
||||
./jeannie backup-status
|
||||
./jeannie synthetic-checks
|
||||
|
|
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
|
|||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||
and recent repo-server/application-controller errors.
|
||||
|
||||
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
|
||||
`bootstrap/platform/grafana-dashboards`. `./jeannie up` provisions
|
||||
them through the platform stack; `./jeannie grafana-dashboards apply`
|
||||
rebuilds just the dashboard ConfigMaps and restarts Grafana.
|
||||
|
||||
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
||||
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||
|
||||
|
|
|
|||
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./{{ main_script }} scorecard
|
||||
./{{ main_script }} gitops-status
|
||||
./{{ main_script }} cert-check
|
||||
./{{ main_script }} grafana-dashboards list
|
||||
./{{ main_script }} release-snapshot
|
||||
./{{ main_script }} backup-status
|
||||
./{{ main_script }} synthetic-checks
|
||||
|
|
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
|
|||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||
and recent repo-server/application-controller errors.
|
||||
|
||||
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
|
||||
`bootstrap/platform/grafana-dashboards`. `./{{ main_script }} up` provisions
|
||||
them through the platform stack; `./{{ main_script }} grafana-dashboards apply`
|
||||
rebuilds just the dashboard ConfigMaps and restarts Grafana.
|
||||
|
||||
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
|
||||
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,295 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})",
|
||||
"legendFormat": "ready nodes",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Ready Nodes",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(kube_deployment_spec_replicas - kube_deployment_status_replicas_available)",
|
||||
"legendFormat": "missing replicas",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Missing Deployment Replicas",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(increase(kube_pod_container_status_restarts_total[15m]))",
|
||||
"legendFormat": "restarts",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Pod Restarts In 15m",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (code) (rate(traefik_service_requests_total{code=~\"4..|5..\"}[5m]))",
|
||||
"legendFormat": "{{code}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Traefik Error Rate",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up",
|
||||
"legendFormat": "{{job}} {{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Scrape Target Availability",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"homelab",
|
||||
"jeannie",
|
||||
"availability"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Homelab Availability",
|
||||
"uid": "homelab-availability",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
|
@ -0,0 +1,303 @@
|
|||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
"id": null,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Node Memory Used",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"}))",
|
||||
"legendFormat": "{{instance}} {{mountpoint}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Important Filesystems Used",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 8
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"cpu\"})",
|
||||
"legendFormat": "{{node}} cpu cores requested",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"cpu\"})",
|
||||
"legendFormat": "{{node}} cpu cores limited",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "CPU Requests And Limits By Node",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 8
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"memory\"})",
|
||||
"legendFormat": "{{node}} memory requested",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"memory\"})",
|
||||
"legendFormat": "{{node}} memory limited",
|
||||
"refId": "B"
|
||||
}
|
||||
],
|
||||
"title": "Memory Requests And Limits By Node",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"max": 100,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 16
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "table",
|
||||
"placement": "bottom"
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes)",
|
||||
"legendFormat": "{{namespace}}/{{persistentvolumeclaim}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "PVC Usage",
|
||||
"type": "timeseries"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"homelab",
|
||||
"jeannie",
|
||||
"capacity"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "Homelab Capacity",
|
||||
"uid": "homelab-capacity",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
|
|
@ -107,6 +107,7 @@ EOT
|
|||
|
||||
platform_topology_key = "kubernetes.io/hostname"
|
||||
prometheus_stack_node_selector = var.prometheus_stack_node_selector
|
||||
grafana_dashboard_files = fileset("${path.module}/grafana-dashboards", "*.json")
|
||||
argocd_node_selector = {
|
||||
"kubernetes.io/os" = "linux"
|
||||
"homelab.dev/workload-class" = "platform"
|
||||
|
|
@ -1496,6 +1497,15 @@ resource "helm_release" "prometheus_stack" {
|
|||
}
|
||||
grafana = {
|
||||
nodeSelector = local.prometheus_stack_node_selector
|
||||
sidecar = {
|
||||
dashboards = {
|
||||
enabled = true
|
||||
label = "grafana_dashboard"
|
||||
labelValue = "1"
|
||||
searchNamespace = var.observability.namespace
|
||||
folderAnnotation = "grafana_folder"
|
||||
}
|
||||
}
|
||||
persistence = {
|
||||
enabled = true
|
||||
type = "sts"
|
||||
|
|
@ -1542,6 +1552,27 @@ resource "helm_release" "prometheus_stack" {
|
|||
]
|
||||
}
|
||||
|
||||
resource "kubernetes_config_map" "grafana_dashboards" {
|
||||
for_each = local.grafana_dashboard_files
|
||||
|
||||
metadata {
|
||||
name = "homelab-grafana-${trimsuffix(each.key, ".json")}"
|
||||
namespace = var.observability.namespace
|
||||
labels = {
|
||||
grafana_dashboard = "1"
|
||||
}
|
||||
annotations = {
|
||||
grafana_folder = "Homelab"
|
||||
}
|
||||
}
|
||||
|
||||
data = {
|
||||
(each.key) = file("${path.module}/grafana-dashboards/${each.key}")
|
||||
}
|
||||
|
||||
depends_on = [helm_release.prometheus_stack]
|
||||
}
|
||||
|
||||
resource "helm_release" "extra_tools" {
|
||||
for_each = var.extra_helm_releases
|
||||
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ TSV rows and prints compact grouped summaries, detailed output, or JSON.
|
|||
or degraded apps, repository secret presence, recent events, and recent
|
||||
repo-server/application-controller errors.
|
||||
|
||||
`grafana-dashboards {list|apply}`
|
||||
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
|
||||
`bootstrap/platform/grafana-dashboards`.
|
||||
|
||||
`cert-check`
|
||||
: Verify public DNS, TLS certificate expiry, public website/Gitea HTTP status,
|
||||
and DuckDNS-to-OCI edge IP drift from the canonical inventory.
|
||||
|
|
|
|||
10
jeannie
10
jeannie
|
|
@ -6027,6 +6027,11 @@ scorecard() {
|
|||
"${REPO_ROOT}/scripts/scorecard"
|
||||
}
|
||||
|
||||
grafana_dashboards() {
|
||||
require_debian_server "grafana-dashboards"
|
||||
"${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}"
|
||||
}
|
||||
|
||||
workers_manage() {
|
||||
require_debian_server "workers"
|
||||
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
||||
|
|
@ -6110,6 +6115,9 @@ case "${1:-}" in
|
|||
scorecard)
|
||||
scorecard
|
||||
;;
|
||||
grafana-dashboards)
|
||||
grafana_dashboards "$@"
|
||||
;;
|
||||
workers)
|
||||
workers_manage "$@"
|
||||
;;
|
||||
|
|
@ -6267,7 +6275,7 @@ case "${1:-}" in
|
|||
echo "Log: ${JEANNIE_LOG_FILE}"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|grafana-dashboards {list|apply}|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@ set -euo pipefail
|
|||
|
||||
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}"
|
||||
GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1"
|
||||
|
||||
DETAIL_ARGS=()
|
||||
VERBOSE=false
|
||||
|
|
@ -175,9 +177,9 @@ compact_host_capacity() {
|
|||
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
||||
mem_pct=$((mem_used * 100 / mem_total))
|
||||
if ((mem_pct >= 85)); then
|
||||
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers."
|
||||
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
||||
else
|
||||
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB"
|
||||
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
||||
fi
|
||||
else
|
||||
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
||||
|
|
@ -195,11 +197,11 @@ compact_host_capacity() {
|
|||
use_pct="${line%% *}"
|
||||
use_pct="${use_pct%%%}"
|
||||
if ((use_pct >= 90)); then
|
||||
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services."
|
||||
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||
elif ((use_pct >= 80)); then
|
||||
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage."
|
||||
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||
else
|
||||
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }"
|
||||
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
@ -254,7 +256,7 @@ compact_kubernetes_capacity() {
|
|||
|
||||
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
||||
if [[ -n "${top_nodes}" ]]; then
|
||||
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}"
|
||||
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=3"
|
||||
else
|
||||
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
||||
fi
|
||||
|
|
@ -291,7 +293,7 @@ echo \"total_vms=\${total_vms:-0}\"
|
|||
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
||||
storage_pct="${storage_pct%%%}"
|
||||
if ((${storage_pct%.*} >= 85)); then
|
||||
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
|
||||
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Grafana: ${GRAFANA_CAPACITY_URL}"
|
||||
else
|
||||
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -0,0 +1,66 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DASHBOARD_DIR="${HOMELAB_GRAFANA_DASHBOARD_DIR:-${REPO_ROOT}/bootstrap/platform/grafana-dashboards}"
|
||||
NAMESPACE="${HOMELAB_GRAFANA_NAMESPACE:-monitoring}"
|
||||
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: ./jeannie grafana-dashboards {list|apply}
|
||||
|
||||
list Show repo-managed dashboard files.
|
||||
apply Rebuild Grafana dashboard ConfigMaps and restart Grafana.
|
||||
EOF
|
||||
}
|
||||
|
||||
list_dashboards() {
|
||||
find "$DASHBOARD_DIR" -maxdepth 1 -type f -name '*.json' -print | sort
|
||||
}
|
||||
|
||||
apply_dashboards() {
|
||||
local file
|
||||
local base
|
||||
local name
|
||||
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
echo "kubectl is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$KUBECONFIG_PATH" ]; then
|
||||
echo "kubeconfig is missing: $KUBECONFIG_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while IFS= read -r file; do
|
||||
base="$(basename "$file")"
|
||||
name="homelab-grafana-${base%.json}"
|
||||
echo "Applying Grafana dashboard ConfigMap ${NAMESPACE}/${name} from ${base}..."
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" create configmap "$name" \
|
||||
--from-file="$base=$file" \
|
||||
--dry-run=client -o yaml |
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" apply -f -
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" label configmap "$name" grafana_dashboard=1 --overwrite
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" annotate configmap "$name" grafana_folder=Homelab --overwrite
|
||||
done < <(list_dashboards)
|
||||
|
||||
echo "Restarting Grafana so the sidecar reloads dashboards..."
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" rollout restart deployment/prometheus-stack-grafana
|
||||
}
|
||||
|
||||
case "${1:-list}" in
|
||||
list)
|
||||
list_dashboards
|
||||
;;
|
||||
apply|rebuild)
|
||||
apply_dashboards
|
||||
;;
|
||||
-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Loading…
Reference in New Issue