Add Grafana dashboards for Jeannie checks

This commit is contained in:
juvdiaz 2026-06-29 18:32:46 -06:00
parent 4d7836475a
commit dd15faabb0
9 changed files with 729 additions and 8 deletions

View File

@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
./jeannie scorecard
./jeannie gitops-status
./jeannie cert-check
./jeannie grafana-dashboards list
./jeannie release-snapshot
./jeannie backup-status
./jeannie synthetic-checks
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors.
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
`bootstrap/platform/grafana-dashboards`. `./jeannie up` provisions
them through the platform stack; `./jeannie grafana-dashboards apply`
rebuilds just the dashboard ConfigMaps and restarts Grafana.
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.

View File

@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
./{{ main_script }} scorecard
./{{ main_script }} gitops-status
./{{ main_script }} cert-check
./{{ main_script }} grafana-dashboards list
./{{ main_script }} release-snapshot
./{{ main_script }} backup-status
./{{ main_script }} synthetic-checks
@ -331,6 +332,11 @@ summaries by default, with details or JSON available for future TUI/web output.
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
and recent repo-server/application-controller errors.
`grafana-dashboards` manages repo-provisioned Grafana dashboards under
`bootstrap/platform/grafana-dashboards`. `./{{ main_script }} up` provisions
them through the platform stack; `./{{ main_script }} grafana-dashboards apply`
rebuilds just the dashboard ConfigMaps and restarts Grafana.
`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea
HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory.

View File

@ -0,0 +1,295 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\"})",
"legendFormat": "ready nodes",
"refId": "A"
}
],
"title": "Ready Nodes",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 3
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "sum(kube_deployment_spec_replicas - kube_deployment_status_replicas_available)",
"legendFormat": "missing replicas",
"refId": "A"
}
],
"title": "Missing Deployment Replicas",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "sum(increase(kube_pod_container_status_restarts_total[15m]))",
"legendFormat": "restarts",
"refId": "A"
}
],
"title": "Pod Restarts In 15m",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 4,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (code) (rate(traefik_service_requests_total{code=~\"4..|5..\"}[5m]))",
"legendFormat": "{{code}}",
"refId": "A"
}
],
"title": "Traefik Error Rate",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 5,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "up",
"legendFormat": "{{job}} {{instance}}",
"refId": "A"
}
],
"title": "Scrape Target Availability",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"homelab",
"jeannie",
"availability"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Homelab Availability",
"uid": "homelab-availability",
"version": 1,
"weekStart": ""
}

View File

@ -0,0 +1,303 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"title": "Node Memory Used",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 0
},
"id": 2,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * (1 - (node_filesystem_avail_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"} / node_filesystem_size_bytes{fstype!~\"tmpfs|overlay|squashfs\",mountpoint=~\"/|/data|/var/lib/docker|/nvme-storage\"}))",
"legendFormat": "{{instance}} {{mountpoint}}",
"refId": "A"
}
],
"title": "Important Filesystems Used",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 3,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"cpu\"})",
"legendFormat": "{{node}} cpu cores requested",
"refId": "A"
},
{
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"cpu\"})",
"legendFormat": "{{node}} cpu cores limited",
"refId": "B"
}
],
"title": "CPU Requests And Limits By Node",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 8
},
"id": 4,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (node) (kube_pod_container_resource_requests{resource=\"memory\"})",
"legendFormat": "{{node}} memory requested",
"refId": "A"
},
{
"expr": "sum by (node) (kube_pod_container_resource_limits{resource=\"memory\"})",
"legendFormat": "{{node}} memory limited",
"refId": "B"
}
],
"title": "Memory Requests And Limits By Node",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 16
},
"id": 5,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * (1 - kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes)",
"legendFormat": "{{namespace}}/{{persistentvolumeclaim}}",
"refId": "A"
}
],
"title": "PVC Usage",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [
"homelab",
"jeannie",
"capacity"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Homelab Capacity",
"uid": "homelab-capacity",
"version": 1,
"weekStart": ""
}

View File

@ -107,6 +107,7 @@ EOT
platform_topology_key = "kubernetes.io/hostname"
prometheus_stack_node_selector = var.prometheus_stack_node_selector
grafana_dashboard_files = fileset("${path.module}/grafana-dashboards", "*.json")
argocd_node_selector = {
"kubernetes.io/os" = "linux"
"homelab.dev/workload-class" = "platform"
@ -1496,6 +1497,15 @@ resource "helm_release" "prometheus_stack" {
}
grafana = {
nodeSelector = local.prometheus_stack_node_selector
sidecar = {
dashboards = {
enabled = true
label = "grafana_dashboard"
labelValue = "1"
searchNamespace = var.observability.namespace
folderAnnotation = "grafana_folder"
}
}
persistence = {
enabled = true
type = "sts"
@ -1542,6 +1552,27 @@ resource "helm_release" "prometheus_stack" {
]
}
resource "kubernetes_config_map" "grafana_dashboards" {
for_each = local.grafana_dashboard_files
metadata {
name = "homelab-grafana-${trimsuffix(each.key, ".json")}"
namespace = var.observability.namespace
labels = {
grafana_dashboard = "1"
}
annotations = {
grafana_folder = "Homelab"
}
}
data = {
(each.key) = file("${path.module}/grafana-dashboards/${each.key}")
}
depends_on = [helm_release.prometheus_stack]
}
resource "helm_release" "extra_tools" {
for_each = var.extra_helm_releases

View File

@ -80,6 +80,10 @@ TSV rows and prints compact grouped summaries, detailed output, or JSON.
or degraded apps, repository secret presence, recent events, and recent
repo-server/application-controller errors.
`grafana-dashboards {list|apply}`
: List or rebuild repo-provisioned Grafana dashboard ConfigMaps from
`bootstrap/platform/grafana-dashboards`.
`cert-check`
: Verify public DNS, TLS certificate expiry, public website/Gitea HTTP status,
and DuckDNS-to-OCI edge IP drift from the canonical inventory.

10
jeannie
View File

@ -6027,6 +6027,11 @@ scorecard() {
"${REPO_ROOT}/scripts/scorecard"
}
grafana_dashboards() {
require_debian_server "grafana-dashboards"
"${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}"
}
workers_manage() {
require_debian_server "workers"
"${REPO_ROOT}/scripts/workers" "${@:2}"
@ -6110,6 +6115,9 @@ case "${1:-}" in
scorecard)
scorecard
;;
grafana-dashboards)
grafana_dashboards "$@"
;;
workers)
workers_manage "$@"
;;
@ -6267,7 +6275,7 @@ case "${1:-}" in
echo "Log: ${JEANNIE_LOG_FILE}"
;;
*)
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|recover-power [--dry-run]|scorecard|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|artifact-cache {status|up|down|instructions}|golden-ledger {show|check}|route-inventory|grafana-dashboards {list|apply}|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
exit 1
;;
esac

View File

@ -3,6 +3,8 @@ set -euo pipefail
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}"
GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1"
DETAIL_ARGS=()
VERBOSE=false
@ -175,9 +177,9 @@ compact_host_capacity() {
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
mem_pct=$((mem_used * 100 / mem_total))
if ((mem_pct >= 85)); then
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers."
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
else
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB"
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
fi
else
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
@ -195,11 +197,11 @@ compact_host_capacity() {
use_pct="${line%% *}"
use_pct="${use_pct%%%}"
if ((use_pct >= 90)); then
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services."
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
elif ((use_pct >= 80)); then
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage."
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
else
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }"
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
fi
done
@ -254,7 +256,7 @@ compact_kubernetes_capacity() {
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
if [[ -n "${top_nodes}" ]]; then
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}"
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=3"
else
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
fi
@ -291,7 +293,7 @@ echo \"total_vms=\${total_vms:-0}\"
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
storage_pct="${storage_pct%%%}"
if ((${storage_pct%.*} >= 85)); then
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Grafana: ${GRAFANA_CAPACITY_URL}"
else
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
fi

66
scripts/grafana-dashboards Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DASHBOARD_DIR="${HOMELAB_GRAFANA_DASHBOARD_DIR:-${REPO_ROOT}/bootstrap/platform/grafana-dashboards}"
NAMESPACE="${HOMELAB_GRAFANA_NAMESPACE:-monitoring}"
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-${HOME}/.kube/config}}"
usage() {
cat <<EOF
Usage: ./jeannie grafana-dashboards {list|apply}
list Show repo-managed dashboard files.
apply Rebuild Grafana dashboard ConfigMaps and restart Grafana.
EOF
}
list_dashboards() {
find "$DASHBOARD_DIR" -maxdepth 1 -type f -name '*.json' -print | sort
}
apply_dashboards() {
local file
local base
local name
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl is required." >&2
exit 1
fi
if [ ! -s "$KUBECONFIG_PATH" ]; then
echo "kubeconfig is missing: $KUBECONFIG_PATH" >&2
exit 1
fi
while IFS= read -r file; do
base="$(basename "$file")"
name="homelab-grafana-${base%.json}"
echo "Applying Grafana dashboard ConfigMap ${NAMESPACE}/${name} from ${base}..."
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" create configmap "$name" \
--from-file="$base=$file" \
--dry-run=client -o yaml |
kubectl --kubeconfig "$KUBECONFIG_PATH" apply -f -
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" label configmap "$name" grafana_dashboard=1 --overwrite
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" annotate configmap "$name" grafana_folder=Homelab --overwrite
done < <(list_dashboards)
echo "Restarting Grafana so the sidecar reloads dashboards..."
kubectl --kubeconfig "$KUBECONFIG_PATH" -n "$NAMESPACE" rollout restart deployment/prometheus-stack-grafana
}
case "${1:-list}" in
list)
list_dashboards
;;
apply|rebuild)
apply_dashboards
;;
-h|--help|help)
usage
;;
*)
usage >&2
exit 1
;;
esac