Add Grafana graph and query links to reports
This commit is contained in:
parent
cc98e2f639
commit
35bf3b956c
|
|
@ -327,6 +327,9 @@ inventory, and docs freshness into a compact pass/warn/fail report.
|
||||||
Report-oriented commands can share the reusable renderer in
|
Report-oriented commands can share the reusable renderer in
|
||||||
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
||||||
summaries by default, with details or JSON available for future TUI/web output.
|
summaries by default, with details or JSON available for future TUI/web output.
|
||||||
|
Rows may include separate `fix`, `graph`, and `query` fields so a warning can
|
||||||
|
point to both the next `jeannie` command and the matching Grafana panel or
|
||||||
|
Explore query.
|
||||||
|
|
||||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
|
|
|
||||||
|
|
@ -327,6 +327,9 @@ inventory, and docs freshness into a compact pass/warn/fail report.
|
||||||
Report-oriented commands can share the reusable renderer in
|
Report-oriented commands can share the reusable renderer in
|
||||||
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
||||||
summaries by default, with details or JSON available for future TUI/web output.
|
summaries by default, with details or JSON available for future TUI/web output.
|
||||||
|
Rows may include separate `fix`, `graph`, and `query` fields so a warning can
|
||||||
|
point to both the next `jeannie` command and the matching Grafana panel or
|
||||||
|
Explore query.
|
||||||
|
|
||||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||||
|
|
|
||||||
|
|
@ -73,7 +73,9 @@ docs freshness into a compact pass/warn/fail report.
|
||||||
|
|
||||||
Report renderer
|
Report renderer
|
||||||
: Report-oriented commands can share `scripts/report-render`, which reads status
|
: Report-oriented commands can share `scripts/report-render`, which reads status
|
||||||
TSV rows and prints compact grouped summaries, detailed output, or JSON.
|
TSV rows and prints compact grouped summaries, detailed output, or JSON. Rows
|
||||||
|
may include separate `fix`, `graph`, and `query` fields for Grafana-backed
|
||||||
|
checks.
|
||||||
|
|
||||||
`gitops-status`
|
`gitops-status`
|
||||||
: Print a focused Argo CD status view with application sync/health, out-of-sync
|
: Print a focused Argo CD status view with application sync/health, out-of-sync
|
||||||
|
|
|
||||||
|
|
@ -5,6 +5,7 @@ KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}"
|
GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}"
|
||||||
GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1"
|
GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1"
|
||||||
|
GRAFANA_AVAILABILITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-availability/homelab-availability?orgId=1"
|
||||||
|
|
||||||
DETAIL_ARGS=()
|
DETAIL_ARGS=()
|
||||||
VERBOSE=false
|
VERBOSE=false
|
||||||
|
|
@ -45,9 +46,37 @@ emit() {
|
||||||
local check="$3"
|
local check="$3"
|
||||||
local summary="$4"
|
local summary="$4"
|
||||||
local detail="${5:-}"
|
local detail="${5:-}"
|
||||||
local hint="${6:-}"
|
local fix="${6:-}"
|
||||||
|
local graph="${7:-}"
|
||||||
|
local query="${8:-}"
|
||||||
|
|
||||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${hint}"
|
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${fix}" "${graph}" "${query}"
|
||||||
|
}
|
||||||
|
|
||||||
|
grafana_explore_url() {
|
||||||
|
local expr="$1"
|
||||||
|
python3 - "${GRAFANA_BASE_URL%/}" "$expr" <<'PY'
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
|
||||||
|
base_url, expr = sys.argv[1:3]
|
||||||
|
left = {
|
||||||
|
"datasource": "prometheus",
|
||||||
|
"queries": [
|
||||||
|
{
|
||||||
|
"refId": "A",
|
||||||
|
"expr": expr,
|
||||||
|
"range": True,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"range": {
|
||||||
|
"from": "now-6h",
|
||||||
|
"to": "now",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
print(f"{base_url}/explore?left={urllib.parse.quote(json.dumps(left, separators=(',', ':')))}")
|
||||||
|
PY
|
||||||
}
|
}
|
||||||
|
|
||||||
section() {
|
section() {
|
||||||
|
|
@ -177,9 +206,9 @@ compact_host_capacity() {
|
||||||
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
||||||
mem_pct=$((mem_used * 100 / mem_total))
|
mem_pct=$((mem_used * 100 / mem_total))
|
||||||
if ((mem_pct >= 85)); then
|
if ((mem_pct >= 85)); then
|
||||||
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers." "${GRAFANA_CAPACITY_URL}&viewPanel=1" "$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')"
|
||||||
else
|
else
|
||||||
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1"
|
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')"
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
||||||
|
|
@ -197,11 +226,11 @@ compact_host_capacity() {
|
||||||
use_pct="${line%% *}"
|
use_pct="${line%% *}"
|
||||||
use_pct="${use_pct%%%}"
|
use_pct="${use_pct%%%}"
|
||||||
if ((use_pct >= 90)); then
|
if ((use_pct >= 90)); then
|
||||||
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')"
|
||||||
elif ((use_pct >= 80)); then
|
elif ((use_pct >= 80)); then
|
||||||
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')"
|
||||||
else
|
else
|
||||||
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2"
|
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "" "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')"
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
@ -222,6 +251,15 @@ compact_kubernetes_capacity() {
|
||||||
local problem_pods
|
local problem_pods
|
||||||
local missing_resources
|
local missing_resources
|
||||||
local top_nodes
|
local top_nodes
|
||||||
|
local node_ready_query
|
||||||
|
local problem_pods_query
|
||||||
|
local resource_policy_query
|
||||||
|
local node_pressure_query
|
||||||
|
|
||||||
|
node_ready_query="$(grafana_explore_url 'kube_node_status_condition{condition="Ready",status="true"} == 0')"
|
||||||
|
problem_pods_query="$(grafana_explore_url 'sum by (namespace, pod, phase) (kube_pod_status_phase{phase!~"Running|Succeeded"} == 1)')"
|
||||||
|
resource_policy_query="$(grafana_explore_url 'sum by (namespace, pod) (kube_pod_container_resource_requests{resource=~"cpu|memory"})')"
|
||||||
|
node_pressure_query="$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')"
|
||||||
|
|
||||||
if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then
|
if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then
|
||||||
emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG."
|
emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG."
|
||||||
|
|
@ -234,31 +272,31 @@ compact_kubernetes_capacity() {
|
||||||
|
|
||||||
not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')"
|
not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')"
|
||||||
if ((not_ready > 0)); then
|
if ((not_ready > 0)); then
|
||||||
emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster"
|
emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster" "${GRAFANA_AVAILABILITY_URL}&viewPanel=1" "${node_ready_query}"
|
||||||
else
|
else
|
||||||
emit ok "Kubernetes" "Node readiness" "all ready"
|
emit ok "Kubernetes" "Node readiness" "all ready" "" "" "${GRAFANA_AVAILABILITY_URL}&viewPanel=1" "${node_ready_query}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')"
|
problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')"
|
||||||
if ((problem_pods > 0)); then
|
if ((problem_pods > 0)); then
|
||||||
emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide"
|
emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide" "${GRAFANA_AVAILABILITY_URL}&viewPanel=2" "${problem_pods_query}"
|
||||||
else
|
else
|
||||||
emit ok "Kubernetes" "Problem pods" "none"
|
emit ok "Kubernetes" "Problem pods" "none" "" "" "${GRAFANA_AVAILABILITY_URL}&viewPanel=2" "${problem_pods_query}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
||||||
awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')"
|
awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')"
|
||||||
if ((missing_resources > 0)); then
|
if ((missing_resources > 0)); then
|
||||||
emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget"
|
emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget" "${GRAFANA_CAPACITY_URL}&viewPanel=3" "${resource_policy_query}"
|
||||||
else
|
else
|
||||||
emit ok "Kubernetes" "Resource policy" "requests/limits present"
|
emit ok "Kubernetes" "Resource policy" "requests/limits present" "" "" "${GRAFANA_CAPACITY_URL}&viewPanel=3" "${resource_policy_query}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
||||||
if [[ -n "${top_nodes}" ]]; then
|
if [[ -n "${top_nodes}" ]]; then
|
||||||
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=3"
|
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "${node_pressure_query}"
|
||||||
else
|
else
|
||||||
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "${node_pressure_query}"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -293,7 +331,7 @@ echo \"total_vms=\${total_vms:-0}\"
|
||||||
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
||||||
storage_pct="${storage_pct%%%}"
|
storage_pct="${storage_pct%%%}"
|
||||||
if ((${storage_pct%.*} >= 85)); then
|
if ((${storage_pct%.*} >= 85)); then
|
||||||
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Grafana: ${GRAFANA_CAPACITY_URL}"
|
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
|
||||||
else
|
else
|
||||||
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
||||||
fi
|
fi
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ usage() {
|
||||||
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json]
|
||||||
|
|
||||||
Reads TSV rows from stdin:
|
Reads TSV rows from stdin:
|
||||||
status<TAB>area<TAB>check<TAB>summary<TAB>detail<TAB>hint
|
status<TAB>area<TAB>check<TAB>summary<TAB>detail<TAB>fix<TAB>graph<TAB>query
|
||||||
|
|
||||||
Statuses: ok, warn, fail, skip
|
Statuses: ok, warn, fail, skip
|
||||||
EOF
|
EOF
|
||||||
|
|
@ -70,9 +70,9 @@ with open(input_file, encoding="utf-8") as handle:
|
||||||
if not raw:
|
if not raw:
|
||||||
continue
|
continue
|
||||||
parts = raw.split("\t")
|
parts = raw.split("\t")
|
||||||
while len(parts) < 6:
|
while len(parts) < 8:
|
||||||
parts.append("")
|
parts.append("")
|
||||||
status, area, check, summary, detail, hint = parts[:6]
|
status, area, check, summary, detail, fix, graph, query = parts[:8]
|
||||||
status = status.strip().lower() or "skip"
|
status = status.strip().lower() or "skip"
|
||||||
rows.append(
|
rows.append(
|
||||||
{
|
{
|
||||||
|
|
@ -81,7 +81,9 @@ with open(input_file, encoding="utf-8") as handle:
|
||||||
"check": check.strip(),
|
"check": check.strip(),
|
||||||
"summary": summary.strip(),
|
"summary": summary.strip(),
|
||||||
"detail": detail.strip(),
|
"detail": detail.strip(),
|
||||||
"hint": hint.strip(),
|
"fix": fix.strip(),
|
||||||
|
"graph": graph.strip(),
|
||||||
|
"query": query.strip(),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -133,15 +135,19 @@ for area, area_rows in groups.items():
|
||||||
if mode == "details" or row["status"] in {"fail", "warn"}:
|
if mode == "details" or row["status"] in {"fail", "warn"}:
|
||||||
if row["detail"]:
|
if row["detail"]:
|
||||||
print(f" detail: {row['detail']}")
|
print(f" detail: {row['detail']}")
|
||||||
if row["hint"]:
|
if row["fix"]:
|
||||||
print(f" fix: {row['hint']}")
|
print(f" fix: {row['fix']}")
|
||||||
|
if row["graph"]:
|
||||||
|
print(f" graph: {row['graph']}")
|
||||||
|
if row["query"]:
|
||||||
|
print(f" query: {row['query']}")
|
||||||
|
|
||||||
problems = [row for row in rows if row["status"] in {"fail", "warn"}]
|
problems = [row for row in rows if row["status"] in {"fail", "warn"}]
|
||||||
if problems:
|
if problems:
|
||||||
first = problems[0]
|
first = problems[0]
|
||||||
print("\nNext Fix")
|
print("\nNext Fix")
|
||||||
if first["hint"]:
|
if first["fix"]:
|
||||||
print(first["hint"])
|
print(first["fix"])
|
||||||
else:
|
else:
|
||||||
print(f"Investigate {first['area']} / {first['check']}")
|
print(f"Investigate {first['area']} / {first['check']}")
|
||||||
PY
|
PY
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue