diff --git a/README.md b/README.md index a08a9a3..c40af94 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,9 @@ inventory, and docs freshness into a compact pass/warn/fail report. Report-oriented commands can share the reusable renderer in `scripts/report-render`. It reads status TSV rows and prints compact grouped summaries by default, with details or JSON available for future TUI/web output. +Rows may include separate `fix`, `graph`, and `query` fields so a warning can +point to both the next `jeannie` command and the matching Grafana panel or +Explore query. `gitops-status` prints a focused Argo CD view: application sync and health, out-of-sync or degraded apps, repository secret presence, recent Argo CD events, diff --git a/README.md.tmpl b/README.md.tmpl index b2fd134..c619506 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -327,6 +327,9 @@ inventory, and docs freshness into a compact pass/warn/fail report. Report-oriented commands can share the reusable renderer in `scripts/report-render`. It reads status TSV rows and prints compact grouped summaries by default, with details or JSON available for future TUI/web output. +Rows may include separate `fix`, `graph`, and `query` fields so a warning can +point to both the next `jeannie` command and the matching Grafana panel or +Explore query. `gitops-status` prints a focused Argo CD view: application sync and health, out-of-sync or degraded apps, repository secret presence, recent Argo CD events, diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index f7020b1..d0b408f 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -73,7 +73,9 @@ docs freshness into a compact pass/warn/fail report. Report renderer : Report-oriented commands can share `scripts/report-render`, which reads status -TSV rows and prints compact grouped summaries, detailed output, or JSON. +TSV rows and prints compact grouped summaries, detailed output, or JSON. Rows +may include separate `fix`, `graph`, and `query` fields for Grafana-backed +checks. `gitops-status` : Print a focused Argo CD status view with application sync/health, out-of-sync diff --git a/scripts/capacity-report b/scripts/capacity-report index 6f9a031..4492dff 100755 --- a/scripts/capacity-report +++ b/scripts/capacity-report @@ -5,6 +5,7 @@ KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}" GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1" +GRAFANA_AVAILABILITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-availability/homelab-availability?orgId=1" DETAIL_ARGS=() VERBOSE=false @@ -45,9 +46,37 @@ emit() { local check="$3" local summary="$4" local detail="${5:-}" - local hint="${6:-}" + local fix="${6:-}" + local graph="${7:-}" + local query="${8:-}" - printf '%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${hint}" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${fix}" "${graph}" "${query}" +} + +grafana_explore_url() { + local expr="$1" + python3 - "${GRAFANA_BASE_URL%/}" "$expr" <<'PY' +import json +import sys +import urllib.parse + +base_url, expr = sys.argv[1:3] +left = { + "datasource": "prometheus", + "queries": [ + { + "refId": "A", + "expr": expr, + "range": True, + } + ], + "range": { + "from": "now-6h", + "to": "now", + }, +} +print(f"{base_url}/explore?left={urllib.parse.quote(json.dumps(left, separators=(',', ':')))}") +PY } section() { @@ -177,9 +206,9 @@ compact_host_capacity() { if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then mem_pct=$((mem_used * 100 / mem_total)) if ((mem_pct >= 85)); then - emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1" + emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers." "${GRAFANA_CAPACITY_URL}&viewPanel=1" "$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')" else - emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=1" + emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')" fi else emit skip "Debian" "Memory" "unavailable" "free output could not be parsed" @@ -197,11 +226,11 @@ compact_host_capacity() { use_pct="${line%% *}" use_pct="${use_pct%%%}" if ((use_pct >= 90)); then - emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2" + emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')" elif ((use_pct >= 80)); then - emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2" + emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')" else - emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=2" + emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "" "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')" fi done @@ -222,6 +251,15 @@ compact_kubernetes_capacity() { local problem_pods local missing_resources local top_nodes + local node_ready_query + local problem_pods_query + local resource_policy_query + local node_pressure_query + + node_ready_query="$(grafana_explore_url 'kube_node_status_condition{condition="Ready",status="true"} == 0')" + problem_pods_query="$(grafana_explore_url 'sum by (namespace, pod, phase) (kube_pod_status_phase{phase!~"Running|Succeeded"} == 1)')" + resource_policy_query="$(grafana_explore_url 'sum by (namespace, pod) (kube_pod_container_resource_requests{resource=~"cpu|memory"})')" + node_pressure_query="$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')" if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG." @@ -234,31 +272,31 @@ compact_kubernetes_capacity() { not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')" if ((not_ready > 0)); then - emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster" + emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster" "${GRAFANA_AVAILABILITY_URL}&viewPanel=1" "${node_ready_query}" else - emit ok "Kubernetes" "Node readiness" "all ready" + emit ok "Kubernetes" "Node readiness" "all ready" "" "" "${GRAFANA_AVAILABILITY_URL}&viewPanel=1" "${node_ready_query}" fi problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')" if ((problem_pods > 0)); then - emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide" + emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide" "${GRAFANA_AVAILABILITY_URL}&viewPanel=2" "${problem_pods_query}" else - emit ok "Kubernetes" "Problem pods" "none" + emit ok "Kubernetes" "Problem pods" "none" "" "" "${GRAFANA_AVAILABILITY_URL}&viewPanel=2" "${problem_pods_query}" fi missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null | awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')" if ((missing_resources > 0)); then - emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget" + emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget" "${GRAFANA_CAPACITY_URL}&viewPanel=3" "${resource_policy_query}" else - emit ok "Kubernetes" "Resource policy" "requests/limits present" + emit ok "Kubernetes" "Resource policy" "requests/limits present" "" "" "${GRAFANA_CAPACITY_URL}&viewPanel=3" "${resource_policy_query}" fi top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)" if [[ -n "${top_nodes}" ]]; then - emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "Grafana: ${GRAFANA_CAPACITY_URL}&viewPanel=3" + emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "${node_pressure_query}" else - emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready" + emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "${node_pressure_query}" fi } @@ -293,7 +331,7 @@ echo \"total_vms=\${total_vms:-0}\" if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then storage_pct="${storage_pct%%%}" if ((${storage_pct%.*} >= 85)); then - emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Grafana: ${GRAFANA_CAPACITY_URL}" + emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage." else emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" fi diff --git a/scripts/report-render b/scripts/report-render index fd93444..a30767b 100755 --- a/scripts/report-render +++ b/scripts/report-render @@ -18,7 +18,7 @@ usage() { Usage: report-render [--title TITLE] [--details] [--only failures|warnings|problems|all] [--json] Reads TSV rows from stdin: - statusareachecksummarydetailhint + statusareachecksummarydetailfixgraphquery Statuses: ok, warn, fail, skip EOF @@ -70,9 +70,9 @@ with open(input_file, encoding="utf-8") as handle: if not raw: continue parts = raw.split("\t") - while len(parts) < 6: + while len(parts) < 8: parts.append("") - status, area, check, summary, detail, hint = parts[:6] + status, area, check, summary, detail, fix, graph, query = parts[:8] status = status.strip().lower() or "skip" rows.append( { @@ -81,7 +81,9 @@ with open(input_file, encoding="utf-8") as handle: "check": check.strip(), "summary": summary.strip(), "detail": detail.strip(), - "hint": hint.strip(), + "fix": fix.strip(), + "graph": graph.strip(), + "query": query.strip(), } ) @@ -133,15 +135,19 @@ for area, area_rows in groups.items(): if mode == "details" or row["status"] in {"fail", "warn"}: if row["detail"]: print(f" detail: {row['detail']}") - if row["hint"]: - print(f" fix: {row['hint']}") + if row["fix"]: + print(f" fix: {row['fix']}") + if row["graph"]: + print(f" graph: {row['graph']}") + if row["query"]: + print(f" query: {row['query']}") problems = [row for row in rows if row["status"] in {"fail", "warn"}] if problems: first = problems[0] print("\nNext Fix") - if first["hint"]: - print(first["hint"]) + if first["fix"]: + print(first["fix"]) else: print(f"Investigate {first['area']} / {first['check']}") PY