diff --git a/README.md b/README.md index 0db3974..ec05a89 100644 --- a/README.md +++ b/README.md @@ -274,7 +274,9 @@ Run a read-only health snapshot from the Debian server with: It reports host memory/disk, systemd services, Docker Compose stacks, Kubernetes health when the API is reachable, Pimox worker VM status, RPi -services, Tailscale, and key local/public HTTP endpoints. +services, Tailscale, key local/public HTTP endpoints, and "what broke" signals +such as deployment readiness, pod restarts, node pressure, disk pressure, +Traefik 5xx/404 log evidence, and recent Gitea errors. Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index bb7714a..c4caba0 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -48,7 +48,9 @@ cluster state. `status` : Print a cascade health report for the configured homelab, from bootstrap readiness through public website health. Set `LAB_STATUS_DETAILS=true` to append -the older detailed tables. +the older detailed tables. The cascade includes "what broke" signals for recent +deployment readiness, pod restarts, node pressure, disk pressure, Traefik +5xx/404 log evidence, and recent Gitea errors. `nuke` : Destroy Kubernetes state and Pimox worker VMs. Requires @@ -214,6 +216,17 @@ model. `HOMELAB_INVENTORY_FILE` : Path to the canonical non-secret inventory. Defaults to `./homelab.yml`. +`LAB_STATUS_LOG_SINCE` +: Log window for status log checks. Defaults to `30m`. + +`LAB_STATUS_RESTART_THRESHOLD` +: Pod restart count that turns the status restart-pressure check into a warning. +Defaults to `3`. + +`LAB_STATUS_DISK_USE_THRESHOLD` +: Disk usage percentage that turns the status disk check into a warning. +Defaults to `85`. + `HOMELAB_STATE_DIR` : Local runtime state directory. Defaults to `${XDG_DATA_HOME:-$HOME/.local/share}/homelab`. diff --git a/jeannie b/jeannie index d5437bb..93f5467 100755 --- a/jeannie +++ b/jeannie @@ -4767,6 +4767,103 @@ status_no_problem_pods() { fi } +status_recent_deployments() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get deployments -A \ + -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.readyReplicas,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,AGE:.metadata.creationTimestamp' \ + --no-headers 2>/dev/null | + awk '$3 != $4 || $5 != $4 || $6 != $4 { print }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_pod_restart_pressure() { + local threshold="${LAB_STATUS_RESTART_THRESHOLD:-3}" + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | + awk -v threshold="${threshold}" '$5 + 0 >= threshold { print $1 "/" $2 ": restarts=" $5 " age=" $6 }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_node_pressure() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}{"="}{.status}{" "}{end}{"\n"}{end}' 2>/dev/null | + awk '/MemoryPressure=True|DiskPressure=True|PIDPressure=True|NetworkUnavailable=True/ { print }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_disk_pressure() { + local threshold="${LAB_STATUS_DISK_USE_THRESHOLD:-85}" + local rows + + rows="$(df -P / /data 2>/dev/null | awk -v threshold="${threshold}" 'NR > 1 { gsub(/%/, "", $5); if ($5 + 0 >= threshold) print $6 ": " $5 "% used" }')" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_traefik_error_rates() { + local since="${LAB_STATUS_LOG_SINCE:-30m}" + local sample + local count_502 + local count_404 + + sample="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik logs -l app.kubernetes.io/name=traefik --since="${since}" --tail=2000 2>/dev/null || true)" + if [[ -z "${sample}" ]]; then + echo "no recent Traefik logs available" >&2 + return 1 + fi + + count_502="$(grep -Eac '(^|[^0-9])50[234]([^0-9]|$)|level=error' <<<"${sample}" || true)" + count_404="$(grep -Eac '(^|[^0-9])404([^0-9]|$)' <<<"${sample}" || true)" + if ((count_502 > 0)); then + printf '5xx/error lines in last %s: %s\n' "${since}" "${count_502}" >&2 + return 1 + fi + printf '404 lines in last %s: %s\n' "${since}" "${count_404}" >&2 +} + +status_gitea_recent_errors() { + local since="${LAB_STATUS_LOG_SINCE:-30m}" + local container="${GITEA_CONTAINER_NAME:-homelab-gitea}" + local sample + local matches + + if ! command -v docker >/dev/null 2>&1; then + echo "docker not installed" >&2 + return 1 + fi + if ! sudo docker inspect "${container}" >/dev/null 2>&1; then + echo "container ${container} not found" >&2 + return 1 + fi + + sample="$(sudo docker logs --since "${since}" "${container}" 2>&1 || true)" + matches="$(grep -Eai 'error|panic|fatal|authentication failed|denied| 50[0-9] ' <<<"${sample}" | tail -10 || true)" + if [[ -n "${matches}" ]]; then + printf '%s\n' "${matches}" >&2 + return 1 + fi +} + status_registry_ok() { status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/" } @@ -4843,6 +4940,14 @@ status_cascade_report() { status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1)) status_cascade_warn "No problem pods" status_no_problem_pods + status_section "What Broke Signals" + status_cascade_warn "Recent deployments healthy" status_recent_deployments + status_cascade_warn "Pod restart pressure" status_pod_restart_pressure + status_cascade_warn "Node pressure" status_node_pressure + status_cascade_warn "Disk usage" status_disk_pressure + status_cascade_warn "Traefik 5xx/404 signals" status_traefik_error_rates + status_cascade_warn "Gitea recent errors" status_gitea_recent_errors + status_section "Platform And Apps" status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1)) status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1))