Add what-broke status signals

README.md

@@ -274,7 +274,9 @@ Run a read-only health snapshot from the Debian server with:
 
 It reports host memory/disk, systemd services, Docker Compose stacks,
 Kubernetes health when the API is reachable, Pimox worker VM status, RPi
-services, Tailscale, and key local/public HTTP endpoints.
+services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
+such as deployment readiness, pod restarts, node pressure, disk pressure,
+Traefik 5xx/404 log evidence, and recent Gitea errors.
 
 Focused doctor commands run narrower read-only checks and print the most likely
 next step:

docs/jeannie.1.md

@@ -48,7 +48,9 @@ cluster state.
 `status`
 : Print a cascade health report for the configured homelab, from bootstrap
 readiness through public website health. Set `LAB_STATUS_DETAILS=true` to append
-the older detailed tables.
+the older detailed tables. The cascade includes "what broke" signals for recent
+deployment readiness, pod restarts, node pressure, disk pressure, Traefik
+5xx/404 log evidence, and recent Gitea errors.
 
 `nuke`
 : Destroy Kubernetes state and Pimox worker VMs. Requires
@@ -214,6 +216,17 @@ model.
 `HOMELAB_INVENTORY_FILE`
 : Path to the canonical non-secret inventory. Defaults to `./homelab.yml`.
 
+`LAB_STATUS_LOG_SINCE`
+: Log window for status log checks. Defaults to `30m`.
+
+`LAB_STATUS_RESTART_THRESHOLD`
+: Pod restart count that turns the status restart-pressure check into a warning.
+Defaults to `3`.
+
+`LAB_STATUS_DISK_USE_THRESHOLD`
+: Disk usage percentage that turns the status disk check into a warning.
+Defaults to `85`.
+
 `HOMELAB_STATE_DIR`
 : Local runtime state directory. Defaults to
 `${XDG_DATA_HOME:-$HOME/.local/share}/homelab`.

jeannie

@@ -4767,6 +4767,103 @@ status_no_problem_pods() {
     fi
 }
 
+status_recent_deployments() {
+    local rows
+
+    rows="$(
+        kubectl --kubeconfig "${KUBECONFIG_PATH}" get deployments -A \
+            -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.readyReplicas,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,AGE:.metadata.creationTimestamp' \
+            --no-headers 2>/dev/null |
+            awk '$3 != $4 || $5 != $4 || $6 != $4 { print }'
+    )"
+    if [[ -n "${rows}" ]]; then
+        printf '%s\n' "${rows}" >&2
+        return 1
+    fi
+}
+
+status_pod_restart_pressure() {
+    local threshold="${LAB_STATUS_RESTART_THRESHOLD:-3}"
+    local rows
+
+    rows="$(
+        kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null |
+            awk -v threshold="${threshold}" '$5 + 0 >= threshold { print $1 "/" $2 ": restarts=" $5 " age=" $6 }'
+    )"
+    if [[ -n "${rows}" ]]; then
+        printf '%s\n' "${rows}" >&2
+        return 1
+    fi
+}
+
+status_node_pressure() {
+    local rows
+
+    rows="$(
+        kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}{"="}{.status}{" "}{end}{"\n"}{end}' 2>/dev/null |
+            awk '/MemoryPressure=True|DiskPressure=True|PIDPressure=True|NetworkUnavailable=True/ { print }'
+    )"
+    if [[ -n "${rows}" ]]; then
+        printf '%s\n' "${rows}" >&2
+        return 1
+    fi
+}
+
+status_disk_pressure() {
+    local threshold="${LAB_STATUS_DISK_USE_THRESHOLD:-85}"
+    local rows
+
+    rows="$(df -P / /data 2>/dev/null | awk -v threshold="${threshold}" 'NR > 1 { gsub(/%/, "", $5); if ($5 + 0 >= threshold) print $6 ": " $5 "% used" }')"
+    if [[ -n "${rows}" ]]; then
+        printf '%s\n' "${rows}" >&2
+        return 1
+    fi
+}
+
+status_traefik_error_rates() {
+    local since="${LAB_STATUS_LOG_SINCE:-30m}"
+    local sample
+    local count_502
+    local count_404
+
+    sample="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik logs -l app.kubernetes.io/name=traefik --since="${since}" --tail=2000 2>/dev/null || true)"
+    if [[ -z "${sample}" ]]; then
+        echo "no recent Traefik logs available" >&2
+        return 1
+    fi
+
+    count_502="$(grep -Eac '(^|[^0-9])50[234]([^0-9]|$)|level=error' <<<"${sample}" || true)"
+    count_404="$(grep -Eac '(^|[^0-9])404([^0-9]|$)' <<<"${sample}" || true)"
+    if ((count_502 > 0)); then
+        printf '5xx/error lines in last %s: %s\n' "${since}" "${count_502}" >&2
+        return 1
+    fi
+    printf '404 lines in last %s: %s\n' "${since}" "${count_404}" >&2
+}
+
+status_gitea_recent_errors() {
+    local since="${LAB_STATUS_LOG_SINCE:-30m}"
+    local container="${GITEA_CONTAINER_NAME:-homelab-gitea}"
+    local sample
+    local matches
+
+    if ! command -v docker >/dev/null 2>&1; then
+        echo "docker not installed" >&2
+        return 1
+    fi
+    if ! sudo docker inspect "${container}" >/dev/null 2>&1; then
+        echo "container ${container} not found" >&2
+        return 1
+    fi
+
+    sample="$(sudo docker logs --since "${since}" "${container}" 2>&1 || true)"
+    matches="$(grep -Eai 'error|panic|fatal|authentication failed|denied| 50[0-9] ' <<<"${sample}" | tail -10 || true)"
+    if [[ -n "${matches}" ]]; then
+        printf '%s\n' "${matches}" >&2
+        return 1
+    fi
+}
+
 status_registry_ok() {
     status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/"
 }
@@ -4843,6 +4940,14 @@ status_cascade_report() {
     status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1))
     status_cascade_warn "No problem pods" status_no_problem_pods
 
+    status_section "What Broke Signals"
+    status_cascade_warn "Recent deployments healthy" status_recent_deployments
+    status_cascade_warn "Pod restart pressure" status_pod_restart_pressure
+    status_cascade_warn "Node pressure" status_node_pressure
+    status_cascade_warn "Disk usage" status_disk_pressure
+    status_cascade_warn "Traefik 5xx/404 signals" status_traefik_error_rates
+    status_cascade_warn "Gitea recent errors" status_gitea_recent_errors
+
     status_section "Platform And Apps"
     status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1))
     status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1))