From d8f518c8e960317a9f0e7a4d3a804eb6db5559b8 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 13:40:33 -0600 Subject: [PATCH] Add cascade health checks to status --- docs/jeannie.1.md | 12 ++- jeannie | 233 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 244 insertions(+), 1 deletion(-) diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index f9013fe..bd9b9f9 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -46,7 +46,9 @@ cluster state. : Start the Kubernetes control-plane runtime and desired worker VMs. `status` -: Print a compact health report for the configured homelab. +: Print a cascade health report for the configured homelab, from bootstrap +readiness through public website health. Set `LAB_STATUS_DETAILS=true` to append +the older detailed tables. `nuke` : Destroy Kubernetes state and Pimox worker VMs. Requires @@ -170,6 +172,14 @@ model. : Print full step output as it runs. Without this, noisy commands are compact and full output is written to the log file. +`LAB_STATUS_DETAILS=true` +: Append detailed host, Docker, Kubernetes, Pimox, RPi, Tailscale, and HTTP +tables after the normal status cascade. + +`LAB_STATUS_SHOW_OK_OUTPUT=true` +: Show short command output for successful cascade checks. Failed and warning +checks already show short output by default. + `LAB_AUTO_APPROVE=false` : Disable automatic OpenTofu approval for apply paths that support confirmation. diff --git a/jeannie b/jeannie index 56921ef..99f3df9 100755 --- a/jeannie +++ b/jeannie @@ -4600,6 +4600,229 @@ status_http() { fi } +status_http_ok() { + local url="$1" + local status + + if ! command -v curl >/dev/null 2>&1; then + echo "curl not installed" >&2 + return 1 + fi + + status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { + echo "unreachable ${url}" >&2 + return 1 + } + case "${status}" in + 2* | 3*) + return 0 + ;; + *) + echo "HTTP ${status} from ${url}" >&2 + return 1 + ;; + esac +} + +status_cascade_check() { + local description="$1" + local output + shift + + printf '%-36s ' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 + fi + + printf 'fail\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /' + fi + return 1 +} + +status_cascade_warn() { + local description="$1" + local output + shift + + printf '%-36s ' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 + fi + + printf 'warn\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 +} + +status_container_running() { + local container="$1" + + if ! command -v docker >/dev/null 2>&1; then + echo "docker not installed" >&2 + return 1 + fi + if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then + echo "container ${container} is not running" >&2 + return 1 + fi +} + +status_kubernetes_api_ok() { + if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl not installed" >&2 + return 1 + fi + if [[ ! -s "${KUBECONFIG_PATH}" ]]; then + echo "kubeconfig missing: ${KUBECONFIG_PATH}" >&2 + return 1 + fi + kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null +} + +status_kubernetes_nodes_ready() { + local nodes + local not_ready + + nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes --no-headers 2>/dev/null)" || return 1 + if [[ -z "${nodes}" ]]; then + echo "no Kubernetes nodes found" >&2 + return 1 + fi + not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")" + if [[ -n "${not_ready}" ]]; then + echo "not ready: ${not_ready}" >&2 + return 1 + fi +} + +status_kubernetes_deployment_ready() { + local namespace="$1" + local deployment="$2" + + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null +} + +status_no_problem_pods() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | + awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_registry_ok() { + status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/" +} + +status_rpi_uptime_kuma_ok() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \ + "curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null" +} + +status_pimox_workers_running() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + local failures=0 + + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "invalid worker count ${worker_count}" >&2 + return 1 + fi + + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + continue + fi + vmid=$((worker_base_vmid + index - 1)) + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then + echo "VM ${vmid} is not running" >&2 + failures=$((failures + 1)) + fi + done + + ((failures == 0)) +} + +status_cascade_report() { + local failures=0 + + status_section "Status Cascade" + status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1)) + status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1)) + status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1)) + status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1)) + status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1)) + + status_section "Git And Local Services" + status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1)) + status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1)) + status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/" + status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/" + status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/" + + status_section "RPi Bootstrap And DNS" + status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1)) + status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1)) + status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1)) + status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip + status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok + + status_section "Pimox And Cluster" + status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1)) + status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1)) + status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1)) + status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1)) + status_cascade_warn "No problem pods" status_no_problem_pods + + status_section "Platform And Apps" + status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1)) + status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1)) + status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1)) + status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server + + status_section "Edge And Public URLs" + status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1)) + status_cascade_check "Traefik LoadBalancer HTTP" status_http_ok "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" || failures=$((failures + 1)) + status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" || failures=$((failures + 1)) + status_cascade_check "Gitea public URL" status_http_ok "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" || failures=$((failures + 1)) + + if ((failures > 0)); then + printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2 + return 1 + fi + + printf '\nStatus cascade passed.\n' +} + backstage_brain_enabled() { local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}" local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}" @@ -4900,8 +5123,16 @@ fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" } status_report() { + local cascade_status=0 + require_debian_server "status" + status_cascade_report || cascade_status=$? + + if ! truthy "${LAB_STATUS_DETAILS:-false}"; then + return "${cascade_status}" + fi + status_section "Host" printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)" printf 'date: %s\n' "$(date -Is)" @@ -4941,6 +5172,8 @@ status_report() { status_http "arr radarr" "http://127.0.0.1:7878/" status_http "arr sonarr" "http://127.0.0.1:8989/" status_http "arr prowlarr" "http://127.0.0.1:9696/" + + return "${cascade_status}" } doctor_edge() {