Add cascade health checks to status
This commit is contained in:
juvdiaz
parent
1d73c33703
commit
d8f518c8e9
|
|
@ -46,7 +46,9 @@ cluster state.
|
||||||
: Start the Kubernetes control-plane runtime and desired worker VMs.
|
: Start the Kubernetes control-plane runtime and desired worker VMs.
|
||||||
|
|
||||||
`status`
|
`status`
|
||||||
: Print a compact health report for the configured homelab.
|
: Print a cascade health report for the configured homelab, from bootstrap
|
||||||
|
readiness through public website health. Set `LAB_STATUS_DETAILS=true` to append
|
||||||
|
the older detailed tables.
|
||||||
|
|
||||||
`nuke`
|
`nuke`
|
||||||
: Destroy Kubernetes state and Pimox worker VMs. Requires
|
: Destroy Kubernetes state and Pimox worker VMs. Requires
|
||||||
|
|
@ -170,6 +172,14 @@ model.
|
||||||
: Print full step output as it runs. Without this, noisy commands are compact and
|
: Print full step output as it runs. Without this, noisy commands are compact and
|
||||||
full output is written to the log file.
|
full output is written to the log file.
|
||||||
|
|
||||||
|
`LAB_STATUS_DETAILS=true`
|
||||||
|
: Append detailed host, Docker, Kubernetes, Pimox, RPi, Tailscale, and HTTP
|
||||||
|
tables after the normal status cascade.
|
||||||
|
|
||||||
|
`LAB_STATUS_SHOW_OK_OUTPUT=true`
|
||||||
|
: Show short command output for successful cascade checks. Failed and warning
|
||||||
|
checks already show short output by default.
|
||||||
|
|
||||||
`LAB_AUTO_APPROVE=false`
|
`LAB_AUTO_APPROVE=false`
|
||||||
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
: Disable automatic OpenTofu approval for apply paths that support confirmation.
|
||||||
|
|
||||||
|
|
|
||||||
233
jeannie
233
jeannie
|
|
@ -4600,6 +4600,229 @@ status_http() {
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
status_http_ok() {
|
||||||
|
local url="$1"
|
||||||
|
local status
|
||||||
|
|
||||||
|
if ! command -v curl >/dev/null 2>&1; then
|
||||||
|
echo "curl not installed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || {
|
||||||
|
echo "unreachable ${url}" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
case "${status}" in
|
||||||
|
2* | 3*)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "HTTP ${status} from ${url}" >&2
|
||||||
|
return 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
status_cascade_check() {
|
||||||
|
local description="$1"
|
||||||
|
local output
|
||||||
|
shift
|
||||||
|
|
||||||
|
printf '%-36s ' "${description}"
|
||||||
|
if output="$("$@" 2>&1)"; then
|
||||||
|
printf 'ok\n'
|
||||||
|
if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then
|
||||||
|
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'fail\n'
|
||||||
|
if [[ -n "${output}" ]]; then
|
||||||
|
printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
status_cascade_warn() {
|
||||||
|
local description="$1"
|
||||||
|
local output
|
||||||
|
shift
|
||||||
|
|
||||||
|
printf '%-36s ' "${description}"
|
||||||
|
if output="$("$@" 2>&1)"; then
|
||||||
|
printf 'ok\n'
|
||||||
|
if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then
|
||||||
|
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'warn\n'
|
||||||
|
if [[ -n "${output}" ]]; then
|
||||||
|
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
|
||||||
|
fi
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
status_container_running() {
|
||||||
|
local container="$1"
|
||||||
|
|
||||||
|
if ! command -v docker >/dev/null 2>&1; then
|
||||||
|
echo "docker not installed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then
|
||||||
|
echo "container ${container} is not running" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
status_kubernetes_api_ok() {
|
||||||
|
if ! command -v kubectl >/dev/null 2>&1; then
|
||||||
|
echo "kubectl not installed" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ ! -s "${KUBECONFIG_PATH}" ]]; then
|
||||||
|
echo "kubeconfig missing: ${KUBECONFIG_PATH}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
status_kubernetes_nodes_ready() {
|
||||||
|
local nodes
|
||||||
|
local not_ready
|
||||||
|
|
||||||
|
nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes --no-headers 2>/dev/null)" || return 1
|
||||||
|
if [[ -z "${nodes}" ]]; then
|
||||||
|
echo "no Kubernetes nodes found" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")"
|
||||||
|
if [[ -n "${not_ready}" ]]; then
|
||||||
|
echo "not ready: ${not_ready}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
status_kubernetes_deployment_ready() {
|
||||||
|
local namespace="$1"
|
||||||
|
local deployment="$2"
|
||||||
|
|
||||||
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
status_no_problem_pods() {
|
||||||
|
local rows
|
||||||
|
|
||||||
|
rows="$(
|
||||||
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null |
|
||||||
|
awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }'
|
||||||
|
)"
|
||||||
|
if [[ -n "${rows}" ]]; then
|
||||||
|
printf '%s\n' "${rows}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
status_registry_ok() {
|
||||||
|
status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/"
|
||||||
|
}
|
||||||
|
|
||||||
|
status_rpi_uptime_kuma_ok() {
|
||||||
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
||||||
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
||||||
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
||||||
|
local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}"
|
||||||
|
|
||||||
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \
|
||||||
|
"curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null"
|
||||||
|
}
|
||||||
|
|
||||||
|
status_pimox_workers_running() {
|
||||||
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||||
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||||
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
||||||
|
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
||||||
|
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
|
||||||
|
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
||||||
|
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
||||||
|
local index
|
||||||
|
local vmid
|
||||||
|
local failures=0
|
||||||
|
|
||||||
|
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "invalid worker count ${worker_count}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
for ((index = 1; index <= worker_count; index++)); do
|
||||||
|
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
vmid=$((worker_base_vmid + index - 1))
|
||||||
|
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then
|
||||||
|
echo "VM ${vmid} is not running" >&2
|
||||||
|
failures=$((failures + 1))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
((failures == 0))
|
||||||
|
}
|
||||||
|
|
||||||
|
status_cascade_report() {
|
||||||
|
local failures=0
|
||||||
|
|
||||||
|
status_section "Status Cascade"
|
||||||
|
status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1))
|
||||||
|
|
||||||
|
status_section "Git And Local Services"
|
||||||
|
status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1))
|
||||||
|
status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/"
|
||||||
|
status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/"
|
||||||
|
status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/"
|
||||||
|
|
||||||
|
status_section "RPi Bootstrap And DNS"
|
||||||
|
status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1))
|
||||||
|
status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1))
|
||||||
|
status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip
|
||||||
|
status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok
|
||||||
|
|
||||||
|
status_section "Pimox And Cluster"
|
||||||
|
status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1))
|
||||||
|
status_cascade_warn "No problem pods" status_no_problem_pods
|
||||||
|
|
||||||
|
status_section "Platform And Apps"
|
||||||
|
status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1))
|
||||||
|
status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server
|
||||||
|
|
||||||
|
status_section "Edge And Public URLs"
|
||||||
|
status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Traefik LoadBalancer HTTP" status_http_ok "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" || failures=$((failures + 1))
|
||||||
|
status_cascade_check "Gitea public URL" status_http_ok "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" || failures=$((failures + 1))
|
||||||
|
|
||||||
|
if ((failures > 0)); then
|
||||||
|
printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '\nStatus cascade passed.\n'
|
||||||
|
}
|
||||||
|
|
||||||
backstage_brain_enabled() {
|
backstage_brain_enabled() {
|
||||||
local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}"
|
local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}"
|
||||||
local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}"
|
local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}"
|
||||||
|
|
@ -4900,8 +5123,16 @@ fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}"
|
||||||
}
|
}
|
||||||
|
|
||||||
status_report() {
|
status_report() {
|
||||||
|
local cascade_status=0
|
||||||
|
|
||||||
require_debian_server "status"
|
require_debian_server "status"
|
||||||
|
|
||||||
|
status_cascade_report || cascade_status=$?
|
||||||
|
|
||||||
|
if ! truthy "${LAB_STATUS_DETAILS:-false}"; then
|
||||||
|
return "${cascade_status}"
|
||||||
|
fi
|
||||||
|
|
||||||
status_section "Host"
|
status_section "Host"
|
||||||
printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)"
|
printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)"
|
||||||
printf 'date: %s\n' "$(date -Is)"
|
printf 'date: %s\n' "$(date -Is)"
|
||||||
|
|
@ -4941,6 +5172,8 @@ status_report() {
|
||||||
status_http "arr radarr" "http://127.0.0.1:7878/"
|
status_http "arr radarr" "http://127.0.0.1:7878/"
|
||||||
status_http "arr sonarr" "http://127.0.0.1:8989/"
|
status_http "arr sonarr" "http://127.0.0.1:8989/"
|
||||||
status_http "arr prowlarr" "http://127.0.0.1:9696/"
|
status_http "arr prowlarr" "http://127.0.0.1:9696/"
|
||||||
|
|
||||||
|
return "${cascade_status}"
|
||||||
}
|
}
|
||||||
|
|
||||||
doctor_edge() {
|
doctor_edge() {
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue