Add reusable report UI for status doctors
This commit is contained in:
juvdiaz
parent
00cfff3ae7
commit
19964cfc20
29
README.md
29
README.md
|
|
@ -341,11 +341,13 @@ Run a read-only health snapshot from the Debian server with:
|
|||
and bootstrap through cluster lifecycle, reports, observability, recovery,
|
||||
access, apps, security, AI, and destructive commands.
|
||||
|
||||
It reports host memory/disk, systemd services, Docker Compose stacks,
|
||||
Kubernetes health when the API is reachable, Pimox worker VM status, RPi
|
||||
services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
|
||||
such as deployment readiness, pod restarts, node pressure, disk pressure,
|
||||
Traefik 5xx/404 log evidence, and recent Gitea errors.
|
||||
`status` uses the reusable report UI and defaults to showing only failures and
|
||||
warnings so the terminal stays readable. Use `./jeannie status --all`
|
||||
to show every check, `--details` for row details, `--json` for structured
|
||||
output, or `--verbose` for the older full host/Docker/Kubernetes/Pimox/RPi/
|
||||
Tailscale/HTTP dump. The cascade includes "what broke" signals such as
|
||||
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||
5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
|
||||
readiness, pod restart storms, unavailable deployments, node/PVC storage
|
||||
|
|
@ -400,12 +402,12 @@ output. It can run safe report commands itself, such as
|
|||
finding means, diagnostic commands, whether a Jeannie fix command already
|
||||
exists, and risk/auto-fix safety.
|
||||
|
||||
Report-oriented commands can share the reusable renderer in
|
||||
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
||||
summaries by default, with details or JSON available for future TUI/web output.
|
||||
Rows may include separate `fix`, `graph`, and `query` fields so a warning can
|
||||
point to both the next `jeannie` command and the matching Grafana panel or
|
||||
Explore query.
|
||||
Report-oriented commands share the reusable UI module in `scripts/report-ui`
|
||||
and renderer in `scripts/report-render`. Commands emit status TSV rows, and the
|
||||
renderer prints compact grouped summaries by default, with details or JSON
|
||||
available for future TUI/web output. Rows may include separate `fix`, `graph`,
|
||||
and `query` fields so a warning can point to both the next `jeannie` command and
|
||||
the matching Grafana panel or Explore query.
|
||||
|
||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||
|
|
@ -485,8 +487,9 @@ dirty file count, and command details.
|
|||
entry, GitOps automation, cluster foundation, DNS, observability, and security
|
||||
learning paths. `./jeannie map --dot` emits Graphviz DOT.
|
||||
|
||||
Focused doctor commands run narrower read-only checks and print the most likely
|
||||
next step:
|
||||
Focused doctor commands run narrower read-only checks. They use the same
|
||||
problems-first UI as `status`; add `--all`, `--details`, `--json`, or
|
||||
`--verbose` when you need more context:
|
||||
|
||||
```bash
|
||||
./jeannie doctor-edge
|
||||
|
|
|
|||
|
|
@ -341,11 +341,13 @@ Run a read-only health snapshot from the Debian server with:
|
|||
and bootstrap through cluster lifecycle, reports, observability, recovery,
|
||||
access, apps, security, AI, and destructive commands.
|
||||
|
||||
It reports host memory/disk, systemd services, Docker Compose stacks,
|
||||
Kubernetes health when the API is reachable, Pimox worker VM status, RPi
|
||||
services, Tailscale, key local/public HTTP endpoints, and "what broke" signals
|
||||
such as deployment readiness, pod restarts, node pressure, disk pressure,
|
||||
Traefik 5xx/404 log evidence, and recent Gitea errors.
|
||||
`status` uses the reusable report UI and defaults to showing only failures and
|
||||
warnings so the terminal stays readable. Use `./{{ main_script }} status --all`
|
||||
to show every check, `--details` for row details, `--json` for structured
|
||||
output, or `--verbose` for the older full host/Docker/Kubernetes/Pimox/RPi/
|
||||
Tailscale/HTTP dump. The cascade includes "what broke" signals such as
|
||||
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||
5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
High-signal Prometheus alerts live in `apps/homelab-alerts`. They cover node
|
||||
readiness, pod restart storms, unavailable deployments, node/PVC storage
|
||||
|
|
@ -400,12 +402,12 @@ output. It can run safe report commands itself, such as
|
|||
finding means, diagnostic commands, whether a Jeannie fix command already
|
||||
exists, and risk/auto-fix safety.
|
||||
|
||||
Report-oriented commands can share the reusable renderer in
|
||||
`scripts/report-render`. It reads status TSV rows and prints compact grouped
|
||||
summaries by default, with details or JSON available for future TUI/web output.
|
||||
Rows may include separate `fix`, `graph`, and `query` fields so a warning can
|
||||
point to both the next `jeannie` command and the matching Grafana panel or
|
||||
Explore query.
|
||||
Report-oriented commands share the reusable UI module in `scripts/report-ui`
|
||||
and renderer in `scripts/report-render`. Commands emit status TSV rows, and the
|
||||
renderer prints compact grouped summaries by default, with details or JSON
|
||||
available for future TUI/web output. Rows may include separate `fix`, `graph`,
|
||||
and `query` fields so a warning can point to both the next `jeannie` command and
|
||||
the matching Grafana panel or Explore query.
|
||||
|
||||
`gitops-status` prints a focused Argo CD view: application sync and health,
|
||||
out-of-sync or degraded apps, repository secret presence, recent Argo CD events,
|
||||
|
|
@ -485,8 +487,9 @@ dirty file count, and command details.
|
|||
entry, GitOps automation, cluster foundation, DNS, observability, and security
|
||||
learning paths. `./{{ main_script }} map --dot` emits Graphviz DOT.
|
||||
|
||||
Focused doctor commands run narrower read-only checks and print the most likely
|
||||
next step:
|
||||
Focused doctor commands run narrower read-only checks. They use the same
|
||||
problems-first UI as `status`; add `--all`, `--details`, `--json`, or
|
||||
`--verbose` when you need more context:
|
||||
|
||||
```bash
|
||||
./{{ main_script }} doctor-edge
|
||||
|
|
|
|||
|
|
@ -55,11 +55,13 @@ cluster state.
|
|||
: Start the Kubernetes control-plane runtime and desired worker VMs.
|
||||
|
||||
`status`
|
||||
: Print a cascade health report for the configured homelab, from bootstrap
|
||||
readiness through public website health. Set `LAB_STATUS_DETAILS=true` to append
|
||||
the older detailed tables. The cascade includes "what broke" signals for recent
|
||||
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||
5xx/404 log evidence, and recent Gitea errors.
|
||||
: Print a problems-first cascade health report for the configured homelab, from
|
||||
bootstrap readiness through public website health. By default it only shows
|
||||
failures and warnings. Use `status --all` for every check, `status --details`
|
||||
for row details, `status --json` for structured output, or `status --verbose`
|
||||
for the older detailed tables. The cascade includes "what broke" signals for
|
||||
recent deployment readiness, pod restarts, node pressure, disk pressure,
|
||||
Traefik 5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
`capacity`
|
||||
: Print a compact read-only placement report covering Debian, Kubernetes,
|
||||
|
|
@ -91,10 +93,12 @@ health, capacity pressure, security posture, public certificates, inventory, and
|
|||
docs freshness into a compact pass/warn/fail report.
|
||||
|
||||
Report renderer
|
||||
: Report-oriented commands can share `scripts/report-render`, which reads status
|
||||
TSV rows and prints compact grouped summaries, detailed output, or JSON. Rows
|
||||
may include separate `fix`, `graph`, and `query` fields for Grafana-backed
|
||||
checks.
|
||||
: Report-oriented commands share `scripts/report-ui` and `scripts/report-render`.
|
||||
The UI module emits status TSV rows, and the renderer prints compact grouped
|
||||
summaries, detailed output, or JSON. Rows may include separate `fix`, `graph`,
|
||||
and `query` fields for Grafana-backed checks. Default doctor/status output is
|
||||
problems-first; pass `--all`, `--details`, `--json`, or `--verbose` when more
|
||||
context is needed.
|
||||
|
||||
`gitops-status`
|
||||
: Print a focused Argo CD status view with application sync/health, out-of-sync
|
||||
|
|
@ -265,23 +269,26 @@ Docker roots, Tailscale IPs, Gitea reachability, Pimox storage, and OCI edge SSH
|
|||
It checks Debian free disk, Docker and containerd, Raspberry Pi Docker
|
||||
writability, Pimox storage, OCI edge disk, and Pi-hole DNS.
|
||||
|
||||
`doctor-versions`
|
||||
: Check Kubernetes component version alignment.
|
||||
`doctor-versions [--all|--details|--verbose|--json]`
|
||||
: Check Kubernetes component version alignment. Default output shows only
|
||||
warnings and failures; `--verbose` prints the old version table.
|
||||
|
||||
`doctor-edge`
|
||||
`doctor-edge [--all|--details|--verbose|--json]`
|
||||
: Diagnose public edge routing, Traefik reachability, and Gitea edge exposure.
|
||||
Default output shows only warnings and failures.
|
||||
|
||||
`doctor-gitea`
|
||||
`doctor-gitea [--all|--details|--verbose|--json]`
|
||||
: Diagnose Debian Gitea container, local access, edge access, and repository
|
||||
health.
|
||||
health. Default output shows only warnings and failures.
|
||||
|
||||
`doctor-rpi`
|
||||
`doctor-rpi [--all|--details|--verbose|--json]`
|
||||
: Diagnose Raspberry Pi services, Pi-hole DNS, Unbound DNS, fallback DNS,
|
||||
Uptime Kuma, and Docker root state.
|
||||
Uptime Kuma, and Docker root state. Default output shows only warnings and
|
||||
failures.
|
||||
|
||||
`doctor-cluster`
|
||||
`doctor-cluster [--all|--details|--verbose|--json]`
|
||||
: Diagnose Kubernetes API, nodes, core pods, MetalLB, Traefik, app health, and
|
||||
Pimox worker tailnet egress.
|
||||
Pimox worker tailnet egress. Default output shows only warnings and failures.
|
||||
|
||||
`inventory-check`
|
||||
: Validate `homelab.yml` with a real YAML parser when available.
|
||||
|
|
@ -411,8 +418,8 @@ Defaults to `85`.
|
|||
full output is written to the log file.
|
||||
|
||||
`LAB_STATUS_DETAILS=true`
|
||||
: Append detailed host, Docker, Kubernetes, Pimox, RPi, Tailscale, and HTTP
|
||||
tables after the normal status cascade.
|
||||
: With `status --verbose`, append detailed host, Docker, Kubernetes, Pimox, RPi,
|
||||
Tailscale, and HTTP tables after the normal status cascade.
|
||||
|
||||
`LAB_STATUS_SHOW_OK_OUTPUT=true`
|
||||
: Show short command output for successful cascade checks. Failed and warning
|
||||
|
|
|
|||
249
jeannie
249
jeannie
|
|
@ -17,6 +17,9 @@ JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}"
|
|||
JEANNIE_STEP_INDEX=0
|
||||
JEANNIE_STEP_TOTAL=0
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source "${REPO_ROOT}/scripts/report-ui"
|
||||
|
||||
trap 'rm -f "${BUILDX_CONFIG}"' EXIT
|
||||
|
||||
load_homelab_inventory_defaults() {
|
||||
|
|
@ -661,8 +664,89 @@ doctor_versions_report() {
|
|||
return "${status}"
|
||||
}
|
||||
|
||||
doctor_versions_compact() {
|
||||
local api_version
|
||||
local api_minor
|
||||
local node_rows
|
||||
local kubelet_minor_count
|
||||
local containerd_major_count
|
||||
local mismatch_count
|
||||
local failures=0
|
||||
|
||||
ensure_python3
|
||||
report_ui_begin
|
||||
|
||||
if ! status_kubernetes_api_ok; then
|
||||
report_ui_emit fail "Kubernetes" "API" "not reachable" "kubectl readyz failed" "./jeannie start-cluster"
|
||||
report_ui_render "Doctor Versions"
|
||||
report_ui_cleanup
|
||||
return 1
|
||||
fi
|
||||
|
||||
api_version="$(
|
||||
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
|
||||
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
|
||||
)"
|
||||
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
|
||||
node_rows="$(
|
||||
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
|
||||
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
|
||||
)"
|
||||
|
||||
if [[ -z "${node_rows}" ]]; then
|
||||
report_ui_emit fail "Kubernetes" "Nodes" "none found" "kubectl returned no node rows" "./jeannie doctor-cluster"
|
||||
report_ui_render "Doctor Versions"
|
||||
report_ui_cleanup
|
||||
return 1
|
||||
fi
|
||||
|
||||
kubelet_minor_count="$(
|
||||
printf '%s\n' "${node_rows}" |
|
||||
awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
|
||||
sort -u |
|
||||
wc -l |
|
||||
tr -d ' '
|
||||
)"
|
||||
mismatch_count="$(
|
||||
printf '%s\n' "${node_rows}" |
|
||||
awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) count++ } END { print count + 0 }'
|
||||
)"
|
||||
containerd_major_count="$(
|
||||
printf '%s\n' "${node_rows}" |
|
||||
awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
|
||||
sort -u |
|
||||
wc -l |
|
||||
tr -d ' '
|
||||
)"
|
||||
|
||||
report_ui_emit ok "Kubernetes" "API version" "${api_version}"
|
||||
if [[ "${kubelet_minor_count}" != "1" || "${mismatch_count}" != "0" ]]; then
|
||||
report_ui_emit fail "Kubernetes" "Kubelet versions" "${mismatch_count} node(s) differ from ${api_minor}" "unique kubelet minors=${kubelet_minor_count}" "./jeannie doctor-versions --verbose"
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
report_ui_emit ok "Kubernetes" "Kubelet versions" "aligned to ${api_minor}"
|
||||
fi
|
||||
if [[ "${containerd_major_count}" -gt 1 ]]; then
|
||||
report_ui_emit warn "Runtime" "Containerd versions" "mixed major versions" "unique majors=${containerd_major_count}" "./jeannie doctor-versions --verbose"
|
||||
else
|
||||
report_ui_emit ok "Runtime" "Containerd versions" "single major version"
|
||||
fi
|
||||
|
||||
report_ui_render "Doctor Versions"
|
||||
report_ui_cleanup
|
||||
return "${failures}"
|
||||
}
|
||||
|
||||
doctor_versions() {
|
||||
doctor_versions_report
|
||||
require_debian_server "doctor-versions"
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
if [[ "${REPORT_UI_VERBOSE}" == "true" ]]; then
|
||||
doctor_versions_report
|
||||
exit "$?"
|
||||
fi
|
||||
doctor_versions_compact
|
||||
exit "$?"
|
||||
}
|
||||
|
||||
|
|
@ -4597,6 +4681,10 @@ ensure_existing_cluster_started_for_up() {
|
|||
}
|
||||
|
||||
status_section() {
|
||||
STATUS_CURRENT_SECTION="$1"
|
||||
if truthy "${JEANNIE_REPORT_MODE:-false}"; then
|
||||
return 0
|
||||
fi
|
||||
printf '\n== %s ==\n' "$1"
|
||||
}
|
||||
|
||||
|
|
@ -4732,6 +4820,11 @@ status_cascade_check() {
|
|||
local output
|
||||
shift
|
||||
|
||||
if truthy "${JEANNIE_REPORT_MODE:-false}"; then
|
||||
report_ui_check fail "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@" || return 1
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%-36s ' "${description}"
|
||||
if output="$("$@" 2>&1)"; then
|
||||
printf 'ok\n'
|
||||
|
|
@ -4753,6 +4846,11 @@ status_cascade_warn() {
|
|||
local output
|
||||
shift
|
||||
|
||||
if truthy "${JEANNIE_REPORT_MODE:-false}"; then
|
||||
report_ui_check warn "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@"
|
||||
return 0
|
||||
fi
|
||||
|
||||
printf '%-36s ' "${description}"
|
||||
if output="$("$@" 2>&1)"; then
|
||||
printf 'ok\n'
|
||||
|
|
@ -5024,6 +5122,10 @@ status_cascade_report() {
|
|||
status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1))
|
||||
status_cascade_check "Gitea public route" status_gitea_public_ok || failures=$((failures + 1))
|
||||
|
||||
if truthy "${JEANNIE_REPORT_MODE:-false}"; then
|
||||
return "${failures}"
|
||||
fi
|
||||
|
||||
if ((failures > 0)); then
|
||||
printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2
|
||||
return 1
|
||||
|
|
@ -5336,9 +5438,21 @@ status_report() {
|
|||
|
||||
require_debian_server "status"
|
||||
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then
|
||||
report_ui_begin
|
||||
JEANNIE_REPORT_MODE=true status_cascade_report || cascade_status=$?
|
||||
report_ui_render "Jeannie Status"
|
||||
report_ui_cleanup
|
||||
return "${cascade_status}"
|
||||
fi
|
||||
|
||||
status_cascade_report || cascade_status=$?
|
||||
|
||||
if ! truthy "${LAB_STATUS_DETAILS:-false}"; then
|
||||
if ! truthy "${LAB_STATUS_DETAILS:-false}" && [[ "${REPORT_UI_DETAILS}" != "true" ]]; then
|
||||
return "${cascade_status}"
|
||||
fi
|
||||
|
||||
|
|
@ -5385,14 +5499,61 @@ status_report() {
|
|||
return "${cascade_status}"
|
||||
}
|
||||
|
||||
check_edge_traefik_backend() {
|
||||
local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
|
||||
local edge_user="${LAB_EDGE_USER:-ubuntu}"
|
||||
local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}"
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
|
||||
"curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${traefik_ip}:80/' | grep -Eq '^[234]'"
|
||||
}
|
||||
|
||||
check_edge_gitea_backend() {
|
||||
local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
|
||||
local edge_user="${LAB_EDGE_USER:-ubuntu}"
|
||||
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}"
|
||||
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
|
||||
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
|
||||
"curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${gitea_ts_ip}:${gitea_http_port}/' | grep -Eq '^[234]'"
|
||||
}
|
||||
|
||||
check_gitea_git_remote() {
|
||||
git ls-remote gitea main >/dev/null
|
||||
}
|
||||
|
||||
check_gitea_ssh() {
|
||||
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
|
||||
local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
|
||||
|
||||
ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" 2>&1 |
|
||||
grep -Eiq 'successfully authenticated|Hi |shell access is disabled'
|
||||
}
|
||||
|
||||
doctor_edge() {
|
||||
local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
|
||||
local edge_user="${LAB_EDGE_USER:-ubuntu}"
|
||||
local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}"
|
||||
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}"
|
||||
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
|
||||
local failures=0
|
||||
|
||||
require_debian_server "doctor-edge"
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then
|
||||
report_ui_begin
|
||||
report_ui_check fail "Public URLs" "Website public" "./jeannie doctor-edge --verbose" "" "" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1))
|
||||
report_ui_check fail "Public URLs" "Gitea public" "./jeannie doctor-gitea" "" "" status_gitea_public_ok || failures=$((failures + 1))
|
||||
report_ui_check fail "Edge" "OCI edge SSH" "./jeannie preflight" "" "" check_edge_ssh || failures=$((failures + 1))
|
||||
report_ui_check fail "Backends" "Traefik from edge" "./jeannie doctor-cluster" "" "" check_edge_traefik_backend || failures=$((failures + 1))
|
||||
report_ui_check fail "Backends" "Gitea from edge" "./jeannie doctor-gitea" "" "" check_edge_gitea_backend || failures=$((failures + 1))
|
||||
report_ui_render "Doctor Edge"
|
||||
report_ui_cleanup
|
||||
return "${failures}"
|
||||
fi
|
||||
|
||||
status_section "Doctor Edge"
|
||||
status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/"
|
||||
|
|
@ -5438,8 +5599,25 @@ doctor_gitea() {
|
|||
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
|
||||
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
|
||||
local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
|
||||
local failures=0
|
||||
|
||||
require_debian_server "doctor-gitea"
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then
|
||||
report_ui_begin
|
||||
report_ui_check fail "Local Gitea" "Container running" "./jeannie deploy-gitea" "" "" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1))
|
||||
report_ui_check fail "Local Gitea" "Local HTTP" "./jeannie deploy-gitea" "" "" status_http_ok "http://127.0.0.1:${gitea_http_port}/" || failures=$((failures + 1))
|
||||
report_ui_check fail "Public Gitea" "Public route" "./jeannie doctor-edge" "" "" status_gitea_public_ok || failures=$((failures + 1))
|
||||
report_ui_check warn "Git" "Gitea remote" "git remote -v" "" "" check_gitea_git_remote
|
||||
report_ui_check warn "Git" "Gitea SSH" "Check Gitea SSH key and remote URL." "" "" check_gitea_ssh
|
||||
report_ui_check fail "Edge" "Edge backend reachability" "./jeannie doctor-edge" "" "" check_edge_gitea_backend || failures=$((failures + 1))
|
||||
report_ui_render "Doctor Gitea"
|
||||
report_ui_cleanup
|
||||
return "${failures}"
|
||||
fi
|
||||
|
||||
status_section "Doctor Gitea"
|
||||
status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/"
|
||||
|
|
@ -5478,7 +5656,24 @@ EOF
|
|||
}
|
||||
|
||||
doctor_rpi() {
|
||||
local failures=0
|
||||
|
||||
require_debian_server "doctor-rpi"
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then
|
||||
report_ui_begin
|
||||
report_ui_check fail "RPi" "SSH" "./jeannie rpi-services" "" "" check_rpi_ssh || failures=$((failures + 1))
|
||||
report_ui_check fail "RPi" "Docker root state" "./jeannie rpi-services" "" "" check_rpi_docker_root_state || failures=$((failures + 1))
|
||||
report_ui_check fail "DNS" "Pi-hole DNS" "./jeannie rpi-services" "" "" check_pihole_dns_query || failures=$((failures + 1))
|
||||
report_ui_check warn "DNS" "RPi Tailscale IP" "Check tailscaled on RPi." "" "" check_rpi_tailscale_ip
|
||||
report_ui_check warn "Services" "Uptime Kuma HTTP" "./jeannie rpi-services" "" "" status_rpi_uptime_kuma_ok
|
||||
report_ui_render "Doctor RPi"
|
||||
report_ui_cleanup
|
||||
return "${failures}"
|
||||
fi
|
||||
|
||||
status_rpi_services
|
||||
|
||||
|
|
@ -5502,7 +5697,28 @@ EOF
|
|||
}
|
||||
|
||||
doctor_cluster() {
|
||||
local failures=0
|
||||
|
||||
require_debian_server "doctor-cluster"
|
||||
if ! report_ui_parse_args "$@"; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then
|
||||
report_ui_begin
|
||||
report_ui_check fail "Host Runtime" "kubelet" "./jeannie start-cluster" "" "" check_systemd_active kubelet || failures=$((failures + 1))
|
||||
report_ui_check fail "Host Runtime" "containerd" "sudo systemctl restart containerd" "" "" check_systemd_active containerd || failures=$((failures + 1))
|
||||
report_ui_check warn "Host Runtime" "docker" "sudo systemctl restart docker" "" "" check_systemd_active docker
|
||||
report_ui_check fail "Kubernetes" "API" "./jeannie start-cluster" "" "" status_kubernetes_api_ok || failures=$((failures + 1))
|
||||
report_ui_check fail "Kubernetes" "Nodes Ready" "kubectl get nodes -o wide" "" "" status_kubernetes_nodes_ready || failures=$((failures + 1))
|
||||
report_ui_check warn "Kubernetes" "Problem pods" "kubectl get pods -A -o wide" "" "" status_no_problem_pods
|
||||
report_ui_check fail "Pimox" "Workers running" "./jeannie start-cluster" "" "" status_pimox_workers_running || failures=$((failures + 1))
|
||||
report_ui_check fail "Pimox" "Worker tailnet egress" "./jeannie workers tailnet" "" "" check_pimox_worker_tailnet_egress || failures=$((failures + 1))
|
||||
report_ui_check fail "Networking" "Traefik LoadBalancer" "./jeannie doctor-edge" "" "" status_http_reachable "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" || failures=$((failures + 1))
|
||||
report_ui_render "Doctor Cluster"
|
||||
report_ui_cleanup
|
||||
return "${failures}"
|
||||
fi
|
||||
|
||||
status_section "Doctor Cluster"
|
||||
status_systemd_units kubelet containerd docker
|
||||
|
|
@ -6200,10 +6416,11 @@ Cluster Lifecycle
|
|||
workers <list|tailnet|start|stop|drain|uncordon|recreate-plan|rebalance>
|
||||
Manage Pimox/Kubernetes workers.
|
||||
move-prometheus-stack-workers Move monitoring workloads to worker nodes.
|
||||
doctor-versions Check Kubernetes/container tooling versions.
|
||||
doctor-versions [--all|--details|--verbose|--json]
|
||||
Check Kubernetes/container tooling versions.
|
||||
|
||||
Operator View And Reports
|
||||
status Cascading health check from host to public URLs.
|
||||
status [--all|--details|--verbose|--json] Problems-first cascade from host to public URLs.
|
||||
scorecard Compact pass/warn/fail operator scorecard.
|
||||
capacity Compact capacity and placement report.
|
||||
capacity-advisor Decide whether to add VMs or fix placement/resources.
|
||||
|
|
@ -6236,10 +6453,14 @@ Recovery And State
|
|||
state-backup Back up local OpenTofu state files.
|
||||
|
||||
Focused Doctors
|
||||
doctor-edge Diagnose edge routing and public exposure.
|
||||
doctor-gitea Diagnose Gitea local/public access.
|
||||
doctor-rpi Diagnose RPi services and DNS.
|
||||
doctor-cluster Diagnose Kubernetes cluster health.
|
||||
doctor-edge [--all|--details|--verbose|--json]
|
||||
Diagnose edge routing and public exposure.
|
||||
doctor-gitea [--all|--details|--verbose|--json]
|
||||
Diagnose Gitea local/public access.
|
||||
doctor-rpi [--all|--details|--verbose|--json]
|
||||
Diagnose RPi services and DNS.
|
||||
doctor-cluster [--all|--details|--verbose|--json]
|
||||
Diagnose Kubernetes cluster health.
|
||||
|
||||
Access, Secrets, And Policy
|
||||
access-audit Audit SSH, Gitea, Kubernetes, and Tailscale access.
|
||||
|
|
@ -6304,7 +6525,7 @@ case "${1:-}" in
|
|||
start_cluster
|
||||
;;
|
||||
status)
|
||||
status_report
|
||||
status_report "${@:2}"
|
||||
;;
|
||||
capacity)
|
||||
capacity_report "$@"
|
||||
|
|
@ -6435,19 +6656,19 @@ case "${1:-}" in
|
|||
move_prometheus_stack_workers
|
||||
;;
|
||||
doctor-versions)
|
||||
doctor_versions
|
||||
doctor_versions "${@:2}"
|
||||
;;
|
||||
doctor-edge)
|
||||
doctor_edge
|
||||
doctor_edge "${@:2}"
|
||||
;;
|
||||
doctor-gitea)
|
||||
doctor_gitea
|
||||
doctor_gitea "${@:2}"
|
||||
;;
|
||||
doctor-rpi)
|
||||
doctor_rpi
|
||||
doctor_rpi "${@:2}"
|
||||
;;
|
||||
doctor-cluster)
|
||||
doctor_cluster
|
||||
doctor_cluster "${@:2}"
|
||||
;;
|
||||
preflight)
|
||||
homelab_preflight
|
||||
|
|
|
|||
|
|
@ -121,6 +121,17 @@ print(
|
|||
f"skip={counts.get('skip', 0)}"
|
||||
)
|
||||
|
||||
if not rows:
|
||||
if only == "problems":
|
||||
print("\nNo failing or warning items.")
|
||||
elif only == "failures":
|
||||
print("\nNo failing items.")
|
||||
elif only == "warnings":
|
||||
print("\nNo warning items.")
|
||||
else:
|
||||
print("\nNo report rows.")
|
||||
raise SystemExit(0)
|
||||
|
||||
groups = OrderedDict()
|
||||
for row in rows:
|
||||
groups.setdefault(row["area"], []).append(row)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
REPORT_UI_ONLY="${REPORT_UI_ONLY:-problems}"
|
||||
REPORT_UI_DETAILS=false
|
||||
REPORT_UI_JSON=false
|
||||
# shellcheck disable=SC2034
|
||||
REPORT_UI_VERBOSE=false
|
||||
REPORT_UI_ROWS_FILE=""
|
||||
|
||||
report_ui_usage_flags() {
|
||||
cat <<'EOF'
|
||||
Report UI flags:
|
||||
--all Show ok, skip, warning, and failing items.
|
||||
--details Show the compact report plus detail fields.
|
||||
--json Print structured JSON.
|
||||
--verbose Use the older full diagnostic output when supported.
|
||||
EOF
|
||||
}
|
||||
|
||||
report_ui_parse_args() {
|
||||
REPORT_UI_ONLY="${REPORT_UI_ONLY:-problems}"
|
||||
REPORT_UI_DETAILS=false
|
||||
REPORT_UI_JSON=false
|
||||
REPORT_UI_VERBOSE=false
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--all)
|
||||
REPORT_UI_ONLY="all"
|
||||
shift
|
||||
;;
|
||||
--details)
|
||||
REPORT_UI_DETAILS=true
|
||||
shift
|
||||
;;
|
||||
--json)
|
||||
REPORT_UI_JSON=true
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
# shellcheck disable=SC2034
|
||||
REPORT_UI_VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
report_ui_usage_flags
|
||||
return 2
|
||||
;;
|
||||
*)
|
||||
echo "unknown report option: $1" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
report_ui_begin() {
|
||||
REPORT_UI_ROWS_FILE="$(mktemp "${TMPDIR:-/tmp}/jeannie-report-rows.XXXXXX")"
|
||||
}
|
||||
|
||||
report_ui_cleanup() {
|
||||
if [[ -n "${REPORT_UI_ROWS_FILE:-}" && -f "${REPORT_UI_ROWS_FILE}" ]]; then
|
||||
rm -f "${REPORT_UI_ROWS_FILE}"
|
||||
fi
|
||||
REPORT_UI_ROWS_FILE=""
|
||||
}
|
||||
|
||||
report_ui_one_line() {
|
||||
local value="${1:-}"
|
||||
value="${value//$'\t'/ }"
|
||||
value="${value//$'\r'/ }"
|
||||
value="${value//$'\n'/ | }"
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
report_ui_first_line() {
|
||||
awk 'NF && $0 !~ /^\+/ { print; exit }'
|
||||
}
|
||||
|
||||
report_ui_trim_detail() {
|
||||
sed -n '1,20p'
|
||||
}
|
||||
|
||||
report_ui_emit() {
|
||||
local status="$1"
|
||||
local area="$2"
|
||||
local check="$3"
|
||||
local summary="$4"
|
||||
local detail="${5:-}"
|
||||
local fix="${6:-}"
|
||||
local graph="${7:-}"
|
||||
local query="${8:-}"
|
||||
|
||||
if [[ -z "${REPORT_UI_ROWS_FILE:-}" ]]; then
|
||||
echo "report_ui_begin was not called" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$(report_ui_one_line "${status}")" \
|
||||
"$(report_ui_one_line "${area}")" \
|
||||
"$(report_ui_one_line "${check}")" \
|
||||
"$(report_ui_one_line "${summary}")" \
|
||||
"$(report_ui_one_line "${detail}")" \
|
||||
"$(report_ui_one_line "${fix}")" \
|
||||
"$(report_ui_one_line "${graph}")" \
|
||||
"$(report_ui_one_line "${query}")" >>"${REPORT_UI_ROWS_FILE}"
|
||||
}
|
||||
|
||||
report_ui_check() {
|
||||
local severity="$1"
|
||||
local area="$2"
|
||||
local check="$3"
|
||||
local fix="${4:-}"
|
||||
local graph="${5:-}"
|
||||
local query="${6:-}"
|
||||
shift 6
|
||||
local output
|
||||
local status
|
||||
local summary
|
||||
local detail
|
||||
|
||||
set +e
|
||||
output="$("$@" 2>&1)"
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
report_ui_emit ok "${area}" "${check}" "ok" "${output}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
summary="$(printf '%s\n' "${output}" | report_ui_first_line)"
|
||||
if [[ -z "${summary}" ]]; then
|
||||
summary="check exited with status ${status}"
|
||||
fi
|
||||
detail="$(printf '%s\n' "${output}" | report_ui_trim_detail)"
|
||||
if [[ "${severity}" == "fail" ]]; then
|
||||
report_ui_emit fail "${area}" "${check}" "${summary}" "${detail}" "${fix}" "${graph}" "${query}"
|
||||
return 1
|
||||
fi
|
||||
report_ui_emit warn "${area}" "${check}" "${summary}" "${detail}" "${fix}" "${graph}" "${query}"
|
||||
return 0
|
||||
}
|
||||
|
||||
report_ui_render() {
|
||||
local title="$1"
|
||||
local -a args=(--title "${title}" --only "${REPORT_UI_ONLY}")
|
||||
|
||||
if [[ "${REPORT_UI_DETAILS}" == "true" ]]; then
|
||||
args+=(--details)
|
||||
fi
|
||||
if [[ "${REPORT_UI_JSON}" == "true" ]]; then
|
||||
args+=(--json)
|
||||
fi
|
||||
|
||||
"${REPO_ROOT}/scripts/report-render" "${args[@]}" <"${REPORT_UI_ROWS_FILE}"
|
||||
}
|
||||
Loading…
Reference in New Issue