diff --git a/README.md b/README.md index 29edcda..799b97d 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks, Kubernetes health when the API is reachable, Pimox worker VM status, RPi services, Tailscale, and key local/public HTTP endpoints. +Focused doctor commands run narrower read-only checks and print the most likely +next step: + +```bash +./jeannie doctor-edge +./jeannie doctor-gitea +./jeannie doctor-rpi +./jeannie doctor-cluster +``` + ## RPi Services The Raspberry Pi runs lightweight always-on services outside Kubernetes from @@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in `infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. +`./jeannie status` and `./jeannie doctor-rpi` test Pi-hole DNS, +direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and +whether Docker is currently using the NVMe path or `/var/lib/docker`. ## Adding Nodes diff --git a/README.md.tmpl b/README.md.tmpl index 9f51770..e302282 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks, Kubernetes health when the API is reachable, Pimox worker VM status, RPi services, Tailscale, and key local/public HTTP endpoints. +Focused doctor commands run narrower read-only checks and print the most likely +next step: + +```bash +./{{ main_script }} doctor-edge +./{{ main_script }} doctor-gitea +./{{ main_script }} doctor-rpi +./{{ main_script }} doctor-cluster +``` + ## RPi Services The Raspberry Pi runs lightweight always-on services outside Kubernetes from @@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in `infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided, the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`. +`./{{ main_script }} status` and `./{{ main_script }} doctor-rpi` test Pi-hole DNS, +direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and +whether Docker is currently using the NVMe path or `/var/lib/docker`. ## Adding Nodes diff --git a/jeannie b/jeannie index 30686ce..8bb16f7 100755 --- a/jeannie +++ b/jeannie @@ -3620,13 +3620,97 @@ status_rpi_services() { local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" + local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" + local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" + local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}" + local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" status_section "RPi Services" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " +set +e +dns_query() { + server=\"\$1\" + port=\"\$2\" + name=\"\$3\" + python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY' +import random +import socket +import struct +import sys + +server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.') +query_id = random.randrange(0, 65536) +packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0) +for label in name.split('.'): + packet += bytes([len(label)]) + label.encode('ascii') +packet += b'\x00' + struct.pack('!HH', 1, 1) +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.settimeout(5) +sock.sendto(packet, (server, port)) +data, _ = sock.recvfrom(512) +if len(data) < 12: + raise SystemExit('short DNS response') +response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12]) +rcode = flags & 0x000f +if response_id != query_id: + raise SystemExit('mismatched DNS response id') +if rcode != 0: + raise SystemExit(f'DNS rcode {rcode}') +if an < 1: + raise SystemExit('DNS response had no answers') +print(f'ok answers={an}') +PY +} +check() { + label=\"\$1\" + shift + printf '%-34s ' \"\$label\" + output=\"\$("\$@" 2>&1)\" + status=\$? + if [ \"\$status\" -eq 0 ]; then + printf 'ok' + if [ -n \"\$output\" ]; then + printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\" + fi + printf '\n' + else + printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\" + fi + return 0 +} +docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\" +printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\" +if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then + if mountpoint -q '${docker_nvme_root}'; then + printf '%-34s ok - nvme mount active\n' 'Docker root class' + else + printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class' + fi +elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then + printf '%-34s ok - fallback root active\n' 'Docker root class' +else + printf '%-34s warn - unexpected docker root\n' 'Docker root class' +fi if [ -d '${install_dir}' ]; then cd '${install_dir}' - sudo docker info --format 'DockerRootDir={{.DockerRootDir}}' 2>/dev/null || true sudo docker compose ps || true + unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\" + check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com + if [ -n \"\$unbound_ip\" ]; then + check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com + else + printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query' + fi + upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\" + if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then + printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config' + else + printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config' + fi + check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com + check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com + check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\" else echo 'RPi services install directory is missing: ${install_dir}' fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" @@ -3676,6 +3760,120 @@ status_report() { status_http "arr prowlarr" "http://127.0.0.1:9696/" } +doctor_edge() { + local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local traefik_ip="${LAB_TRAEFIK_LB_IP:-192.168.100.240}" + local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}" + local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" + + require_debian_server "doctor-edge" + + status_section "Doctor Edge" + status_http "website public" "https://lab2025.duckdns.org/" + status_http "gitea public" "https://lab2025.duckdns.org/git/" + + printf '\n-- edge backend reachability\n' + if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " +set +e +printf '%-28s ' 'traefik backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable +printf '%-28s ' 'gitea backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable +printf '\n-- tailscale\n' +tailscale status || true +"; then + printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}" + fi + + cat <<'EOF' + +Next steps: +- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster. +- If only Gitea is unreachable, run ./jeannie doctor-gitea. +- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge. +- Runbook: docs/runbooks/edge-failures.md +EOF +} + +doctor_gitea() { + local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}" + local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" + local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}" + + require_debian_server "doctor-gitea" + + status_section "Doctor Gitea" + status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/" + status_http "gitea public" "https://lab2025.duckdns.org/git/" + status_compose_stack "gitea" "/data/homelab-gitea" + + status_run "gitea git remote" git ls-remote gitea main + + printf '\n-- gitea SSH\n' + ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" || true + + printf '\n-- edge to gitea backend\n' + if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "curl -I --max-time 5 'http://${gitea_ts_ip}:${gitea_http_port}/'"; then + printf 'edge host cannot reach Gitea backend %s:%s\n' "${gitea_ts_ip}" "${gitea_http_port}" + fi + + cat <<'EOF' + +Next steps: +- If local Gitea is down, run ./jeannie deploy-gitea. +- If local works but public /git/ fails, check Tailscale and bootstrap/edge. +- If Git SSH fails, check the gitea remote and registered SSH key. +- Runbook: docs/runbooks/gitea-failures.md +EOF +} + +doctor_rpi() { + require_debian_server "doctor-rpi" + + status_rpi_services + + cat <<'EOF' + +Next steps: +- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check. +- If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks. +- If Pi-hole DNS fails, clients using the RPi as DNS will be affected. +- Reapply the stack with ./jeannie rpi-services. +EOF +} + +doctor_cluster() { + require_debian_server "doctor-cluster" + + status_section "Doctor Cluster" + status_systemd_units kubelet containerd docker + status_kubernetes + status_pimox_workers + status_http "traefik lb" "http://192.168.100.240/" + + printf '\n-- local CRI containers\n' + if command -v crictl >/dev/null 2>&1; then + sudo crictl ps -a | head -50 || true + else + printf 'crictl not installed\n' + fi + + cat <<'EOF' + +Next steps: +- If API server is unreachable after a stop, run ./jeannie start-cluster. +- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node. +- If Pimox workers are stopped, start them with ./jeannie start-cluster. +- If kubelet versions drift, run ./jeannie doctor-versions. +- Runbook: docs/runbooks/cluster-stop-start-failures.md +EOF +} + nuke() { local worker_ssh_targets local worker_targets @@ -3866,6 +4064,18 @@ case "${1:-}" in doctor-versions) doctor_versions ;; + doctor-edge) + doctor_edge + ;; + doctor-gitea) + doctor_gitea + ;; + doctor-rpi) + doctor_rpi + ;; + doctor-cluster) + doctor_cluster + ;; openwrt) openwrt ;; @@ -3873,7 +4083,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|openwrt|nuke}" exit 1 ;; esac