Add focused homelab doctor checks

README.md

@@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks,
 Kubernetes health when the API is reachable, Pimox worker VM status, RPi
 services, Tailscale, and key local/public HTTP endpoints.
 
+Focused doctor commands run narrower read-only checks and print the most likely
+next step:
+
+```bash
+./jeannie doctor-edge
+./jeannie doctor-gitea
+./jeannie doctor-rpi
+./jeannie doctor-cluster
+```
+
 ## RPi Services
 
 The Raspberry Pi runs lightweight always-on services outside Kubernetes from
@@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in
 `infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without
 deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided,
 the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`.
+`./jeannie status` and `./jeannie doctor-rpi` test Pi-hole DNS,
+direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
+whether Docker is currently using the NVMe path or `/var/lib/docker`.
 
 ## Adding Nodes
 

README.md.tmpl

@@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks,
 Kubernetes health when the API is reachable, Pimox worker VM status, RPi
 services, Tailscale, and key local/public HTTP endpoints.
 
+Focused doctor commands run narrower read-only checks and print the most likely
+next step:
+
+```bash
+./{{ main_script }} doctor-edge
+./{{ main_script }} doctor-gitea
+./{{ main_script }} doctor-rpi
+./{{ main_script }} doctor-cluster
+```
+
 ## RPi Services
 
 The Raspberry Pi runs lightweight always-on services outside Kubernetes from
@@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in
 `infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without
 deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided,
 the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`.
+`./{{ main_script }} status` and `./{{ main_script }} doctor-rpi` test Pi-hole DNS,
+direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
+whether Docker is currently using the NVMe path or `/var/lib/docker`.
 
 ## Adding Nodes
 

jeannie

@@ -3620,13 +3620,97 @@ status_rpi_services() {
     local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
     local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
     local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
+    local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
+    local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
+    local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}"
+    local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}"
+    local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}"
 
     status_section "RPi Services"
     ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
+set +e
+dns_query() {
+  server=\"\$1\"
+  port=\"\$2\"
+  name=\"\$3\"
+  python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY'
+import random
+import socket
+import struct
+import sys
+
+server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.')
+query_id = random.randrange(0, 65536)
+packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0)
+for label in name.split('.'):
+    packet += bytes([len(label)]) + label.encode('ascii')
+packet += b'\x00' + struct.pack('!HH', 1, 1)
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.settimeout(5)
+sock.sendto(packet, (server, port))
+data, _ = sock.recvfrom(512)
+if len(data) < 12:
+    raise SystemExit('short DNS response')
+response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12])
+rcode = flags & 0x000f
+if response_id != query_id:
+    raise SystemExit('mismatched DNS response id')
+if rcode != 0:
+    raise SystemExit(f'DNS rcode {rcode}')
+if an < 1:
+    raise SystemExit('DNS response had no answers')
+print(f'ok answers={an}')
+PY
+}
+check() {
+  label=\"\$1\"
+  shift
+  printf '%-34s ' \"\$label\"
+  output=\"\$("\$@" 2>&1)\"
+  status=\$?
+  if [ \"\$status\" -eq 0 ]; then
+    printf 'ok'
+    if [ -n \"\$output\" ]; then
+      printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\"
+    fi
+    printf '\n'
+  else
+    printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\"
+  fi
+  return 0
+}
+docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\"
+printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\"
+if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then
+  if mountpoint -q '${docker_nvme_root}'; then
+    printf '%-34s ok - nvme mount active\n' 'Docker root class'
+  else
+    printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class'
+  fi
+elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then
+  printf '%-34s ok - fallback root active\n' 'Docker root class'
+else
+  printf '%-34s warn - unexpected docker root\n' 'Docker root class'
+fi
 if [ -d '${install_dir}' ]; then
   cd '${install_dir}'
-  sudo docker info --format 'DockerRootDir={{.DockerRootDir}}' 2>/dev/null || true
   sudo docker compose ps || true
+  unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\"
+  check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com
+  if [ -n \"\$unbound_ip\" ]; then
+    check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com
+  else
+    printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query'
+  fi
+  upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\"
+  if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then
+    printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config'
+  else
+    printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config'
+  fi
+  check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com
+  check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com
+  check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\"
 else
   echo 'RPi services install directory is missing: ${install_dir}'
 fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}"
@@ -3676,6 +3760,120 @@ status_report() {
     status_http "arr prowlarr" "http://127.0.0.1:9696/"
 }
 
+doctor_edge() {
+    local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
+    local edge_user="${LAB_EDGE_USER:-ubuntu}"
+    local traefik_ip="${LAB_TRAEFIK_LB_IP:-192.168.100.240}"
+    local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
+    local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
+
+    require_debian_server "doctor-edge"
+
+    status_section "Doctor Edge"
+    status_http "website public" "https://lab2025.duckdns.org/"
+    status_http "gitea public" "https://lab2025.duckdns.org/git/"
+
+    printf '\n-- edge backend reachability\n'
+    if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "
+set +e
+printf '%-28s ' 'traefik backend'
+curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable
+printf '%-28s ' 'gitea backend'
+curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable
+printf '\n-- tailscale\n'
+tailscale status || true
+"; then
+        printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}"
+    fi
+
+    cat <<'EOF'
+
+Next steps:
+- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster.
+- If only Gitea is unreachable, run ./jeannie doctor-gitea.
+- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge.
+- Runbook: docs/runbooks/edge-failures.md
+EOF
+}
+
+doctor_gitea() {
+    local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
+    local edge_user="${LAB_EDGE_USER:-ubuntu}"
+    local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
+    local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
+    local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
+    local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
+
+    require_debian_server "doctor-gitea"
+
+    status_section "Doctor Gitea"
+    status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/"
+    status_http "gitea public" "https://lab2025.duckdns.org/git/"
+    status_compose_stack "gitea" "/data/homelab-gitea"
+
+    status_run "gitea git remote" git ls-remote gitea main
+
+    printf '\n-- gitea SSH\n'
+    ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" || true
+
+    printf '\n-- edge to gitea backend\n'
+    if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
+        "curl -I --max-time 5 'http://${gitea_ts_ip}:${gitea_http_port}/'"; then
+        printf 'edge host cannot reach Gitea backend %s:%s\n' "${gitea_ts_ip}" "${gitea_http_port}"
+    fi
+
+    cat <<'EOF'
+
+Next steps:
+- If local Gitea is down, run ./jeannie deploy-gitea.
+- If local works but public /git/ fails, check Tailscale and bootstrap/edge.
+- If Git SSH fails, check the gitea remote and registered SSH key.
+- Runbook: docs/runbooks/gitea-failures.md
+EOF
+}
+
+doctor_rpi() {
+    require_debian_server "doctor-rpi"
+
+    status_rpi_services
+
+    cat <<'EOF'
+
+Next steps:
+- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check.
+- If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks.
+- If Pi-hole DNS fails, clients using the RPi as DNS will be affected.
+- Reapply the stack with ./jeannie rpi-services.
+EOF
+}
+
+doctor_cluster() {
+    require_debian_server "doctor-cluster"
+
+    status_section "Doctor Cluster"
+    status_systemd_units kubelet containerd docker
+    status_kubernetes
+    status_pimox_workers
+    status_http "traefik lb" "http://192.168.100.240/"
+
+    printf '\n-- local CRI containers\n'
+    if command -v crictl >/dev/null 2>&1; then
+        sudo crictl ps -a | head -50 || true
+    else
+        printf 'crictl not installed\n'
+    fi
+
+    cat <<'EOF'
+
+Next steps:
+- If API server is unreachable after a stop, run ./jeannie start-cluster.
+- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node.
+- If Pimox workers are stopped, start them with ./jeannie start-cluster.
+- If kubelet versions drift, run ./jeannie doctor-versions.
+- Runbook: docs/runbooks/cluster-stop-start-failures.md
+EOF
+}
+
 nuke() {
     local worker_ssh_targets
     local worker_targets
@@ -3866,6 +4064,18 @@ case "${1:-}" in
     doctor-versions)
         doctor_versions
         ;;
+    doctor-edge)
+        doctor_edge
+        ;;
+    doctor-gitea)
+        doctor_gitea
+        ;;
+    doctor-rpi)
+        doctor_rpi
+        ;;
+    doctor-cluster)
+        doctor_cluster
+        ;;
     openwrt)
         openwrt
         ;;
@@ -3873,7 +4083,7 @@ case "${1:-}" in
         nuke
         ;;
     *)
-        echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}"
+        echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|openwrt|nuke}"
         exit 1
         ;;
 esac