jv
/
my-homelab-configs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add focused homelab doctor checks

This commit is contained in:
juvdiaz 2026-06-27 16:35:30 -06:00
parent d5ff049cae
commit 7beb0778c9
3 changed files with 238 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks,
Kubernetes health when the API is reachable, Pimox worker VM status, RPi
services, Tailscale, and key local/public HTTP endpoints.
Focused doctor commands run narrower read-only checks and print the most likely
next step:
```bash
./jeannie doctor-edge
./jeannie doctor-gitea
./jeannie doctor-rpi
./jeannie doctor-cluster
```
## RPi Services
The Raspberry Pi runs lightweight always-on services outside Kubernetes from
@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in
`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without
deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided,
the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`.
`./jeannie status` and `./jeannie doctor-rpi` test Pi-hole DNS,
direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
whether Docker is currently using the NVMe path or `/var/lib/docker`.
## Adding Nodes

13
README.md.tmpl
View File

@ -245,6 +245,16 @@ It reports host memory/disk, systemd services, Docker Compose stacks,
Kubernetes health when the API is reachable, Pimox worker VM status, RPi
services, Tailscale, and key local/public HTTP endpoints.
Focused doctor commands run narrower read-only checks and print the most likely
next step:
```bash
./{{ main_script }} doctor-edge
./{{ main_script }} doctor-gitea
./{{ main_script }} doctor-rpi
./{{ main_script }} doctor-cluster
```
## RPi Services
The Raspberry Pi runs lightweight always-on services outside Kubernetes from
@ -272,6 +282,9 @@ continue resolving if the Unbound container stops. Managed adlists live in
`infra/rpi-services/adlists.txt`; the bootstrap inserts those lists without
deleting manually added Pi-hole lists. If `PIHOLE_WEBPASSWORD` is not provided,
the bootstrap generates one and keeps it in `/opt/homelab-rpi-services/.env`.
`./{{ main_script }} status` and `./{{ main_script }} doctor-rpi` test Pi-hole DNS,
direct Unbound DNS, public fallback resolver readiness, Uptime Kuma HTTP, and
whether Docker is currently using the NVMe path or `/var/lib/docker`.
## Adding Nodes

214
jeannie
View File

@ -3620,13 +3620,97 @@ status_rpi_services() {
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}"
local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}"
local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}"
status_section "RPi Services"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
set +e
dns_query() {
server=\"\$1\"
port=\"\$2\"
name=\"\$3\"
python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY'
import random
import socket
import struct
import sys
server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.')
query_id = random.randrange(0, 65536)
packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0)
for label in name.split('.'):
packet += bytes([len(label)]) + label.encode('ascii')
packet += b'\x00' + struct.pack('!HH', 1, 1)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(5)
sock.sendto(packet, (server, port))
data, _ = sock.recvfrom(512)
if len(data) < 12:
raise SystemExit('short DNS response')
response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12])
rcode = flags & 0x000f
if response_id != query_id:
raise SystemExit('mismatched DNS response id')
if rcode != 0:
raise SystemExit(f'DNS rcode {rcode}')
if an < 1:
raise SystemExit('DNS response had no answers')
print(f'ok answers={an}')
PY
}
check() {
label=\"\$1\"
shift
printf '%-34s ' \"\$label\"
output=\"\$("\$@" 2>&1)\"
status=\$?
if [ \"\$status\" -eq 0 ]; then
printf 'ok'
if [ -n \"\$output\" ]; then
printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\"
fi
printf '\n'
else
printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\"
fi
return 0
}
docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\"
printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\"
if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then
if mountpoint -q '${docker_nvme_root}'; then
printf '%-34s ok - nvme mount active\n' 'Docker root class'
else
printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class'
fi
elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then
printf '%-34s ok - fallback root active\n' 'Docker root class'
else
printf '%-34s warn - unexpected docker root\n' 'Docker root class'
fi
if [ -d '${install_dir}' ]; then
cd '${install_dir}'
sudo docker info --format 'DockerRootDir={{.DockerRootDir}}' 2>/dev/null || true
sudo docker compose ps || true
unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\"
check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com
if [ -n \"\$unbound_ip\" ]; then
check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com
else
printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query'
fi
upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\"
if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then
printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config'
else
printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config'
fi
check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com
check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com
check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\"
else
echo 'RPi services install directory is missing: ${install_dir}'
fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}"
@ -3676,6 +3760,120 @@ status_report() {
status_http "arr prowlarr" "http://127.0.0.1:9696/"
}
doctor_edge() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
local traefik_ip="${LAB_TRAEFIK_LB_IP:-192.168.100.240}"
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
require_debian_server "doctor-edge"
status_section "Doctor Edge"
status_http "website public" "https://lab2025.duckdns.org/"
status_http "gitea public" "https://lab2025.duckdns.org/git/"
printf '\n-- edge backend reachability\n'
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "
set +e
printf '%-28s ' 'traefik backend'
curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable
printf '%-28s ' 'gitea backend'
curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable
printf '\n-- tailscale\n'
tailscale status || true
"; then
printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}"
fi
cat <<'EOF'
Next steps:
- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster.
- If only Gitea is unreachable, run ./jeannie doctor-gitea.
- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge.
- Runbook: docs/runbooks/edge-failures.md
EOF
}
doctor_gitea() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
require_debian_server "doctor-gitea"
status_section "Doctor Gitea"
status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/"
status_http "gitea public" "https://lab2025.duckdns.org/git/"
status_compose_stack "gitea" "/data/homelab-gitea"
status_run "gitea git remote" git ls-remote gitea main
printf '\n-- gitea SSH\n'
ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" || true
printf '\n-- edge to gitea backend\n'
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
"curl -I --max-time 5 'http://${gitea_ts_ip}:${gitea_http_port}/'"; then
printf 'edge host cannot reach Gitea backend %s:%s\n' "${gitea_ts_ip}" "${gitea_http_port}"
fi
cat <<'EOF'
Next steps:
- If local Gitea is down, run ./jeannie deploy-gitea.
- If local works but public /git/ fails, check Tailscale and bootstrap/edge.
- If Git SSH fails, check the gitea remote and registered SSH key.
- Runbook: docs/runbooks/gitea-failures.md
EOF
}
doctor_rpi() {
require_debian_server "doctor-rpi"
status_rpi_services
cat <<'EOF'
Next steps:
- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check.
- If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks.
- If Pi-hole DNS fails, clients using the RPi as DNS will be affected.
- Reapply the stack with ./jeannie rpi-services.
EOF
}
doctor_cluster() {
require_debian_server "doctor-cluster"
status_section "Doctor Cluster"
status_systemd_units kubelet containerd docker
status_kubernetes
status_pimox_workers
status_http "traefik lb" "http://192.168.100.240/"
printf '\n-- local CRI containers\n'
if command -v crictl >/dev/null 2>&1; then
sudo crictl ps -a | head -50 || true
else
printf 'crictl not installed\n'
fi
cat <<'EOF'
Next steps:
- If API server is unreachable after a stop, run ./jeannie start-cluster.
- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node.
- If Pimox workers are stopped, start them with ./jeannie start-cluster.
- If kubelet versions drift, run ./jeannie doctor-versions.
- Runbook: docs/runbooks/cluster-stop-start-failures.md
EOF
}
nuke() {
local worker_ssh_targets
local worker_targets
@ -3866,6 +4064,18 @@ case "${1:-}" in
doctor-versions)
doctor_versions
;;
doctor-edge)
doctor_edge
;;
doctor-gitea)
doctor_gitea
;;
doctor-rpi)
doctor_rpi
;;
doctor-cluster)
doctor_cluster
;;
openwrt)
openwrt
;;
@ -3873,7 +4083,7 @@ case "${1:-}" in
nuke
;;
*)
echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|openwrt|nuke}"
echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|openwrt|nuke}"
exit 1
;;
esac