diff --git a/README.md b/README.md index 38863ad..8a0a6c0 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie cert-check ./jeannie release-snapshot ./jeannie backup-status +./jeannie synthetic-checks ./jeannie workers list ./jeannie change-journal list ./jeannie map @@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups. restore drill reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API, +Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The +registry push/pull smoke test is opt-in with +`LAB_SYNTHETIC_REGISTRY_PUSH=true`. + `workers` provides named Kubernetes/Pimox worker lifecycle operations: `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the diff --git a/README.md.tmpl b/README.md.tmpl index 4f98808..fee6239 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} cert-check ./{{ main_script }} release-snapshot ./{{ main_script }} backup-status +./{{ main_script }} synthetic-checks ./{{ main_script }} workers list ./{{ main_script }} change-journal list ./{{ main_script }} map @@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups. restore drill reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API, +Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The +registry push/pull smoke test is opt-in with +`LAB_SYNTHETIC_REGISTRY_PUSH=true`. + `workers` provides named Kubernetes/Pimox worker lifecycle operations: `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 663cf34..50e6f0c 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -81,6 +81,11 @@ pointers. reports, and repo-managed Pi-hole restore inputs. The main `status` cascade includes this as a warning signal. +`synthetic-checks` +: Run end-to-end probes for Gitea HTTP/SSH, registry API, Pi-hole DNS, Traefik +LAN HTTP, and temporary Kubernetes DNS/ingress pods. Registry push/pull is +opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`. + `workers ` : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. diff --git a/jeannie b/jeannie index fd075e7..5e93ab3 100755 --- a/jeannie +++ b/jeannie @@ -5992,6 +5992,11 @@ backup_status() { "${REPO_ROOT}/scripts/backup-status" } +synthetic_checks() { + require_debian_server "synthetic-checks" + "${REPO_ROOT}/scripts/synthetic-checks" +} + workers_manage() { require_debian_server "workers" "${REPO_ROOT}/scripts/workers" "${@:2}" @@ -6054,6 +6059,9 @@ case "${1:-}" in backup-status) backup_status ;; + synthetic-checks) + synthetic_checks + ;; workers) workers_manage "$@" ;; @@ -6211,7 +6219,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/synthetic-checks b/scripts/synthetic-checks new file mode 100755 index 0000000..08cabf1 --- /dev/null +++ b/scripts/synthetic-checks @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" +GITEA_HTTP_URL="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}" +GITEA_LOCAL_URL="${LAB_GITEA_LOCAL_URL:-http://127.0.0.1:3000/}" +GITEA_SSH_REMOTE="${LAB_GITOPS_REPO_URL:-ssh://git@192.168.100.73:32222/jv/my-homelab-configs.git}" +REGISTRY_ENDPOINT="${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}" +PIHOLE_HOST="${LAB_RPI_HOST:-192.168.100.89}" +TRAEFIK_LB_IP="${LAB_TRAEFIK_LB_IP:-192.168.100.240}" +PUBLIC_URL="${LAB_PUBLIC_URL:-https://lab2025.duckdns.org/}" + +failures=0 +warnings=0 + +section() { + printf '\n== %s ==\n' "$1" +} + +check() { + local label="$1" + shift + local output + local status + + printf '%-34s ' "$label" + set +e + output="$("$@" 2>&1)" + status=$? + set -e + if [ "$status" -eq 0 ]; then + printf 'ok' + if [ -n "$output" ]; then + printf ' - %s' "$(printf '%s' "$output" | head -n 1)" + fi + printf '\n' + else + printf 'fail - %s\n' "$(printf '%s' "$output" | head -n 1)" + failures=$((failures + 1)) + fi +} + +warn_check() { + local label="$1" + shift + local output + local status + + printf '%-34s ' "$label" + set +e + output="$("$@" 2>&1)" + status=$? + set -e + if [ "$status" -eq 0 ]; then + printf 'ok' + if [ -n "$output" ]; then + printf ' - %s' "$(printf '%s' "$output" | head -n 1)" + fi + printf '\n' + else + printf 'warn - %s\n' "$(printf '%s' "$output" | head -n 1)" + warnings=$((warnings + 1)) + fi +} + +http_ok() { + local url="$1" + local status + + status="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 10 "$url")" + case "$status" in + 200|301|302|307|308|401|403|404) + echo "http=$status" + ;; + *) + echo "http=$status" + return 1 + ;; + esac +} + +dns_query() { + local server="$1" + local name="$2" + + if command -v dig >/dev/null 2>&1; then + dig "@$server" "$name" A +time=3 +tries=1 +short | grep -q . + else + python3 - "$server" "$name" <<'PY' +import random +import socket +import struct +import sys + +server, name = sys.argv[1], sys.argv[2].rstrip(".") +query_id = random.randrange(0, 65536) +packet = struct.pack("!HHHHHH", query_id, 0x0100, 1, 0, 0, 0) +for label in name.split("."): + packet += bytes([len(label)]) + label.encode("ascii") +packet += b"\x00" + struct.pack("!HH", 1, 1) +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.settimeout(5) +sock.sendto(packet, (server, 53)) +data, _ = sock.recvfrom(512) +if len(data) < 12: + raise SystemExit("short DNS response") +_response_id, flags, _qd, answers, _ns, _ar = struct.unpack("!HHHHHH", data[:12]) +if flags & 0x000f: + raise SystemExit(f"dns rcode {flags & 0x000f}") +if answers < 1: + raise SystemExit("no answers") +PY + fi +} + +kubectl_ok() { + command -v kubectl >/dev/null 2>&1 && [ -s "$KUBECONFIG_PATH" ] && + kubectl --kubeconfig "$KUBECONFIG_PATH" get --raw=/readyz >/dev/null 2>&1 +} + +section "Gitea And Git" +check "Gitea public HTTP" http_ok "$GITEA_HTTP_URL" +check "Gitea local HTTP" http_ok "$GITEA_LOCAL_URL" +warn_check "Gitea SSH ls-remote" git ls-remote "$GITEA_SSH_REMOTE" HEAD + +section "Registry" +check "Registry API" http_ok "http://${REGISTRY_ENDPOINT}/v2/" +if [ "${LAB_SYNTHETIC_REGISTRY_PUSH:-false}" = "true" ]; then + check "Registry push/pull smoke" sh -c "docker pull busybox:latest >/dev/null && docker tag busybox:latest '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' && docker push '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null && docker pull '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null" +else + printf '%-34s skip - set LAB_SYNTHETIC_REGISTRY_PUSH=true to push/pull a tiny image\n' "Registry push/pull smoke" +fi + +section "DNS" +check "Pi-hole from Debian" dns_query "$PIHOLE_HOST" cloudflare.com + +section "Kubernetes Synthetic Checks" +if kubectl_ok; then + check "Cluster DNS temporary pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup kubernetes.default.svc.cluster.local + check "Cluster egress DNS pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-egress-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup cloudflare.com + check "Ingress from cluster" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-ingress-check --rm -i --restart=Never --image=curlimages/curl:8.10.1 -- curl -ksS --max-time 10 "$PUBLIC_URL" +else + printf 'Kubernetes API unavailable; skipping pod-based checks.\n' + warnings=$((warnings + 1)) +fi + +section "LoadBalancer" +check "Traefik LAN HTTP" http_ok "http://${TRAEFIK_LB_IP}/" + +section "Summary" +printf 'failures=%s warnings=%s\n' "$failures" "$warnings" + +if [ "$failures" -gt 0 ]; then + exit 1 +fi