Add internal synthetic checks

README.md

@@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
 ./jeannie cert-check
 ./jeannie release-snapshot
 ./jeannie backup-status
+./jeannie synthetic-checks
 ./jeannie workers list
 ./jeannie change-journal list
 ./jeannie map
@@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
 restore drill reports, and repo-managed Pi-hole restore inputs. The main
 `status` cascade includes this as a warning signal.
 
+`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API,
+Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
+registry push/pull smoke test is opt-in with
+`LAB_SYNTHETIC_REGISTRY_PUSH=true`.
+
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
 `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the

README.md.tmpl

@@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
 ./{{ main_script }} cert-check
 ./{{ main_script }} release-snapshot
 ./{{ main_script }} backup-status
+./{{ main_script }} synthetic-checks
 ./{{ main_script }} workers list
 ./{{ main_script }} change-journal list
 ./{{ main_script }} map
@@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
 restore drill reports, and repo-managed Pi-hole restore inputs. The main
 `status` cascade includes this as a warning signal.
 
+`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API,
+Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
+registry push/pull smoke test is opt-in with
+`LAB_SYNTHETIC_REGISTRY_PUSH=true`.
+
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
 `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the

docs/jeannie.1.md

@@ -81,6 +81,11 @@ pointers.
 reports, and repo-managed Pi-hole restore inputs. The main `status` cascade
 includes this as a warning signal.
 
+`synthetic-checks`
+: Run end-to-end probes for Gitea HTTP/SSH, registry API, Pi-hole DNS, Traefik
+LAN HTTP, and temporary Kubernetes DNS/ingress pods. Registry push/pull is
+opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
+
 `workers <command>`
 : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
 `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.

jeannie

@@ -5992,6 +5992,11 @@ backup_status() {
     "${REPO_ROOT}/scripts/backup-status"
 }
 
+synthetic_checks() {
+    require_debian_server "synthetic-checks"
+    "${REPO_ROOT}/scripts/synthetic-checks"
+}
+
 workers_manage() {
     require_debian_server "workers"
     "${REPO_ROOT}/scripts/workers" "${@:2}"
@@ -6054,6 +6059,9 @@ case "${1:-}" in
     backup-status)
         backup_status
         ;;
+    synthetic-checks)
+        synthetic_checks
+        ;;
     workers)
         workers_manage "$@"
         ;;
@@ -6211,7 +6219,7 @@ case "${1:-}" in
         echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     *)
-        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
+        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
         exit 1
         ;;
 esac

scripts/synthetic-checks

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
+GITEA_HTTP_URL="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}"
+GITEA_LOCAL_URL="${LAB_GITEA_LOCAL_URL:-http://127.0.0.1:3000/}"
+GITEA_SSH_REMOTE="${LAB_GITOPS_REPO_URL:-ssh://git@192.168.100.73:32222/jv/my-homelab-configs.git}"
+REGISTRY_ENDPOINT="${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}"
+PIHOLE_HOST="${LAB_RPI_HOST:-192.168.100.89}"
+TRAEFIK_LB_IP="${LAB_TRAEFIK_LB_IP:-192.168.100.240}"
+PUBLIC_URL="${LAB_PUBLIC_URL:-https://lab2025.duckdns.org/}"
+
+failures=0
+warnings=0
+
+section() {
+    printf '\n== %s ==\n' "$1"
+}
+
+check() {
+    local label="$1"
+    shift
+    local output
+    local status
+
+    printf '%-34s ' "$label"
+    set +e
+    output="$("$@" 2>&1)"
+    status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        printf 'ok'
+        if [ -n "$output" ]; then
+            printf ' - %s' "$(printf '%s' "$output" | head -n 1)"
+        fi
+        printf '\n'
+    else
+        printf 'fail - %s\n' "$(printf '%s' "$output" | head -n 1)"
+        failures=$((failures + 1))
+    fi
+}
+
+warn_check() {
+    local label="$1"
+    shift
+    local output
+    local status
+
+    printf '%-34s ' "$label"
+    set +e
+    output="$("$@" 2>&1)"
+    status=$?
+    set -e
+    if [ "$status" -eq 0 ]; then
+        printf 'ok'
+        if [ -n "$output" ]; then
+            printf ' - %s' "$(printf '%s' "$output" | head -n 1)"
+        fi
+        printf '\n'
+    else
+        printf 'warn - %s\n' "$(printf '%s' "$output" | head -n 1)"
+        warnings=$((warnings + 1))
+    fi
+}
+
+http_ok() {
+    local url="$1"
+    local status
+
+    status="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 10 "$url")"
+    case "$status" in
+        200|301|302|307|308|401|403|404)
+            echo "http=$status"
+            ;;
+        *)
+            echo "http=$status"
+            return 1
+            ;;
+    esac
+}
+
+dns_query() {
+    local server="$1"
+    local name="$2"
+
+    if command -v dig >/dev/null 2>&1; then
+        dig "@$server" "$name" A +time=3 +tries=1 +short | grep -q .
+    else
+        python3 - "$server" "$name" <<'PY'
+import random
+import socket
+import struct
+import sys
+
+server, name = sys.argv[1], sys.argv[2].rstrip(".")
+query_id = random.randrange(0, 65536)
+packet = struct.pack("!HHHHHH", query_id, 0x0100, 1, 0, 0, 0)
+for label in name.split("."):
+    packet += bytes([len(label)]) + label.encode("ascii")
+packet += b"\x00" + struct.pack("!HH", 1, 1)
+sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+sock.settimeout(5)
+sock.sendto(packet, (server, 53))
+data, _ = sock.recvfrom(512)
+if len(data) < 12:
+    raise SystemExit("short DNS response")
+_response_id, flags, _qd, answers, _ns, _ar = struct.unpack("!HHHHHH", data[:12])
+if flags & 0x000f:
+    raise SystemExit(f"dns rcode {flags & 0x000f}")
+if answers < 1:
+    raise SystemExit("no answers")
+PY
+    fi
+}
+
+kubectl_ok() {
+    command -v kubectl >/dev/null 2>&1 && [ -s "$KUBECONFIG_PATH" ] &&
+        kubectl --kubeconfig "$KUBECONFIG_PATH" get --raw=/readyz >/dev/null 2>&1
+}
+
+section "Gitea And Git"
+check "Gitea public HTTP" http_ok "$GITEA_HTTP_URL"
+check "Gitea local HTTP" http_ok "$GITEA_LOCAL_URL"
+warn_check "Gitea SSH ls-remote" git ls-remote "$GITEA_SSH_REMOTE" HEAD
+
+section "Registry"
+check "Registry API" http_ok "http://${REGISTRY_ENDPOINT}/v2/"
+if [ "${LAB_SYNTHETIC_REGISTRY_PUSH:-false}" = "true" ]; then
+    check "Registry push/pull smoke" sh -c "docker pull busybox:latest >/dev/null && docker tag busybox:latest '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' && docker push '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null && docker pull '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null"
+else
+    printf '%-34s skip - set LAB_SYNTHETIC_REGISTRY_PUSH=true to push/pull a tiny image\n' "Registry push/pull smoke"
+fi
+
+section "DNS"
+check "Pi-hole from Debian" dns_query "$PIHOLE_HOST" cloudflare.com
+
+section "Kubernetes Synthetic Checks"
+if kubectl_ok; then
+    check "Cluster DNS temporary pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup kubernetes.default.svc.cluster.local
+    check "Cluster egress DNS pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-egress-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup cloudflare.com
+    check "Ingress from cluster" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-ingress-check --rm -i --restart=Never --image=curlimages/curl:8.10.1 -- curl -ksS --max-time 10 "$PUBLIC_URL"
+else
+    printf 'Kubernetes API unavailable; skipping pod-based checks.\n'
+    warnings=$((warnings + 1))
+fi
+
+section "LoadBalancer"
+check "Traefik LAN HTTP" http_ok "http://${TRAEFIK_LB_IP}/"
+
+section "Summary"
+printf 'failures=%s warnings=%s\n' "$failures" "$warnings"
+
+if [ "$failures" -gt 0 ]; then
+    exit 1
+fi