Add internal synthetic checks
This commit is contained in:
juvdiaz
parent
4f90fd8904
commit
59eda9feff
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./jeannie cert-check
|
||||
./jeannie release-snapshot
|
||||
./jeannie backup-status
|
||||
./jeannie synthetic-checks
|
||||
./jeannie workers list
|
||||
./jeannie change-journal list
|
||||
./jeannie map
|
||||
|
|
@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
|
|||
restore drill reports, and repo-managed Pi-hole restore inputs. The main
|
||||
`status` cascade includes this as a warning signal.
|
||||
|
||||
`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API,
|
||||
Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
|
||||
registry push/pull smoke test is opt-in with
|
||||
`LAB_SYNTHETIC_REGISTRY_PUSH=true`.
|
||||
|
||||
`workers` provides named Kubernetes/Pimox worker lifecycle operations:
|
||||
`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
|
||||
Use `recreate-plan <index>` before replacing one broken Pimox worker so the
|
||||
|
|
|
|||
|
|
@ -276,6 +276,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./{{ main_script }} cert-check
|
||||
./{{ main_script }} release-snapshot
|
||||
./{{ main_script }} backup-status
|
||||
./{{ main_script }} synthetic-checks
|
||||
./{{ main_script }} workers list
|
||||
./{{ main_script }} change-journal list
|
||||
./{{ main_script }} map
|
||||
|
|
@ -322,6 +323,11 @@ public URL status, and pointers to the latest Gitea/OpenTofu backups.
|
|||
restore drill reports, and repo-managed Pi-hole restore inputs. The main
|
||||
`status` cascade includes this as a warning signal.
|
||||
|
||||
`synthetic-checks` runs end-to-end probes for Gitea HTTP/SSH, registry API,
|
||||
Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
|
||||
registry push/pull smoke test is opt-in with
|
||||
`LAB_SYNTHETIC_REGISTRY_PUSH=true`.
|
||||
|
||||
`workers` provides named Kubernetes/Pimox worker lifecycle operations:
|
||||
`list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
|
||||
Use `recreate-plan <index>` before replacing one broken Pimox worker so the
|
||||
|
|
|
|||
|
|
@ -81,6 +81,11 @@ pointers.
|
|||
reports, and repo-managed Pi-hole restore inputs. The main `status` cascade
|
||||
includes this as a warning signal.
|
||||
|
||||
`synthetic-checks`
|
||||
: Run end-to-end probes for Gitea HTTP/SSH, registry API, Pi-hole DNS, Traefik
|
||||
LAN HTTP, and temporary Kubernetes DNS/ingress pods. Registry push/pull is
|
||||
opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
|
||||
|
||||
`workers <command>`
|
||||
: Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
|
||||
`start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
|
||||
|
|
|
|||
10
jeannie
10
jeannie
|
|
@ -5992,6 +5992,11 @@ backup_status() {
|
|||
"${REPO_ROOT}/scripts/backup-status"
|
||||
}
|
||||
|
||||
synthetic_checks() {
|
||||
require_debian_server "synthetic-checks"
|
||||
"${REPO_ROOT}/scripts/synthetic-checks"
|
||||
}
|
||||
|
||||
workers_manage() {
|
||||
require_debian_server "workers"
|
||||
"${REPO_ROOT}/scripts/workers" "${@:2}"
|
||||
|
|
@ -6054,6 +6059,9 @@ case "${1:-}" in
|
|||
backup-status)
|
||||
backup_status
|
||||
;;
|
||||
synthetic-checks)
|
||||
synthetic_checks
|
||||
;;
|
||||
workers)
|
||||
workers_manage "$@"
|
||||
;;
|
||||
|
|
@ -6211,7 +6219,7 @@ case "${1:-}" in
|
|||
echo "Log: ${JEANNIE_LOG_FILE}"
|
||||
;;
|
||||
*)
|
||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
|
||||
GITEA_HTTP_URL="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}"
|
||||
GITEA_LOCAL_URL="${LAB_GITEA_LOCAL_URL:-http://127.0.0.1:3000/}"
|
||||
GITEA_SSH_REMOTE="${LAB_GITOPS_REPO_URL:-ssh://git@192.168.100.73:32222/jv/my-homelab-configs.git}"
|
||||
REGISTRY_ENDPOINT="${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}"
|
||||
PIHOLE_HOST="${LAB_RPI_HOST:-192.168.100.89}"
|
||||
TRAEFIK_LB_IP="${LAB_TRAEFIK_LB_IP:-192.168.100.240}"
|
||||
PUBLIC_URL="${LAB_PUBLIC_URL:-https://lab2025.duckdns.org/}"
|
||||
|
||||
failures=0
|
||||
warnings=0
|
||||
|
||||
section() {
|
||||
printf '\n== %s ==\n' "$1"
|
||||
}
|
||||
|
||||
check() {
|
||||
local label="$1"
|
||||
shift
|
||||
local output
|
||||
local status
|
||||
|
||||
printf '%-34s ' "$label"
|
||||
set +e
|
||||
output="$("$@" 2>&1)"
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 0 ]; then
|
||||
printf 'ok'
|
||||
if [ -n "$output" ]; then
|
||||
printf ' - %s' "$(printf '%s' "$output" | head -n 1)"
|
||||
fi
|
||||
printf '\n'
|
||||
else
|
||||
printf 'fail - %s\n' "$(printf '%s' "$output" | head -n 1)"
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
warn_check() {
|
||||
local label="$1"
|
||||
shift
|
||||
local output
|
||||
local status
|
||||
|
||||
printf '%-34s ' "$label"
|
||||
set +e
|
||||
output="$("$@" 2>&1)"
|
||||
status=$?
|
||||
set -e
|
||||
if [ "$status" -eq 0 ]; then
|
||||
printf 'ok'
|
||||
if [ -n "$output" ]; then
|
||||
printf ' - %s' "$(printf '%s' "$output" | head -n 1)"
|
||||
fi
|
||||
printf '\n'
|
||||
else
|
||||
printf 'warn - %s\n' "$(printf '%s' "$output" | head -n 1)"
|
||||
warnings=$((warnings + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
http_ok() {
|
||||
local url="$1"
|
||||
local status
|
||||
|
||||
status="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 10 "$url")"
|
||||
case "$status" in
|
||||
200|301|302|307|308|401|403|404)
|
||||
echo "http=$status"
|
||||
;;
|
||||
*)
|
||||
echo "http=$status"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
dns_query() {
|
||||
local server="$1"
|
||||
local name="$2"
|
||||
|
||||
if command -v dig >/dev/null 2>&1; then
|
||||
dig "@$server" "$name" A +time=3 +tries=1 +short | grep -q .
|
||||
else
|
||||
python3 - "$server" "$name" <<'PY'
|
||||
import random
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
|
||||
server, name = sys.argv[1], sys.argv[2].rstrip(".")
|
||||
query_id = random.randrange(0, 65536)
|
||||
packet = struct.pack("!HHHHHH", query_id, 0x0100, 1, 0, 0, 0)
|
||||
for label in name.split("."):
|
||||
packet += bytes([len(label)]) + label.encode("ascii")
|
||||
packet += b"\x00" + struct.pack("!HH", 1, 1)
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.settimeout(5)
|
||||
sock.sendto(packet, (server, 53))
|
||||
data, _ = sock.recvfrom(512)
|
||||
if len(data) < 12:
|
||||
raise SystemExit("short DNS response")
|
||||
_response_id, flags, _qd, answers, _ns, _ar = struct.unpack("!HHHHHH", data[:12])
|
||||
if flags & 0x000f:
|
||||
raise SystemExit(f"dns rcode {flags & 0x000f}")
|
||||
if answers < 1:
|
||||
raise SystemExit("no answers")
|
||||
PY
|
||||
fi
|
||||
}
|
||||
|
||||
kubectl_ok() {
|
||||
command -v kubectl >/dev/null 2>&1 && [ -s "$KUBECONFIG_PATH" ] &&
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" get --raw=/readyz >/dev/null 2>&1
|
||||
}
|
||||
|
||||
section "Gitea And Git"
|
||||
check "Gitea public HTTP" http_ok "$GITEA_HTTP_URL"
|
||||
check "Gitea local HTTP" http_ok "$GITEA_LOCAL_URL"
|
||||
warn_check "Gitea SSH ls-remote" git ls-remote "$GITEA_SSH_REMOTE" HEAD
|
||||
|
||||
section "Registry"
|
||||
check "Registry API" http_ok "http://${REGISTRY_ENDPOINT}/v2/"
|
||||
if [ "${LAB_SYNTHETIC_REGISTRY_PUSH:-false}" = "true" ]; then
|
||||
check "Registry push/pull smoke" sh -c "docker pull busybox:latest >/dev/null && docker tag busybox:latest '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' && docker push '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null && docker pull '${REGISTRY_ENDPOINT}/synthetic/busybox:latest' >/dev/null"
|
||||
else
|
||||
printf '%-34s skip - set LAB_SYNTHETIC_REGISTRY_PUSH=true to push/pull a tiny image\n' "Registry push/pull smoke"
|
||||
fi
|
||||
|
||||
section "DNS"
|
||||
check "Pi-hole from Debian" dns_query "$PIHOLE_HOST" cloudflare.com
|
||||
|
||||
section "Kubernetes Synthetic Checks"
|
||||
if kubectl_ok; then
|
||||
check "Cluster DNS temporary pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup kubernetes.default.svc.cluster.local
|
||||
check "Cluster egress DNS pod" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-egress-dns-check --rm -i --restart=Never --image=busybox:1.36 -- nslookup cloudflare.com
|
||||
check "Ingress from cluster" kubectl --kubeconfig "$KUBECONFIG_PATH" run synthetic-ingress-check --rm -i --restart=Never --image=curlimages/curl:8.10.1 -- curl -ksS --max-time 10 "$PUBLIC_URL"
|
||||
else
|
||||
printf 'Kubernetes API unavailable; skipping pod-based checks.\n'
|
||||
warnings=$((warnings + 1))
|
||||
fi
|
||||
|
||||
section "LoadBalancer"
|
||||
check "Traefik LAN HTTP" http_ok "http://${TRAEFIK_LB_IP}/"
|
||||
|
||||
section "Summary"
|
||||
printf 'failures=%s warnings=%s\n' "$failures" "$warnings"
|
||||
|
||||
if [ "$failures" -gt 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
Loading…
Reference in New Issue