diff --git a/README.md b/README.md index f20b2ec..8e43857 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie capacity ./jeannie recover-plan ./jeannie gitops-status +./jeannie cert-check ``` It reports host memory/disk, systemd services, Docker Compose stacks, @@ -305,6 +306,9 @@ edge verification. out-of-sync or degraded apps, repository secret presence, recent Argo CD events, and recent repo-server/application-controller errors. +`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea +HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/README.md.tmpl b/README.md.tmpl index 2151e16..fcb82fa 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -273,6 +273,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} capacity ./{{ main_script }} recover-plan ./{{ main_script }} gitops-status +./{{ main_script }} cert-check ``` It reports host memory/disk, systemd services, Docker Compose stacks, @@ -305,6 +306,9 @@ edge verification. out-of-sync or degraded apps, repository secret presence, recent Argo CD events, and recent repo-server/application-controller errors. +`cert-check` verifies public DNS, TLS certificate expiry, public website/Gitea +HTTP status, and DuckDNS-to-OCI edge IP drift from the canonical inventory. + Focused doctor commands run narrower read-only checks and print the most likely next step: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 80084a1..703270a 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -67,6 +67,10 @@ verification. or degraded apps, repository secret presence, recent events, and recent repo-server/application-controller errors. +`cert-check` +: Verify public DNS, TLS certificate expiry, public website/Gitea HTTP status, +and DuckDNS-to-OCI edge IP drift from the canonical inventory. + Prometheus alerts : High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app. They cover node readiness, restart storms, unavailable deployments, storage diff --git a/jeannie b/jeannie index 70661ea..7d8568d 100755 --- a/jeannie +++ b/jeannie @@ -5977,6 +5977,10 @@ gitops_status() { "${REPO_ROOT}/scripts/gitops-status" } +cert_check() { + "${REPO_ROOT}/scripts/cert-check" +} + case "${1:-}" in up) up @@ -6005,6 +6009,9 @@ case "${1:-}" in gitops-status) gitops_status ;; + cert-check) + cert_check + ;; validate) validate_homelab ;; @@ -6149,7 +6156,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/cert-check b/scripts/cert-check new file mode 100755 index 0000000..4f22b04 --- /dev/null +++ b/scripts/cert-check @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +INVENTORY_FILE="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" +MIN_DAYS="${LAB_CERT_MIN_DAYS:-21}" +DEFAULT_DOMAIN="${LAB_DOMAIN:-lab2025.duckdns.org}" +DEFAULT_PUBLIC_URL="${LAB_PUBLIC_URL:-https://${DEFAULT_DOMAIN}/}" +DEFAULT_GITEA_URL="${LAB_GITEA_URL:-https://${DEFAULT_DOMAIN}/git/jv/my-homelab-configs}" + +failures=0 +hosts_file="" + +cleanup() { + if [ -n "$hosts_file" ] && [ -f "$hosts_file" ]; then + rm -f "$hosts_file" + fi +} +trap cleanup EXIT + +section() { + printf '\n== %s ==\n' "$1" +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +mark_fail() { + failures=$((failures + 1)) + printf 'fail: %s\n' "$1" +} + +inventory_scalar() { + local path="$1" + + if have ruby && [ -f "$INVENTORY_FILE" ]; then + ruby -ryaml -e ' + data = YAML.load_file(ARGV[0]) + value = ARGV[1].split(".").reduce(data) { |memo, key| memo.respond_to?(:[]) ? memo[key] : nil } + puts value if value.is_a?(String) + ' "$INVENTORY_FILE" "$path" 2>/dev/null || true + fi +} + +inventory_hosts() { + if have ruby && [ -f "$INVENTORY_FILE" ]; then + ruby -ryaml -e ' + data = YAML.load_file(ARGV[0]) + domain = data.fetch("domain", {}) + hosts = [domain["base"]] + Array(domain["subdomains"]) + puts hosts.compact.uniq + ' "$INVENTORY_FILE" 2>/dev/null || true + fi +} + +domain_base="$(inventory_scalar domain.base)" +public_url="$(inventory_scalar domain.public_url)" +gitea_url="$(inventory_scalar domain.gitea_url)" + +domain_base="${domain_base:-$DEFAULT_DOMAIN}" +public_url="${public_url:-$DEFAULT_PUBLIC_URL}" +gitea_url="${gitea_url:-$DEFAULT_GITEA_URL}" + +hosts="$(inventory_hosts)" +if [ -z "$hosts" ]; then + hosts="$domain_base" +fi + +hosts_file="$(mktemp "${TMPDIR:-/tmp}/homelab-cert-hosts.XXXXXX")" +printf '%s\n' "$hosts" >"$hosts_file" + +section "DNS Resolution" +while IFS= read -r host; do + [ -n "$host" ] || continue + if have getent; then + if getent hosts "$host" >/dev/null 2>&1; then + printf '%-40s ok\n' "$host" + else + printf '%-40s fail\n' "$host" + failures=$((failures + 1)) + fi + elif have dig; then + if dig "$host" A +time=3 +tries=1 +short | grep -q .; then + printf '%-40s ok\n' "$host" + else + printf '%-40s fail\n' "$host" + failures=$((failures + 1)) + fi + else + echo "getent or dig is required for DNS checks." + failures=$((failures + 1)) + break + fi +done <"$hosts_file" + +section "TLS Expiry" +while IFS= read -r host; do + [ -n "$host" ] || continue + if python3 - "$host" "$MIN_DAYS" <<'PY' +import datetime +import os +import socket +import ssl +import sys + +host = sys.argv[1] +minimum_days = int(sys.argv[2]) +context = ssl.create_default_context() + +try: + with socket.create_connection((host, 443), timeout=8) as sock: + with context.wrap_socket(sock, server_hostname=host) as tls_sock: + cert = tls_sock.getpeercert() +except Exception as exc: + print(f"{host:<40} fail tls_connect={exc}") + sys.exit(1) + +not_after = datetime.datetime.strptime( + cert["notAfter"], "%b %d %H:%M:%S %Y %Z" +).replace(tzinfo=datetime.timezone.utc) +days = (not_after - datetime.datetime.now(datetime.timezone.utc)).days +issuer_parts = cert.get("issuer", ()) +issuer = ",".join("=".join(item) for group in issuer_parts for item in group) +status = "ok" if days >= minimum_days else "warn" +print(f"{host:<40} {status} expires={not_after.date()} days={days} issuer={issuer}") +sys.exit(0 if days >= minimum_days else 1) +PY + then + : + else + failures=$((failures + 1)) + fi +done <"$hosts_file" + +section "Public HTTP" +for url in "$public_url" "$gitea_url"; do + status="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 10 "$url" 2>/dev/null || true)" + case "$status" in + 200|301|302|307|308) + printf '%-60s ok http=%s\n' "$url" "$status" + ;; + *) + printf '%-60s fail http=%s\n' "$url" "${status:-none}" + failures=$((failures + 1)) + ;; + esac +done + +section "Edge IP Drift" +edge_ip="$(inventory_scalar hosts.oci_edge.public_ip)" +if [ -n "$edge_ip" ] && have dig; then + resolved="$(dig "$domain_base" A +time=3 +tries=1 +short | tail -1)" + if [ "$resolved" = "$edge_ip" ]; then + printf '%-40s ok resolved=%s\n' "$domain_base" "$resolved" + else + mark_fail "$domain_base resolves to ${resolved:-none}, expected OCI edge $edge_ip" + fi +else + echo "Skipping edge IP drift check; dig or OCI edge inventory value is unavailable." +fi + +if [ "$failures" -gt 0 ]; then + printf '\ncert-check found %s issue(s).\n' "$failures" + exit 1 +fi + +printf '\ncert-check found no blocking issues.\n'