From 8582c3ea61a3c03d702f94349c0ed49b40efc52b Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 13:27:46 -0600 Subject: [PATCH] Add pre-apply readiness doctor checks --- jeannie | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/jeannie b/jeannie index e02127e..56921ef 100755 --- a/jeannie +++ b/jeannie @@ -1433,6 +1433,68 @@ fi sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'" } +check_local_disk_free_gib() { + local path="$1" + local min_gib="$2" + local available_kib + local min_kib + + available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')" + min_kib=$((min_gib * 1024 * 1024)) + if ((available_kib < min_kib)); then + echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2 + return 1 + fi +} + +check_systemd_active() { + local unit="$1" + + systemctl is-active --quiet "${unit}" +} + +check_rpi_docker_writable() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu +root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" +sudo mkdir -p \"\$root/.homelab-check\" +sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\" +sudo rm -f \"\$root/.homelab-check/write-test\"" +} + +check_edge_disk_free() { + local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu +available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\" +min_kib=$((min_gib * 1024 * 1024)) +if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then + echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2 + exit 1 +fi" +} + +check_pihole_dns_query() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + + if command -v dig >/dev/null 2>&1; then + dig @"${rpi_host}" pi.hole A +time=3 +tries=1 >/dev/null + return $? + fi + if command -v nslookup >/dev/null 2>&1; then + nslookup pi.hole "${rpi_host}" >/dev/null + return $? + fi + + echo "dig or nslookup is required to test Pi-hole DNS directly." >&2 + return 1 +} + homelab_preflight() { local phase="${1:-full}" local failures=0 @@ -1474,6 +1536,29 @@ homelab_preflight() { echo "${phase^} preflight checks passed." } +doctor_preapply() { + local failures=0 + + require_debian_server "doctor-preapply" + + echo "Running pre-apply readiness checks..." + preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1)) + preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1)) + preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1)) + preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1)) + preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1)) + preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) + preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1)) + preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1)) + + if ((failures > 0)); then + echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2 + exit 1 + fi + + echo "Pre-apply doctor checks passed." +} + inventory_check() { "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" } @@ -4030,13 +4115,14 @@ up() { echo "Deploying the homelab infrastructure..." jeannie_log_start "up" - jeannie_step_plan 14 + jeannie_step_plan 15 run_step "Early preflight" homelab_preflight early run_step "Gitea deploy" deploy_gitea run_step "Gitea repo bootstrap" bootstrap_gitea_repo run_step "RPi services" deploy_rpi_services run_step "Full preflight" homelab_preflight full + run_step "Pre-apply doctor" doctor_preapply run_step "Pimox provisioning and workers" run_pimox_pipeline run_step "OpenWrt VM" run_openwrt_pipeline run_step "Worker var file" ensure_cluster_worker_var_file @@ -5475,6 +5561,9 @@ case "${1:-}" in preflight) homelab_preflight ;; + doctor-preapply) + doctor_preapply + ;; inventory-check) inventory_check ;; @@ -5510,7 +5599,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}" exit 1 ;; esac