diff --git a/README.md b/README.md index d7e89ce..0c09cf1 100644 --- a/README.md +++ b/README.md @@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS for controlled practice targets. -`capacity` is the placement report for the hardware you already have. It -summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage, -Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement -guidance for deciding what can safely run next. +`capacity` is the placement report for the hardware you already have. By +default it uses the compact report renderer and shows summarized Debian, +Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use +`./jeannie capacity --details`, `--json`, `--only problems`, or +`--verbose` for the older full capacity dump. `recover-plan` prints the disaster recovery order and lightweight prerequisite checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and diff --git a/README.md.tmpl b/README.md.tmpl index 34cb091..b316b28 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS for controlled practice targets. -`capacity` is the placement report for the hardware you already have. It -summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage, -Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement -guidance for deciding what can safely run next. +`capacity` is the placement report for the hardware you already have. By +default it uses the compact report renderer and shows summarized Debian, +Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use +`./{{ main_script }} capacity --details`, `--json`, `--only problems`, or +`--verbose` for the older full capacity dump. `recover-plan` prints the disaster recovery order and lightweight prerequisite checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index e37fb79..f0c21a8 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -53,9 +53,9 @@ deployment readiness, pod restarts, node pressure, disk pressure, Traefik 5xx/404 log evidence, and recent Gitea errors. `capacity` -: Print a read-only placement report covering Debian memory/disk/Docker usage, -Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk -state, and practical placement guidance. +: Print a compact read-only placement report covering Debian, Kubernetes, +Pimox, RPi, and placement signals. Use `capacity --details`, `--json`, `--only +problems`, or `--verbose` for the older full dump. `recover-plan` : Print the ordered disaster recovery checklist and lightweight prerequisite diff --git a/jeannie b/jeannie index 29df79a..6bc860c 100755 --- a/jeannie +++ b/jeannie @@ -5965,7 +5965,7 @@ kubeconfig_readonly() { capacity_report() { require_debian_server "capacity" - "${REPO_ROOT}/scripts/capacity-report" + "${REPO_ROOT}/scripts/capacity-report" "${@:2}" } recover_plan() { @@ -6071,7 +6071,7 @@ case "${1:-}" in status_report ;; capacity) - capacity_report + capacity_report "$@" ;; recover-plan) recover_plan diff --git a/scripts/capacity-report b/scripts/capacity-report index 363398a..51a67f8 100755 --- a/scripts/capacity-report +++ b/scripts/capacity-report @@ -2,6 +2,51 @@ set -euo pipefail KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}" +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +DETAIL_ARGS=() +VERBOSE=false + +while (($# > 0)); do + case "$1" in + --verbose) + VERBOSE=true + shift + ;; + --details|--json) + DETAIL_ARGS+=("$1") + shift + ;; + --only) + DETAIL_ARGS+=("$1" "${2:-all}") + shift 2 + ;; + -h|--help) + cat <<'EOF' +Usage: ./jeannie capacity [--verbose] [--details] [--json] [--only failures|warnings|problems|all] + +Default output is a compact reusable report. Use --verbose for the older full +capacity dump. +EOF + exit 0 + ;; + *) + echo "unknown capacity option: $1" >&2 + exit 1 + ;; + esac +done + +emit() { + local status="$1" + local area="$2" + local check="$3" + local summary="$4" + local detail="${5:-}" + local hint="${6:-}" + + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${hint}" +} section() { printf '\n== %s ==\n' "$1" @@ -108,7 +153,7 @@ Use this report to decide placement: EOF } -main() { +verbose_main() { host_capacity kubernetes_capacity pimox_capacity @@ -116,4 +161,199 @@ main() { summary } -main "$@" +compact_host_capacity() { + local mem_total + local mem_used + local mem_pct + local path + local line + local use_pct + local docker_root + + if command -v free >/dev/null 2>&1; then + read -r mem_total mem_used < <(free -m | awk '/^Mem:/ { print $2, $3 }') + if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then + mem_pct=$((mem_used * 100 / mem_total)) + if ((mem_pct >= 85)); then + emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers." + else + emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" + fi + else + emit skip "Debian" "Memory" "unavailable" "free output could not be parsed" + fi + else + emit skip "Debian" "Memory" "free missing" "" "Install procps." + fi + + for path in / /data /var/lib/docker; do + line="$(df -P "${path}" 2>/dev/null | awk 'NR == 2 { print $5 " " $4 }' || true)" + if [[ -z "${line}" ]]; then + emit skip "Debian" "Disk ${path}" "unavailable" "path not present" + continue + fi + use_pct="${line%% *}" + use_pct="${use_pct%%%}" + if ((use_pct >= 90)); then + emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services." + elif ((use_pct >= 80)); then + emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage." + else + emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" + fi + done + + if command -v docker >/dev/null 2>&1; then + docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" + if [[ -n "${docker_root}" ]]; then + emit ok "Debian" "Docker root" "${docker_root}" "" "Keep Docker on NVMe-backed storage." + else + emit warn "Debian" "Docker root" "unavailable" "docker info failed" "Check docker service." + fi + else + emit skip "Debian" "Docker" "not installed" + fi +} + +compact_kubernetes_capacity() { + local not_ready + local problem_pods + local missing_resources + local top_nodes + + if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then + emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG." + return 0 + fi + if ! kubectl --kubeconfig "${KUBECONFIG}" get --raw=/readyz >/dev/null 2>&1; then + emit warn "Kubernetes" "API" "not reachable" "readyz failed" "./jeannie start-cluster" + return 0 + fi + + not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')" + if ((not_ready > 0)); then + emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster" + else + emit ok "Kubernetes" "Node readiness" "all ready" + fi + + problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')" + if ((problem_pods > 0)); then + emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide" + else + emit ok "Kubernetes" "Problem pods" "none" + fi + + missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null | + awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')" + if ((missing_resources > 0)); then + emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget" + else + emit ok "Kubernetes" "Resource policy" "requests/limits present" + fi + + top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)" + if [[ -n "${top_nodes}" ]]; then + emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" + else + emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready" + fi +} + +compact_pimox_capacity() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}" + local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" + local output + + output="$(ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" "set +e +storage_line=\"\$(sudo pvesm status 2>/dev/null | awk -v storage='${worker_storage}' '\$1 == storage { print \$7 }')\" +running_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 && \$3 == \"running\" { count++ } END { print count + 0 }')\" +total_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 { count++ } END { print count + 0 }')\" +echo \"storage_used_pct=\${storage_line:-unknown}\" +echo \"running_vms=\${running_vms:-0}\" +echo \"total_vms=\${total_vms:-0}\" +" 2>/dev/null || true)" + + if [[ -z "${output}" ]]; then + emit warn "Pimox" "Host" "unreachable" "${pimox_user}@${pimox_host}" "./jeannie doctor-cluster" + return 0 + fi + + local storage_pct + local running_vms + local total_vms + storage_pct="$(printf '%s\n' "${output}" | awk -F= '/storage_used_pct=/ { print $2 }')" + running_vms="$(printf '%s\n' "${output}" | awk -F= '/running_vms=/ { print $2 }')" + total_vms="$(printf '%s\n' "${output}" | awk -F= '/total_vms=/ { print $2 }')" + + if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then + storage_pct="${storage_pct%%%}" + if ((${storage_pct%.*} >= 85)); then + emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage." + else + emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" + fi + else + emit warn "Pimox" "Worker storage" "unknown" "${worker_storage}" "Check pvesm status." + fi + emit ok "Pimox" "VMs" "running=${running_vms:-0}/${total_vms:-0}" +} + +compact_rpi_capacity() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}" + local output + + output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" ' +set +e +docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)" +root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")" +nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")" +echo "docker_root=${docker_root:-unknown}" +echo "root_used=${root_used:-unknown}" +echo "nvme_used=${nvme_used:-unknown}" +' 2>/dev/null || true)" + + if [[ -z "${output}" ]]; then + emit warn "RPi4" "Host" "unreachable" "${rpi_user}@${rpi_host}" "./jeannie doctor-rpi" + return 0 + fi + + local docker_root + local root_used + local nvme_used + docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')" + root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')" + nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')" + emit ok "RPi4" "Docker root" "${docker_root:-unknown}" + emit ok "RPi4" "Root disk" "${root_used:-unknown}" + if [[ "${nvme_used:-unknown}" = "unknown" ]]; then + emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi" + else + emit ok "RPi4" "NVMe mount" "${nvme_used}" + fi +} + +compact_guidance() { + emit ok "Guidance" "Placement" "Prefer app workloads on Pimox workers" "Keep Debian headroom for Gitea, registry, Docker, kubeadm, and Ollama." + emit ok "Guidance" "Storage" "Treat Pimox VM disks as rebuildable" "Durable data still needs explicit backup/restore paths." +} + +compact_main() { + { + compact_host_capacity + compact_kubernetes_capacity + compact_pimox_capacity + compact_rpi_capacity + compact_guidance + } | "${REPO_ROOT}/scripts/report-render" --title "Homelab Capacity" "${DETAIL_ARGS[@]}" +} + +if [[ "${VERBOSE}" == "true" ]]; then + verbose_main +else + compact_main +fi