#!/usr/bin/env bash set -euo pipefail KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" GRAFANA_BASE_URL="${LAB_GRAFANA_URL:-https://grafana.${LAB_DOMAIN:-lab2025.duckdns.org}}" GRAFANA_CAPACITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-capacity/homelab-capacity?orgId=1" GRAFANA_AVAILABILITY_URL="${GRAFANA_BASE_URL%/}/d/homelab-availability/homelab-availability?orgId=1" DETAIL_ARGS=() VERBOSE=false while (($# > 0)); do case "$1" in --verbose) VERBOSE=true shift ;; --details|--json) DETAIL_ARGS+=("$1") shift ;; --only) DETAIL_ARGS+=("$1" "${2:-all}") shift 2 ;; -h|--help) cat <<'EOF' Usage: ./jeannie capacity [--verbose] [--details] [--json] [--only failures|warnings|problems|all] Default output is a compact reusable report. Use --verbose for the older full capacity dump. EOF exit 0 ;; *) echo "unknown capacity option: $1" >&2 exit 1 ;; esac done emit() { local status="$1" local area="$2" local check="$3" local summary="$4" local detail="${5:-}" local fix="${6:-}" local graph="${7:-}" local query="${8:-}" printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${fix}" "${graph}" "${query}" } grafana_explore_url() { local expr="$1" python3 - "${GRAFANA_BASE_URL%/}" "$expr" <<'PY' import json import sys import urllib.parse base_url, expr = sys.argv[1:3] left = { "datasource": "prometheus", "queries": [ { "refId": "A", "expr": expr, "range": True, } ], "range": { "from": "now-6h", "to": "now", }, } print(f"{base_url}/explore?left={urllib.parse.quote(json.dumps(left, separators=(',', ':')))}") PY } section() { printf '\n== %s ==\n' "$1" } run_optional() { local label="$1" shift printf '\n-- %s\n' "${label}" "$@" 2>&1 || true } host_capacity() { section "Debian Host" run_optional "memory" free -h run_optional "disk" df -h / /data /var/lib/docker if command -v docker >/dev/null 2>&1; then run_optional "docker disk usage" sudo docker system df run_optional "largest docker volumes" sudo docker system df -v else printf 'docker not installed\n' fi } kubernetes_capacity() { section "Kubernetes" if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then printf 'kubectl or kubeconfig unavailable\n' return 0 fi run_optional "nodes" kubectl --kubeconfig "${KUBECONFIG}" get nodes -o wide run_optional "node resource summary" kubectl --kubeconfig "${KUBECONFIG}" describe nodes run_optional "top nodes" kubectl --kubeconfig "${KUBECONFIG}" top nodes run_optional "top pods" kubectl --kubeconfig "${KUBECONFIG}" top pods -A --sort-by=memory run_optional "PVCs" kubectl --kubeconfig "${KUBECONFIG}" get pvc -A -o wide run_optional "PVs" kubectl --kubeconfig "${KUBECONFIG}" get pv -o wide printf '\n-- pods without resource requests or limits\n' kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null | awk '/cpuReq=,|memReq=,|memLimit=,/ { print }' || true } pimox_capacity() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}" section "Pimox" ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" ' set +e echo "-- host memory" free -h echo echo "-- storage" df -h / /data 2>/dev/null || df -h echo echo "-- pvesm" sudo pvesm status 2>/dev/null || true echo echo "-- VMs" sudo qm list 2>/dev/null || true echo echo "-- running VM configs" for vmid in $(sudo qm list 2>/dev/null | awk "NR > 1 {print \$1}"); do echo "VM ${vmid}" sudo qm config "${vmid}" 2>/dev/null | awk "/^(memory|cores|sockets|scsi|virtio|sata|ide)[0-9]*:|^memory:|^cores:|^sockets:/" done ' || printf 'unable to reach Pimox host %s@%s\n' "${pimox_user}" "${pimox_host}" } rpi_capacity() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}" section "RPi4" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" ' set +e docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)" echo "-- memory" free -h echo echo "-- disk" df -h / /nvme-storage "${docker_root:-/nvme-storage/docker}" 2>/dev/null || df -h echo echo "-- docker root" printf "%s\n" "${docker_root:-unknown}" echo echo "-- docker disk usage" sudo docker system df 2>/dev/null || true ' || printf 'unable to reach RPi host %s@%s\n' "${rpi_user}" "${rpi_host}" } summary() { section "Capacity Guidance" cat <<'EOF' Use this report to decide placement: - Prefer Kubernetes app workloads on Pimox workers with SSD-backed storage. - Keep Debian control-plane headroom for Docker, Gitea, registry, Ollama, and kubeadm. - Treat Orange Pi VM disks as rebuildable capacity, not durable backup storage. - Keep RPi4 focused on DNS/Uptime Kuma/light services until NVMe stability is proven. - Add capacity only when CPU, memory, and disk all have margin; storage alone is not enough. EOF } verbose_main() { host_capacity kubernetes_capacity pimox_capacity rpi_capacity summary } compact_host_capacity() { local mem_total local mem_used local mem_pct local path local line local use_pct local docker_root if command -v free >/dev/null 2>&1; then read -r mem_total mem_used < <(free -m | awk '/^Mem:/ { print $2, $3 }') if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then mem_pct=$((mem_used * 100 / mem_total)) if ((mem_pct >= 85)); then emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers." "${GRAFANA_CAPACITY_URL}&viewPanel=1" "$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')" else emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" fi else emit skip "Debian" "Memory" "unavailable" "free output could not be parsed" fi else emit skip "Debian" "Memory" "free missing" "" "Install procps." fi for path in / /data /var/lib/docker; do line="$(df -P "${path}" 2>/dev/null | awk 'NR == 2 { print $5 " " $4 }' || true)" if [[ -z "${line}" ]]; then emit skip "Debian" "Disk ${path}" "unavailable" "path not present" continue fi use_pct="${line%% *}" use_pct="${use_pct%%%}" if ((use_pct >= 90)); then emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')" elif ((use_pct >= 80)); then emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage." "${GRAFANA_CAPACITY_URL}&viewPanel=2" "$(grafana_explore_url '100 * (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker|/nvme-storage"}))')" else emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" fi done if command -v docker >/dev/null 2>&1; then docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" if [[ -n "${docker_root}" ]]; then emit ok "Debian" "Docker root" "${docker_root}" "" "Keep Docker on NVMe-backed storage." else emit warn "Debian" "Docker root" "unavailable" "docker info failed" "Check docker service." fi else emit skip "Debian" "Docker" "not installed" fi } compact_kubernetes_capacity() { local not_ready local problem_pods local missing_resources local top_nodes local node_ready_query local problem_pods_query local resource_policy_query local node_pressure_query node_ready_query="$(grafana_explore_url 'kube_node_status_condition{condition="Ready",status="true"} == 0')" problem_pods_query="$(grafana_explore_url 'sum by (namespace, pod, phase) (kube_pod_status_phase{phase!~"Running|Succeeded"} == 1)')" resource_policy_query="$(grafana_explore_url 'sum by (namespace, pod) (kube_pod_container_resource_requests{resource=~"cpu|memory"})')" node_pressure_query="$(grafana_explore_url '100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))')" if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG." return 0 fi if ! kubectl --kubeconfig "${KUBECONFIG}" get --raw=/readyz >/dev/null 2>&1; then emit warn "Kubernetes" "API" "not reachable" "readyz failed" "./jeannie start-cluster" return 0 fi not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')" if ((not_ready > 0)); then emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster" "${GRAFANA_AVAILABILITY_URL}&viewPanel=1" "${node_ready_query}" else emit ok "Kubernetes" "Node readiness" "all ready" fi problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')" if ((problem_pods > 0)); then emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide" "${GRAFANA_AVAILABILITY_URL}&viewPanel=2" "${problem_pods_query}" else emit ok "Kubernetes" "Problem pods" "none" fi missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null | awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')" if ((missing_resources > 0)); then emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget" "${GRAFANA_CAPACITY_URL}&viewPanel=3" "${resource_policy_query}" else emit ok "Kubernetes" "Resource policy" "requests/limits present" fi top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)" if [[ -n "${top_nodes}" ]]; then emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}" else emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready" "" "${GRAFANA_CAPACITY_URL}&viewPanel=1" "${node_pressure_query}" fi } compact_pimox_capacity() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" local output output="$(ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" "set +e storage_line=\"\$(sudo pvesm status 2>/dev/null | awk -v storage='${worker_storage}' '\$1 == storage { print \$7 }')\" running_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 && \$3 == \"running\" { count++ } END { print count + 0 }')\" total_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 { count++ } END { print count + 0 }')\" echo \"storage_used_pct=\${storage_line:-unknown}\" echo \"running_vms=\${running_vms:-0}\" echo \"total_vms=\${total_vms:-0}\" " 2>/dev/null || true)" if [[ -z "${output}" ]]; then emit warn "Pimox" "Host" "unreachable" "${pimox_user}@${pimox_host}" "./jeannie doctor-cluster" return 0 fi local storage_pct local running_vms local total_vms storage_pct="$(printf '%s\n' "${output}" | awk -F= '/storage_used_pct=/ { print $2 }')" running_vms="$(printf '%s\n' "${output}" | awk -F= '/running_vms=/ { print $2 }')" total_vms="$(printf '%s\n' "${output}" | awk -F= '/total_vms=/ { print $2 }')" if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then storage_pct="${storage_pct%%%}" if ((${storage_pct%.*} >= 85)); then emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage." else emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" fi else emit warn "Pimox" "Worker storage" "unknown" "${worker_storage}" "Check pvesm status." fi emit ok "Pimox" "VMs" "running=${running_vms:-0}/${total_vms:-0}" } compact_rpi_capacity() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}" local expected_docker_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" local fallback_docker_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" local output output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" ' set +e docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)" root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")" nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")" docker_root_used="$(df -P "${docker_root:-/nvme-storage/docker}" 2>/dev/null | awk "NR == 2 { print \$5 }")" echo "docker_root=${docker_root:-unknown}" echo "root_used=${root_used:-unknown}" echo "nvme_used=${nvme_used:-unknown}" echo "docker_root_used=${docker_root_used:-unknown}" ' 2>/dev/null || true)" if [[ -z "${output}" ]]; then emit warn "RPi4" "Host" "unreachable" "${rpi_user}@${rpi_host}" "./jeannie doctor-rpi" return 0 fi local docker_root local root_used local nvme_used local docker_root_used docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')" root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')" nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')" docker_root_used="$(printf '%s\n' "${output}" | awk -F= '/docker_root_used=/ { print $2 }')" if [[ "${docker_root}" == "${expected_docker_root}" ]]; then emit ok "RPi4" "Docker root" "${docker_root}" "used=${docker_root_used:-unknown}" elif [[ "${docker_root}" == "${fallback_docker_root}" ]]; then emit warn "RPi4" "Docker root" "${docker_root}" "expected ${expected_docker_root}" "./jeannie doctor-rpi" else emit warn "RPi4" "Docker root" "${docker_root:-unknown}" "expected ${expected_docker_root}" "./jeannie doctor-rpi" fi emit ok "RPi4" "Root disk" "${root_used:-unknown}" if [[ "${nvme_used:-unknown}" = "unknown" ]]; then emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi" else emit ok "RPi4" "NVMe mount" "${nvme_used}" fi } compact_guidance() { emit ok "Guidance" "Placement" "Prefer app workloads on Pimox workers" "Keep Debian headroom for Gitea, registry, Docker, kubeadm, and Ollama." emit ok "Guidance" "Storage" "Treat Pimox VM disks as rebuildable" "Durable data still needs explicit backup/restore paths." } compact_main() { { compact_host_capacity compact_kubernetes_capacity compact_pimox_capacity compact_rpi_capacity compact_guidance } | "${REPO_ROOT}/scripts/report-render" --title "Homelab Capacity" "${DETAIL_ARGS[@]}" } if [[ "${VERBOSE}" == "true" ]]; then verbose_main else compact_main fi