Convert capacity to compact report renderer
This commit is contained in:
juvdiaz
parent
a7156663d9
commit
4d7836475a
|
|
@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
|||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||
for controlled practice targets.
|
||||
|
||||
`capacity` is the placement report for the hardware you already have. It
|
||||
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
||||
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
||||
guidance for deciding what can safely run next.
|
||||
`capacity` is the placement report for the hardware you already have. By
|
||||
default it uses the compact report renderer and shows summarized Debian,
|
||||
Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use
|
||||
`./jeannie capacity --details`, `--json`, `--only problems`, or
|
||||
`--verbose` for the older full capacity dump.
|
||||
|
||||
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||
|
|
|
|||
|
|
@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
|||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||
for controlled practice targets.
|
||||
|
||||
`capacity` is the placement report for the hardware you already have. It
|
||||
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
||||
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
||||
guidance for deciding what can safely run next.
|
||||
`capacity` is the placement report for the hardware you already have. By
|
||||
default it uses the compact report renderer and shows summarized Debian,
|
||||
Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use
|
||||
`./{{ main_script }} capacity --details`, `--json`, `--only problems`, or
|
||||
`--verbose` for the older full capacity dump.
|
||||
|
||||
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||
|
|
|
|||
|
|
@ -53,9 +53,9 @@ deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
|||
5xx/404 log evidence, and recent Gitea errors.
|
||||
|
||||
`capacity`
|
||||
: Print a read-only placement report covering Debian memory/disk/Docker usage,
|
||||
Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk
|
||||
state, and practical placement guidance.
|
||||
: Print a compact read-only placement report covering Debian, Kubernetes,
|
||||
Pimox, RPi, and placement signals. Use `capacity --details`, `--json`, `--only
|
||||
problems`, or `--verbose` for the older full dump.
|
||||
|
||||
`recover-plan`
|
||||
: Print the ordered disaster recovery checklist and lightweight prerequisite
|
||||
|
|
|
|||
4
jeannie
4
jeannie
|
|
@ -5965,7 +5965,7 @@ kubeconfig_readonly() {
|
|||
|
||||
capacity_report() {
|
||||
require_debian_server "capacity"
|
||||
"${REPO_ROOT}/scripts/capacity-report"
|
||||
"${REPO_ROOT}/scripts/capacity-report" "${@:2}"
|
||||
}
|
||||
|
||||
recover_plan() {
|
||||
|
|
@ -6071,7 +6071,7 @@ case "${1:-}" in
|
|||
status_report
|
||||
;;
|
||||
capacity)
|
||||
capacity_report
|
||||
capacity_report "$@"
|
||||
;;
|
||||
recover-plan)
|
||||
recover_plan
|
||||
|
|
|
|||
|
|
@ -2,6 +2,51 @@
|
|||
set -euo pipefail
|
||||
|
||||
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
DETAIL_ARGS=()
|
||||
VERBOSE=false
|
||||
|
||||
while (($# > 0)); do
|
||||
case "$1" in
|
||||
--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
--details|--json)
|
||||
DETAIL_ARGS+=("$1")
|
||||
shift
|
||||
;;
|
||||
--only)
|
||||
DETAIL_ARGS+=("$1" "${2:-all}")
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
cat <<'EOF'
|
||||
Usage: ./jeannie capacity [--verbose] [--details] [--json] [--only failures|warnings|problems|all]
|
||||
|
||||
Default output is a compact reusable report. Use --verbose for the older full
|
||||
capacity dump.
|
||||
EOF
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown capacity option: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
emit() {
|
||||
local status="$1"
|
||||
local area="$2"
|
||||
local check="$3"
|
||||
local summary="$4"
|
||||
local detail="${5:-}"
|
||||
local hint="${6:-}"
|
||||
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${hint}"
|
||||
}
|
||||
|
||||
section() {
|
||||
printf '\n== %s ==\n' "$1"
|
||||
|
|
@ -108,7 +153,7 @@ Use this report to decide placement:
|
|||
EOF
|
||||
}
|
||||
|
||||
main() {
|
||||
verbose_main() {
|
||||
host_capacity
|
||||
kubernetes_capacity
|
||||
pimox_capacity
|
||||
|
|
@ -116,4 +161,199 @@ main() {
|
|||
summary
|
||||
}
|
||||
|
||||
main "$@"
|
||||
compact_host_capacity() {
|
||||
local mem_total
|
||||
local mem_used
|
||||
local mem_pct
|
||||
local path
|
||||
local line
|
||||
local use_pct
|
||||
local docker_root
|
||||
|
||||
if command -v free >/dev/null 2>&1; then
|
||||
read -r mem_total mem_used < <(free -m | awk '/^Mem:/ { print $2, $3 }')
|
||||
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
||||
mem_pct=$((mem_used * 100 / mem_total))
|
||||
if ((mem_pct >= 85)); then
|
||||
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers."
|
||||
else
|
||||
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB"
|
||||
fi
|
||||
else
|
||||
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
||||
fi
|
||||
else
|
||||
emit skip "Debian" "Memory" "free missing" "" "Install procps."
|
||||
fi
|
||||
|
||||
for path in / /data /var/lib/docker; do
|
||||
line="$(df -P "${path}" 2>/dev/null | awk 'NR == 2 { print $5 " " $4 }' || true)"
|
||||
if [[ -z "${line}" ]]; then
|
||||
emit skip "Debian" "Disk ${path}" "unavailable" "path not present"
|
||||
continue
|
||||
fi
|
||||
use_pct="${line%% *}"
|
||||
use_pct="${use_pct%%%}"
|
||||
if ((use_pct >= 90)); then
|
||||
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services."
|
||||
elif ((use_pct >= 80)); then
|
||||
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage."
|
||||
else
|
||||
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }"
|
||||
fi
|
||||
done
|
||||
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
|
||||
if [[ -n "${docker_root}" ]]; then
|
||||
emit ok "Debian" "Docker root" "${docker_root}" "" "Keep Docker on NVMe-backed storage."
|
||||
else
|
||||
emit warn "Debian" "Docker root" "unavailable" "docker info failed" "Check docker service."
|
||||
fi
|
||||
else
|
||||
emit skip "Debian" "Docker" "not installed"
|
||||
fi
|
||||
}
|
||||
|
||||
compact_kubernetes_capacity() {
|
||||
local not_ready
|
||||
local problem_pods
|
||||
local missing_resources
|
||||
local top_nodes
|
||||
|
||||
if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then
|
||||
emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG."
|
||||
return 0
|
||||
fi
|
||||
if ! kubectl --kubeconfig "${KUBECONFIG}" get --raw=/readyz >/dev/null 2>&1; then
|
||||
emit warn "Kubernetes" "API" "not reachable" "readyz failed" "./jeannie start-cluster"
|
||||
return 0
|
||||
fi
|
||||
|
||||
not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')"
|
||||
if ((not_ready > 0)); then
|
||||
emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster"
|
||||
else
|
||||
emit ok "Kubernetes" "Node readiness" "all ready"
|
||||
fi
|
||||
|
||||
problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')"
|
||||
if ((problem_pods > 0)); then
|
||||
emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide"
|
||||
else
|
||||
emit ok "Kubernetes" "Problem pods" "none"
|
||||
fi
|
||||
|
||||
missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
||||
awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')"
|
||||
if ((missing_resources > 0)); then
|
||||
emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget"
|
||||
else
|
||||
emit ok "Kubernetes" "Resource policy" "requests/limits present"
|
||||
fi
|
||||
|
||||
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
||||
if [[ -n "${top_nodes}" ]]; then
|
||||
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}"
|
||||
else
|
||||
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
||||
fi
|
||||
}
|
||||
|
||||
compact_pimox_capacity() {
|
||||
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}"
|
||||
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
|
||||
local output
|
||||
|
||||
output="$(ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" "set +e
|
||||
storage_line=\"\$(sudo pvesm status 2>/dev/null | awk -v storage='${worker_storage}' '\$1 == storage { print \$7 }')\"
|
||||
running_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 && \$3 == \"running\" { count++ } END { print count + 0 }')\"
|
||||
total_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 { count++ } END { print count + 0 }')\"
|
||||
echo \"storage_used_pct=\${storage_line:-unknown}\"
|
||||
echo \"running_vms=\${running_vms:-0}\"
|
||||
echo \"total_vms=\${total_vms:-0}\"
|
||||
" 2>/dev/null || true)"
|
||||
|
||||
if [[ -z "${output}" ]]; then
|
||||
emit warn "Pimox" "Host" "unreachable" "${pimox_user}@${pimox_host}" "./jeannie doctor-cluster"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local storage_pct
|
||||
local running_vms
|
||||
local total_vms
|
||||
storage_pct="$(printf '%s\n' "${output}" | awk -F= '/storage_used_pct=/ { print $2 }')"
|
||||
running_vms="$(printf '%s\n' "${output}" | awk -F= '/running_vms=/ { print $2 }')"
|
||||
total_vms="$(printf '%s\n' "${output}" | awk -F= '/total_vms=/ { print $2 }')"
|
||||
|
||||
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
||||
storage_pct="${storage_pct%%%}"
|
||||
if ((${storage_pct%.*} >= 85)); then
|
||||
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
|
||||
else
|
||||
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
||||
fi
|
||||
else
|
||||
emit warn "Pimox" "Worker storage" "unknown" "${worker_storage}" "Check pvesm status."
|
||||
fi
|
||||
emit ok "Pimox" "VMs" "running=${running_vms:-0}/${total_vms:-0}"
|
||||
}
|
||||
|
||||
compact_rpi_capacity() {
|
||||
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
||||
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
||||
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}"
|
||||
local output
|
||||
|
||||
output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" '
|
||||
set +e
|
||||
docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)"
|
||||
root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")"
|
||||
nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")"
|
||||
echo "docker_root=${docker_root:-unknown}"
|
||||
echo "root_used=${root_used:-unknown}"
|
||||
echo "nvme_used=${nvme_used:-unknown}"
|
||||
' 2>/dev/null || true)"
|
||||
|
||||
if [[ -z "${output}" ]]; then
|
||||
emit warn "RPi4" "Host" "unreachable" "${rpi_user}@${rpi_host}" "./jeannie doctor-rpi"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local docker_root
|
||||
local root_used
|
||||
local nvme_used
|
||||
docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')"
|
||||
root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')"
|
||||
nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')"
|
||||
emit ok "RPi4" "Docker root" "${docker_root:-unknown}"
|
||||
emit ok "RPi4" "Root disk" "${root_used:-unknown}"
|
||||
if [[ "${nvme_used:-unknown}" = "unknown" ]]; then
|
||||
emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi"
|
||||
else
|
||||
emit ok "RPi4" "NVMe mount" "${nvme_used}"
|
||||
fi
|
||||
}
|
||||
|
||||
compact_guidance() {
|
||||
emit ok "Guidance" "Placement" "Prefer app workloads on Pimox workers" "Keep Debian headroom for Gitea, registry, Docker, kubeadm, and Ollama."
|
||||
emit ok "Guidance" "Storage" "Treat Pimox VM disks as rebuildable" "Durable data still needs explicit backup/restore paths."
|
||||
}
|
||||
|
||||
compact_main() {
|
||||
{
|
||||
compact_host_capacity
|
||||
compact_kubernetes_capacity
|
||||
compact_pimox_capacity
|
||||
compact_rpi_capacity
|
||||
compact_guidance
|
||||
} | "${REPO_ROOT}/scripts/report-render" --title "Homelab Capacity" "${DETAIL_ARGS[@]}"
|
||||
}
|
||||
|
||||
if [[ "${VERBOSE}" == "true" ]]; then
|
||||
verbose_main
|
||||
else
|
||||
compact_main
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in New Issue