Convert capacity to compact report renderer
This commit is contained in:
parent
a7156663d9
commit
4d7836475a
|
|
@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
||||||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||||
for controlled practice targets.
|
for controlled practice targets.
|
||||||
|
|
||||||
`capacity` is the placement report for the hardware you already have. It
|
`capacity` is the placement report for the hardware you already have. By
|
||||||
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
default it uses the compact report renderer and shows summarized Debian,
|
||||||
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use
|
||||||
guidance for deciding what can safely run next.
|
`./jeannie capacity --details`, `--json`, `--only problems`, or
|
||||||
|
`--verbose` for the older full capacity dump.
|
||||||
|
|
||||||
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
||||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||||
|
|
|
||||||
|
|
@ -305,10 +305,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
||||||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||||
for controlled practice targets.
|
for controlled practice targets.
|
||||||
|
|
||||||
`capacity` is the placement report for the hardware you already have. It
|
`capacity` is the placement report for the hardware you already have. By
|
||||||
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
default it uses the compact report renderer and shows summarized Debian,
|
||||||
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
Kubernetes, Pimox, RPi, and placement signals without dumping full details. Use
|
||||||
guidance for deciding what can safely run next.
|
`./{{ main_script }} capacity --details`, `--json`, `--only problems`, or
|
||||||
|
`--verbose` for the older full capacity dump.
|
||||||
|
|
||||||
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
`recover-plan` prints the disaster recovery order and lightweight prerequisite
|
||||||
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
checks, from Debian and Gitea through DNS, Pimox, Kubernetes, GitOps apps, and
|
||||||
|
|
|
||||||
|
|
@ -53,9 +53,9 @@ deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||||
5xx/404 log evidence, and recent Gitea errors.
|
5xx/404 log evidence, and recent Gitea errors.
|
||||||
|
|
||||||
`capacity`
|
`capacity`
|
||||||
: Print a read-only placement report covering Debian memory/disk/Docker usage,
|
: Print a compact read-only placement report covering Debian, Kubernetes,
|
||||||
Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk
|
Pimox, RPi, and placement signals. Use `capacity --details`, `--json`, `--only
|
||||||
state, and practical placement guidance.
|
problems`, or `--verbose` for the older full dump.
|
||||||
|
|
||||||
`recover-plan`
|
`recover-plan`
|
||||||
: Print the ordered disaster recovery checklist and lightweight prerequisite
|
: Print the ordered disaster recovery checklist and lightweight prerequisite
|
||||||
|
|
|
||||||
4
jeannie
4
jeannie
|
|
@ -5965,7 +5965,7 @@ kubeconfig_readonly() {
|
||||||
|
|
||||||
capacity_report() {
|
capacity_report() {
|
||||||
require_debian_server "capacity"
|
require_debian_server "capacity"
|
||||||
"${REPO_ROOT}/scripts/capacity-report"
|
"${REPO_ROOT}/scripts/capacity-report" "${@:2}"
|
||||||
}
|
}
|
||||||
|
|
||||||
recover_plan() {
|
recover_plan() {
|
||||||
|
|
@ -6071,7 +6071,7 @@ case "${1:-}" in
|
||||||
status_report
|
status_report
|
||||||
;;
|
;;
|
||||||
capacity)
|
capacity)
|
||||||
capacity_report
|
capacity_report "$@"
|
||||||
;;
|
;;
|
||||||
recover-plan)
|
recover-plan)
|
||||||
recover_plan
|
recover_plan
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,51 @@
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||||
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
|
||||||
|
DETAIL_ARGS=()
|
||||||
|
VERBOSE=false
|
||||||
|
|
||||||
|
while (($# > 0)); do
|
||||||
|
case "$1" in
|
||||||
|
--verbose)
|
||||||
|
VERBOSE=true
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--details|--json)
|
||||||
|
DETAIL_ARGS+=("$1")
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
--only)
|
||||||
|
DETAIL_ARGS+=("$1" "${2:-all}")
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
cat <<'EOF'
|
||||||
|
Usage: ./jeannie capacity [--verbose] [--details] [--json] [--only failures|warnings|problems|all]
|
||||||
|
|
||||||
|
Default output is a compact reusable report. Use --verbose for the older full
|
||||||
|
capacity dump.
|
||||||
|
EOF
|
||||||
|
exit 0
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "unknown capacity option: $1" >&2
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
emit() {
|
||||||
|
local status="$1"
|
||||||
|
local area="$2"
|
||||||
|
local check="$3"
|
||||||
|
local summary="$4"
|
||||||
|
local detail="${5:-}"
|
||||||
|
local hint="${6:-}"
|
||||||
|
|
||||||
|
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "${status}" "${area}" "${check}" "${summary}" "${detail}" "${hint}"
|
||||||
|
}
|
||||||
|
|
||||||
section() {
|
section() {
|
||||||
printf '\n== %s ==\n' "$1"
|
printf '\n== %s ==\n' "$1"
|
||||||
|
|
@ -108,7 +153,7 @@ Use this report to decide placement:
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
main() {
|
verbose_main() {
|
||||||
host_capacity
|
host_capacity
|
||||||
kubernetes_capacity
|
kubernetes_capacity
|
||||||
pimox_capacity
|
pimox_capacity
|
||||||
|
|
@ -116,4 +161,199 @@ main() {
|
||||||
summary
|
summary
|
||||||
}
|
}
|
||||||
|
|
||||||
main "$@"
|
compact_host_capacity() {
|
||||||
|
local mem_total
|
||||||
|
local mem_used
|
||||||
|
local mem_pct
|
||||||
|
local path
|
||||||
|
local line
|
||||||
|
local use_pct
|
||||||
|
local docker_root
|
||||||
|
|
||||||
|
if command -v free >/dev/null 2>&1; then
|
||||||
|
read -r mem_total mem_used < <(free -m | awk '/^Mem:/ { print $2, $3 }')
|
||||||
|
if [[ -n "${mem_total:-}" && "${mem_total}" -gt 0 ]]; then
|
||||||
|
mem_pct=$((mem_used * 100 / mem_total))
|
||||||
|
if ((mem_pct >= 85)); then
|
||||||
|
emit warn "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB" "Move workloads to Pimox workers or stop nonessential containers."
|
||||||
|
else
|
||||||
|
emit ok "Debian" "Memory" "used=${mem_pct}%" "${mem_used}MiB/${mem_total}MiB"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
emit skip "Debian" "Memory" "unavailable" "free output could not be parsed"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
emit skip "Debian" "Memory" "free missing" "" "Install procps."
|
||||||
|
fi
|
||||||
|
|
||||||
|
for path in / /data /var/lib/docker; do
|
||||||
|
line="$(df -P "${path}" 2>/dev/null | awk 'NR == 2 { print $5 " " $4 }' || true)"
|
||||||
|
if [[ -z "${line}" ]]; then
|
||||||
|
emit skip "Debian" "Disk ${path}" "unavailable" "path not present"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
use_pct="${line%% *}"
|
||||||
|
use_pct="${use_pct%%%}"
|
||||||
|
if ((use_pct >= 90)); then
|
||||||
|
emit fail "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Free disk or move data before deploying more services."
|
||||||
|
elif ((use_pct >= 80)); then
|
||||||
|
emit warn "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }" "Review Docker volumes, backups, and registry usage."
|
||||||
|
else
|
||||||
|
emit ok "Debian" "Disk ${path}" "used=${use_pct}%" "available blocks: ${line#* }"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
docker_root="$(docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
|
||||||
|
if [[ -n "${docker_root}" ]]; then
|
||||||
|
emit ok "Debian" "Docker root" "${docker_root}" "" "Keep Docker on NVMe-backed storage."
|
||||||
|
else
|
||||||
|
emit warn "Debian" "Docker root" "unavailable" "docker info failed" "Check docker service."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
emit skip "Debian" "Docker" "not installed"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
compact_kubernetes_capacity() {
|
||||||
|
local not_ready
|
||||||
|
local problem_pods
|
||||||
|
local missing_resources
|
||||||
|
local top_nodes
|
||||||
|
|
||||||
|
if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then
|
||||||
|
emit warn "Kubernetes" "API" "unavailable" "kubectl or kubeconfig missing" "Start the cluster or set KUBECONFIG."
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if ! kubectl --kubeconfig "${KUBECONFIG}" get --raw=/readyz >/dev/null 2>&1; then
|
||||||
|
emit warn "Kubernetes" "API" "not reachable" "readyz failed" "./jeannie start-cluster"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
not_ready="$(kubectl --kubeconfig "${KUBECONFIG}" get nodes --no-headers 2>/dev/null | awk '$2 !~ /Ready/ { count++ } END { print count + 0 }')"
|
||||||
|
if ((not_ready > 0)); then
|
||||||
|
emit fail "Kubernetes" "Node readiness" "${not_ready} not ready" "" "./jeannie doctor-cluster"
|
||||||
|
else
|
||||||
|
emit ok "Kubernetes" "Node readiness" "all ready"
|
||||||
|
fi
|
||||||
|
|
||||||
|
problem_pods="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { count++ } END { print count + 0 }')"
|
||||||
|
if ((problem_pods > 0)); then
|
||||||
|
emit warn "Kubernetes" "Problem pods" "${problem_pods}" "pods not Running/Completed" "kubectl get pods -A -o wide"
|
||||||
|
else
|
||||||
|
emit ok "Kubernetes" "Problem pods" "none"
|
||||||
|
fi
|
||||||
|
|
||||||
|
missing_resources="$(kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
||||||
|
awk '/cpuReq=,|memReq=,|memLimit=,/ { count++ } END { print count + 0 }')"
|
||||||
|
if ((missing_resources > 0)); then
|
||||||
|
emit warn "Kubernetes" "Resource policy" "${missing_resources} pods missing requests/limits" "" "./jeannie resource-budget"
|
||||||
|
else
|
||||||
|
emit ok "Kubernetes" "Resource policy" "requests/limits present"
|
||||||
|
fi
|
||||||
|
|
||||||
|
top_nodes="$(kubectl --kubeconfig "${KUBECONFIG}" top nodes 2>/dev/null | awk 'NR > 1 { print $1 ":cpu=" $3 ",mem=" $5 }' | paste -sd ';' - || true)"
|
||||||
|
if [[ -n "${top_nodes}" ]]; then
|
||||||
|
emit ok "Kubernetes" "Node pressure" "metrics available" "${top_nodes}"
|
||||||
|
else
|
||||||
|
emit skip "Kubernetes" "Node pressure" "metrics unavailable" "metrics-server may not be installed or ready"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
compact_pimox_capacity() {
|
||||||
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||||
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||||
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}"
|
||||||
|
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
|
||||||
|
local output
|
||||||
|
|
||||||
|
output="$(ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" "set +e
|
||||||
|
storage_line=\"\$(sudo pvesm status 2>/dev/null | awk -v storage='${worker_storage}' '\$1 == storage { print \$7 }')\"
|
||||||
|
running_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 && \$3 == \"running\" { count++ } END { print count + 0 }')\"
|
||||||
|
total_vms=\"\$(sudo qm list 2>/dev/null | awk 'NR > 1 { count++ } END { print count + 0 }')\"
|
||||||
|
echo \"storage_used_pct=\${storage_line:-unknown}\"
|
||||||
|
echo \"running_vms=\${running_vms:-0}\"
|
||||||
|
echo \"total_vms=\${total_vms:-0}\"
|
||||||
|
" 2>/dev/null || true)"
|
||||||
|
|
||||||
|
if [[ -z "${output}" ]]; then
|
||||||
|
emit warn "Pimox" "Host" "unreachable" "${pimox_user}@${pimox_host}" "./jeannie doctor-cluster"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local storage_pct
|
||||||
|
local running_vms
|
||||||
|
local total_vms
|
||||||
|
storage_pct="$(printf '%s\n' "${output}" | awk -F= '/storage_used_pct=/ { print $2 }')"
|
||||||
|
running_vms="$(printf '%s\n' "${output}" | awk -F= '/running_vms=/ { print $2 }')"
|
||||||
|
total_vms="$(printf '%s\n' "${output}" | awk -F= '/total_vms=/ { print $2 }')"
|
||||||
|
|
||||||
|
if [[ "${storage_pct}" =~ ^[0-9.]+%?$ ]]; then
|
||||||
|
storage_pct="${storage_pct%%%}"
|
||||||
|
if ((${storage_pct%.*} >= 85)); then
|
||||||
|
emit warn "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}" "Clean old VM disks or add storage."
|
||||||
|
else
|
||||||
|
emit ok "Pimox" "Worker storage" "used=${storage_pct}%" "${worker_storage}"
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
emit warn "Pimox" "Worker storage" "unknown" "${worker_storage}" "Check pvesm status."
|
||||||
|
fi
|
||||||
|
emit ok "Pimox" "VMs" "running=${running_vms:-0}/${total_vms:-0}"
|
||||||
|
}
|
||||||
|
|
||||||
|
compact_rpi_capacity() {
|
||||||
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
||||||
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
||||||
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}"
|
||||||
|
local output
|
||||||
|
|
||||||
|
output="$(ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" '
|
||||||
|
set +e
|
||||||
|
docker_root="$(sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true)"
|
||||||
|
root_used="$(df -P / 2>/dev/null | awk "NR == 2 { print \$5 }")"
|
||||||
|
nvme_used="$(df -P /nvme-storage 2>/dev/null | awk "NR == 2 { print \$5 }")"
|
||||||
|
echo "docker_root=${docker_root:-unknown}"
|
||||||
|
echo "root_used=${root_used:-unknown}"
|
||||||
|
echo "nvme_used=${nvme_used:-unknown}"
|
||||||
|
' 2>/dev/null || true)"
|
||||||
|
|
||||||
|
if [[ -z "${output}" ]]; then
|
||||||
|
emit warn "RPi4" "Host" "unreachable" "${rpi_user}@${rpi_host}" "./jeannie doctor-rpi"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local docker_root
|
||||||
|
local root_used
|
||||||
|
local nvme_used
|
||||||
|
docker_root="$(printf '%s\n' "${output}" | awk -F= '/docker_root=/ { print $2 }')"
|
||||||
|
root_used="$(printf '%s\n' "${output}" | awk -F= '/root_used=/ { print $2 }')"
|
||||||
|
nvme_used="$(printf '%s\n' "${output}" | awk -F= '/nvme_used=/ { print $2 }')"
|
||||||
|
emit ok "RPi4" "Docker root" "${docker_root:-unknown}"
|
||||||
|
emit ok "RPi4" "Root disk" "${root_used:-unknown}"
|
||||||
|
if [[ "${nvme_used:-unknown}" = "unknown" ]]; then
|
||||||
|
emit warn "RPi4" "NVMe mount" "unknown" "df /nvme-storage failed" "./jeannie doctor-rpi"
|
||||||
|
else
|
||||||
|
emit ok "RPi4" "NVMe mount" "${nvme_used}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
compact_guidance() {
|
||||||
|
emit ok "Guidance" "Placement" "Prefer app workloads on Pimox workers" "Keep Debian headroom for Gitea, registry, Docker, kubeadm, and Ollama."
|
||||||
|
emit ok "Guidance" "Storage" "Treat Pimox VM disks as rebuildable" "Durable data still needs explicit backup/restore paths."
|
||||||
|
}
|
||||||
|
|
||||||
|
compact_main() {
|
||||||
|
{
|
||||||
|
compact_host_capacity
|
||||||
|
compact_kubernetes_capacity
|
||||||
|
compact_pimox_capacity
|
||||||
|
compact_rpi_capacity
|
||||||
|
compact_guidance
|
||||||
|
} | "${REPO_ROOT}/scripts/report-render" --title "Homelab Capacity" "${DETAIL_ARGS[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${VERBOSE}" == "true" ]]; then
|
||||||
|
verbose_main
|
||||||
|
else
|
||||||
|
compact_main
|
||||||
|
fi
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue