Add homelab capacity report
This commit is contained in:
parent
1d553771f0
commit
168d61a820
|
|
@ -270,6 +270,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./jeannie status
|
./jeannie status
|
||||||
|
./jeannie capacity
|
||||||
```
|
```
|
||||||
|
|
||||||
It reports host memory/disk, systemd services, Docker Compose stacks,
|
It reports host memory/disk, systemd services, Docker Compose stacks,
|
||||||
|
|
@ -289,6 +290,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
||||||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||||
for controlled practice targets.
|
for controlled practice targets.
|
||||||
|
|
||||||
|
`capacity` is the placement report for the hardware you already have. It
|
||||||
|
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
||||||
|
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
||||||
|
guidance for deciding what can safely run next.
|
||||||
|
|
||||||
Focused doctor commands run narrower read-only checks and print the most likely
|
Focused doctor commands run narrower read-only checks and print the most likely
|
||||||
next step:
|
next step:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -270,6 +270,7 @@ Run a read-only health snapshot from the Debian server with:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
./{{ main_script }} status
|
./{{ main_script }} status
|
||||||
|
./{{ main_script }} capacity
|
||||||
```
|
```
|
||||||
|
|
||||||
It reports host memory/disk, systemd services, Docker Compose stacks,
|
It reports host memory/disk, systemd services, Docker Compose stacks,
|
||||||
|
|
@ -289,6 +290,11 @@ only DNS, Redis, and the Debian Ollama endpoint. The security lab namespace is
|
||||||
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
isolated from the cluster and LAN while allowing DNS plus outbound HTTP/HTTPS
|
||||||
for controlled practice targets.
|
for controlled practice targets.
|
||||||
|
|
||||||
|
`capacity` is the placement report for the hardware you already have. It
|
||||||
|
summarizes Debian memory/disk/Docker usage, Kubernetes node and PVC usage,
|
||||||
|
Pimox VM/storage allocation, RPi Docker/disk state, and ends with placement
|
||||||
|
guidance for deciding what can safely run next.
|
||||||
|
|
||||||
Focused doctor commands run narrower read-only checks and print the most likely
|
Focused doctor commands run narrower read-only checks and print the most likely
|
||||||
next step:
|
next step:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,11 @@ the older detailed tables. The cascade includes "what broke" signals for recent
|
||||||
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
deployment readiness, pod restarts, node pressure, disk pressure, Traefik
|
||||||
5xx/404 log evidence, and recent Gitea errors.
|
5xx/404 log evidence, and recent Gitea errors.
|
||||||
|
|
||||||
|
`capacity`
|
||||||
|
: Print a read-only placement report covering Debian memory/disk/Docker usage,
|
||||||
|
Kubernetes node and PVC usage, Pimox VM/storage allocation, RPi Docker/disk
|
||||||
|
state, and practical placement guidance.
|
||||||
|
|
||||||
Prometheus alerts
|
Prometheus alerts
|
||||||
: High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app.
|
: High-signal alert rules are managed as the `apps/homelab-alerts` GitOps app.
|
||||||
They cover node readiness, restart storms, unavailable deployments, storage
|
They cover node readiness, restart storms, unavailable deployments, storage
|
||||||
|
|
|
||||||
10
jeannie
10
jeannie
|
|
@ -5962,6 +5962,11 @@ kubeconfig_readonly() {
|
||||||
"${REPO_ROOT}/scripts/kubeconfig-readonly"
|
"${REPO_ROOT}/scripts/kubeconfig-readonly"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
capacity_report() {
|
||||||
|
require_debian_server "capacity"
|
||||||
|
"${REPO_ROOT}/scripts/capacity-report"
|
||||||
|
}
|
||||||
|
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
up)
|
up)
|
||||||
up
|
up
|
||||||
|
|
@ -5981,6 +5986,9 @@ case "${1:-}" in
|
||||||
status)
|
status)
|
||||||
status_report
|
status_report
|
||||||
;;
|
;;
|
||||||
|
capacity)
|
||||||
|
capacity_report
|
||||||
|
;;
|
||||||
validate)
|
validate)
|
||||||
validate_homelab
|
validate_homelab
|
||||||
;;
|
;;
|
||||||
|
|
@ -6125,7 +6133,7 @@ case "${1:-}" in
|
||||||
echo "Log: ${JEANNIE_LOG_FILE}"
|
echo "Log: ${JEANNIE_LOG_FILE}"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
|
||||||
exit 1
|
exit 1
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,119 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
KUBECONFIG="${KUBECONFIG:-${KUBECONFIG_PATH:-${HOME}/.kube/config}}"
|
||||||
|
|
||||||
|
section() {
|
||||||
|
printf '\n== %s ==\n' "$1"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_optional() {
|
||||||
|
local label="$1"
|
||||||
|
shift
|
||||||
|
|
||||||
|
printf '\n-- %s\n' "${label}"
|
||||||
|
"$@" 2>&1 || true
|
||||||
|
}
|
||||||
|
|
||||||
|
host_capacity() {
|
||||||
|
section "Debian Host"
|
||||||
|
run_optional "memory" free -h
|
||||||
|
run_optional "disk" df -h / /data /var/lib/docker
|
||||||
|
if command -v docker >/dev/null 2>&1; then
|
||||||
|
run_optional "docker disk usage" sudo docker system df
|
||||||
|
run_optional "largest docker volumes" sudo docker system df -v
|
||||||
|
else
|
||||||
|
printf 'docker not installed\n'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
kubernetes_capacity() {
|
||||||
|
section "Kubernetes"
|
||||||
|
if ! command -v kubectl >/dev/null 2>&1 || [[ ! -s "${KUBECONFIG}" ]]; then
|
||||||
|
printf 'kubectl or kubeconfig unavailable\n'
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
run_optional "nodes" kubectl --kubeconfig "${KUBECONFIG}" get nodes -o wide
|
||||||
|
run_optional "node resource summary" kubectl --kubeconfig "${KUBECONFIG}" describe nodes
|
||||||
|
run_optional "top nodes" kubectl --kubeconfig "${KUBECONFIG}" top nodes
|
||||||
|
run_optional "top pods" kubectl --kubeconfig "${KUBECONFIG}" top pods -A --sort-by=memory
|
||||||
|
run_optional "PVCs" kubectl --kubeconfig "${KUBECONFIG}" get pvc -A -o wide
|
||||||
|
run_optional "PVs" kubectl --kubeconfig "${KUBECONFIG}" get pv -o wide
|
||||||
|
|
||||||
|
printf '\n-- pods without resource requests or limits\n'
|
||||||
|
kubectl --kubeconfig "${KUBECONFIG}" get pods -A -o jsonpath='{range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{" "}{range .spec.containers[*]}{.name}{":cpuReq="}{.resources.requests.cpu}{",memReq="}{.resources.requests.memory}{",memLimit="}{.resources.limits.memory}{" "}{end}{"\n"}{end}' 2>/dev/null |
|
||||||
|
awk '/cpuReq=,|memReq=,|memLimit=,/ { print }' || true
|
||||||
|
}
|
||||||
|
|
||||||
|
pimox_capacity() {
|
||||||
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
||||||
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
||||||
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-${HOME}/.ssh/id_ed25519}}"
|
||||||
|
|
||||||
|
section "Pimox"
|
||||||
|
ssh -i "${pimox_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${pimox_user}@${pimox_host}" '
|
||||||
|
set +e
|
||||||
|
echo "-- host memory"
|
||||||
|
free -h
|
||||||
|
echo
|
||||||
|
echo "-- storage"
|
||||||
|
df -h / /data 2>/dev/null || df -h
|
||||||
|
echo
|
||||||
|
echo "-- pvesm"
|
||||||
|
sudo pvesm status 2>/dev/null || true
|
||||||
|
echo
|
||||||
|
echo "-- VMs"
|
||||||
|
sudo qm list 2>/dev/null || true
|
||||||
|
echo
|
||||||
|
echo "-- running VM configs"
|
||||||
|
for vmid in $(sudo qm list 2>/dev/null | awk "NR > 1 {print \$1}"); do
|
||||||
|
echo "VM ${vmid}"
|
||||||
|
sudo qm config "${vmid}" 2>/dev/null | awk "/^(memory|cores|sockets|scsi|virtio|sata|ide)[0-9]*:|^memory:|^cores:|^sockets:/"
|
||||||
|
done
|
||||||
|
' || printf 'unable to reach Pimox host %s@%s\n' "${pimox_user}" "${pimox_host}"
|
||||||
|
}
|
||||||
|
|
||||||
|
rpi_capacity() {
|
||||||
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
||||||
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
||||||
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-${HOME}/.ssh/id_ed25519}}"
|
||||||
|
|
||||||
|
section "RPi4"
|
||||||
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" '
|
||||||
|
set +e
|
||||||
|
echo "-- memory"
|
||||||
|
free -h
|
||||||
|
echo
|
||||||
|
echo "-- disk"
|
||||||
|
df -h / /nvme-storage /var/lib/docker 2>/dev/null || df -h
|
||||||
|
echo
|
||||||
|
echo "-- docker root"
|
||||||
|
sudo docker info --format "{{.DockerRootDir}}" 2>/dev/null || true
|
||||||
|
echo
|
||||||
|
echo "-- docker disk usage"
|
||||||
|
sudo docker system df 2>/dev/null || true
|
||||||
|
' || printf 'unable to reach RPi host %s@%s\n' "${rpi_user}" "${rpi_host}"
|
||||||
|
}
|
||||||
|
|
||||||
|
summary() {
|
||||||
|
section "Capacity Guidance"
|
||||||
|
cat <<'EOF'
|
||||||
|
Use this report to decide placement:
|
||||||
|
- Prefer Kubernetes app workloads on Pimox workers with SSD-backed storage.
|
||||||
|
- Keep Debian control-plane headroom for Docker, Gitea, registry, Ollama, and kubeadm.
|
||||||
|
- Treat Orange Pi VM disks as rebuildable capacity, not durable backup storage.
|
||||||
|
- Keep RPi4 focused on DNS/Uptime Kuma/light services until NVMe stability is proven.
|
||||||
|
- Add capacity only when CPU, memory, and disk all have margin; storage alone is not enough.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
host_capacity
|
||||||
|
kubernetes_capacity
|
||||||
|
pimox_capacity
|
||||||
|
rpi_capacity
|
||||||
|
summary
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Loading…
Reference in New Issue