diff --git a/README.md b/README.md index 8a0a6c0..68cac5e 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie release-snapshot ./jeannie backup-status ./jeannie synthetic-checks +./jeannie resource-budget ./jeannie workers list ./jeannie change-journal list ./jeannie map @@ -328,6 +329,10 @@ Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The registry push/pull smoke test is opt-in with `LAB_SYNTHETIC_REGISTRY_PUSH=true`. +`resource-budget` checks the report-only policy in `infra/resource-budgets.yml` +against Kubernetes pod requests/limits and Debian host disk/memory signals. Use +it before adding heavier workloads so the HP laptop remains usable. + `workers` provides named Kubernetes/Pimox worker lifecycle operations: `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the diff --git a/README.md.tmpl b/README.md.tmpl index fee6239..586e748 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -277,6 +277,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} release-snapshot ./{{ main_script }} backup-status ./{{ main_script }} synthetic-checks +./{{ main_script }} resource-budget ./{{ main_script }} workers list ./{{ main_script }} change-journal list ./{{ main_script }} map @@ -328,6 +329,10 @@ Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The registry push/pull smoke test is opt-in with `LAB_SYNTHETIC_REGISTRY_PUSH=true`. +`resource-budget` checks the report-only policy in `infra/resource-budgets.yml` +against Kubernetes pod requests/limits and Debian host disk/memory signals. Use +it before adding heavier workloads so the HP laptop remains usable. + `workers` provides named Kubernetes/Pimox worker lifecycle operations: `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. Use `recreate-plan ` before replacing one broken Pimox worker so the diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 50e6f0c..55e519d 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -86,6 +86,10 @@ includes this as a warning signal. LAN HTTP, and temporary Kubernetes DNS/ingress pods. Registry push/pull is opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`. +`resource-budget` +: Check the report-only resource policy in `infra/resource-budgets.yml` against +Kubernetes pod requests/limits and Debian disk/memory signals. + `workers ` : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. diff --git a/infra/resource-budgets.yml b/infra/resource-budgets.yml new file mode 100644 index 0000000..b864c5e --- /dev/null +++ b/infra/resource-budgets.yml @@ -0,0 +1,29 @@ +# Report-only resource budgets for the current homelab hardware. +# Tune these as the HP laptop, Pimox workers, and Raspberry Pi roles change. +cluster: + max_total_cpu_millicores: 9000 + max_total_memory_mib: 22000 + require_requests: true + require_limits: true +namespaces: + website-production: + max_cpu_millicores: 1000 + max_memory_mib: 1024 + demos-static: + max_cpu_millicores: 1000 + max_memory_mib: 1024 + argocd: + max_cpu_millicores: 2000 + max_memory_mib: 3072 + traefik: + max_cpu_millicores: 1000 + max_memory_mib: 1024 +hosts: + debian: + min_root_free_percent: 15 + min_data_free_percent: 20 + max_memory_used_percent: 80 + rpi4: + min_docker_free_percent: 15 + pimox: + min_worker_storage_free_percent: 15 diff --git a/jeannie b/jeannie index 5e93ab3..1c421a8 100755 --- a/jeannie +++ b/jeannie @@ -5997,6 +5997,11 @@ synthetic_checks() { "${REPO_ROOT}/scripts/synthetic-checks" } +resource_budget() { + require_debian_server "resource-budget" + "${REPO_ROOT}/scripts/resource-budget" +} + workers_manage() { require_debian_server "workers" "${REPO_ROOT}/scripts/workers" "${@:2}" @@ -6062,6 +6067,9 @@ case "${1:-}" in synthetic-checks) synthetic_checks ;; + resource-budget) + resource_budget + ;; workers) workers_manage "$@" ;; @@ -6219,7 +6227,7 @@ case "${1:-}" in echo "Log: ${JEANNIE_LOG_FILE}" ;; *) - echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" + echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|workers |change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/resource-budget b/scripts/resource-budget new file mode 100755 index 0000000..a260912 --- /dev/null +++ b/scripts/resource-budget @@ -0,0 +1,192 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +BUDGET_FILE="${HOMELAB_RESOURCE_BUDGET_FILE:-${REPO_ROOT}/infra/resource-budgets.yml}" +KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" + +failures=0 +warnings=0 + +section() { + printf '\n== %s ==\n' "$1" +} + +warn() { + warnings=$((warnings + 1)) + printf 'warn - %s\n' "$1" +} + +fail() { + failures=$((failures + 1)) + printf 'fail - %s\n' "$1" +} + +have() { + command -v "$1" >/dev/null 2>&1 +} + +host_budget() { + section "Host Budget" + + if have free; then + free -m | awk '/^Mem:/ { + used_pct = int(($3 / $2) * 100) + printf "Debian memory used %s%%\n", used_pct + if (used_pct > 80) exit 2 + }' || warn "Debian memory usage is above the default 80% budget" + else + warn "free command unavailable" + fi + + if have df; then + df -P / /data 2>/dev/null | awk 'NR > 1 { gsub("%", "", $5); free = 100 - $5; printf "%-25s free=%s%%\n", $6, free }' || true + else + warn "df command unavailable" + fi +} + +kubernetes_budget() { + section "Kubernetes Budget" + + if ! have kubectl || [ ! -s "$KUBECONFIG_PATH" ]; then + warn "kubectl or kubeconfig unavailable; skipping Kubernetes budget checks" + return 0 + fi + if ! kubectl --kubeconfig "$KUBECONFIG_PATH" get --raw=/readyz >/dev/null 2>&1; then + warn "Kubernetes API unavailable; skipping Kubernetes budget checks" + return 0 + fi + + python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" <<'PY' +import json +import subprocess +import sys + +budget_file, kubeconfig = sys.argv[1:3] + +try: + import yaml +except ImportError: + yaml = None + +def load_budget(path): + if yaml: + with open(path, encoding="utf-8") as handle: + return yaml.safe_load(handle) or {} + return { + "cluster": { + "max_total_cpu_millicores": 9000, + "max_total_memory_mib": 22000, + "require_requests": True, + "require_limits": True, + }, + "namespaces": {}, + } + +def cpu_to_m(value): + if not value: + return 0 + text = str(value) + if text.endswith("m"): + return int(text[:-1]) + return int(float(text) * 1000) + +def mem_to_mib(value): + if not value: + return 0 + text = str(value) + units = { + "Ki": 1 / 1024, + "Mi": 1, + "Gi": 1024, + "Ti": 1024 * 1024, + "K": 1 / 1000, + "M": 1000 / 1024, + "G": 1000 * 1000 / 1024, + } + for suffix, multiplier in units.items(): + if text.endswith(suffix): + return int(float(text[: -len(suffix)]) * multiplier) + return int(float(text) / (1024 * 1024)) + +budget = load_budget(budget_file) +cluster_budget = budget.get("cluster", {}) +namespace_budgets = budget.get("namespaces", {}) +raw = subprocess.check_output( + ["kubectl", "--kubeconfig", kubeconfig, "get", "pods", "-A", "-o", "json"], + text=True, +) +pods = json.loads(raw).get("items", []) +namespace_totals = {} +missing = [] + +for pod in pods: + ns = pod["metadata"]["namespace"] + pod_name = pod["metadata"]["name"] + namespace_totals.setdefault(ns, {"cpu": 0, "memory": 0}) + for container in pod.get("spec", {}).get("containers", []): + name = container["name"] + resources = container.get("resources", {}) + requests = resources.get("requests", {}) + limits = resources.get("limits", {}) + namespace_totals[ns]["cpu"] += cpu_to_m(requests.get("cpu")) + namespace_totals[ns]["memory"] += mem_to_mib(requests.get("memory")) + if cluster_budget.get("require_requests", True) and ("cpu" not in requests or "memory" not in requests): + missing.append(f"{ns}/{pod_name}:{name} missing requests") + if cluster_budget.get("require_limits", True) and ("cpu" not in limits or "memory" not in limits): + missing.append(f"{ns}/{pod_name}:{name} missing limits") + +total_cpu = sum(item["cpu"] for item in namespace_totals.values()) +total_memory = sum(item["memory"] for item in namespace_totals.values()) +print(f"cluster_requests_cpu_m={total_cpu}") +print(f"cluster_requests_memory_mib={total_memory}") + +exit_code = 0 +if total_cpu > int(cluster_budget.get("max_total_cpu_millicores", 9000)): + print("fail - cluster CPU requests exceed budget") + exit_code = 1 +if total_memory > int(cluster_budget.get("max_total_memory_mib", 22000)): + print("fail - cluster memory requests exceed budget") + exit_code = 1 + +for ns, totals in sorted(namespace_totals.items()): + ns_budget = namespace_budgets.get(ns) + if not ns_budget: + continue + print(f"{ns}: cpu_m={totals['cpu']} memory_mib={totals['memory']}") + if totals["cpu"] > int(ns_budget.get("max_cpu_millicores", 999999)): + print(f"fail - {ns} CPU requests exceed budget") + exit_code = 1 + if totals["memory"] > int(ns_budget.get("max_memory_mib", 999999)): + print(f"fail - {ns} memory requests exceed budget") + exit_code = 1 + +if missing: + print("missing_resource_policy:") + for item in missing[:50]: + print(f" {item}") + if len(missing) > 50: + print(f" ... {len(missing) - 50} more") + exit_code = 1 + +raise SystemExit(exit_code) +PY +} + +section "Policy" +if [ -s "$BUDGET_FILE" ]; then + printf 'budget_file=%s\n' "$BUDGET_FILE" +else + fail "budget file is missing: $BUDGET_FILE" +fi + +host_budget +kubernetes_budget || failures=$((failures + 1)) + +section "Summary" +printf 'failures=%s warnings=%s\n' "$failures" "$warnings" + +if [ "$failures" -gt 0 ]; then + exit 1 +fi