Add resource budget policy checks

README.md

@@ -277,6 +277,7 @@ Run a read-only health snapshot from the Debian server with:
 ./jeannie release-snapshot
 ./jeannie backup-status
 ./jeannie synthetic-checks
+./jeannie resource-budget
 ./jeannie workers list
 ./jeannie change-journal list
 ./jeannie map
@@ -328,6 +329,10 @@ Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
 registry push/pull smoke test is opt-in with
 `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
 
+`resource-budget` checks the report-only policy in `infra/resource-budgets.yml`
+against Kubernetes pod requests/limits and Debian host disk/memory signals. Use
+it before adding heavier workloads so the HP laptop remains usable.
+
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
 `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the

README.md.tmpl

@@ -277,6 +277,7 @@ Run a read-only health snapshot from the Debian server with:
 ./{{ main_script }} release-snapshot
 ./{{ main_script }} backup-status
 ./{{ main_script }} synthetic-checks
+./{{ main_script }} resource-budget
 ./{{ main_script }} workers list
 ./{{ main_script }} change-journal list
 ./{{ main_script }} map
@@ -328,6 +329,10 @@ Pi-hole DNS, Traefik LAN HTTP, and temporary Kubernetes DNS/ingress pods. The
 registry push/pull smoke test is opt-in with
 `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
 
+`resource-budget` checks the report-only policy in `infra/resource-budgets.yml`
+against Kubernetes pod requests/limits and Debian host disk/memory signals. Use
+it before adding heavier workloads so the HP laptop remains usable.
+
 `workers` provides named Kubernetes/Pimox worker lifecycle operations:
 `list`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.
 Use `recreate-plan <index>` before replacing one broken Pimox worker so the

docs/jeannie.1.md

@@ -86,6 +86,10 @@ includes this as a warning signal.
 LAN HTTP, and temporary Kubernetes DNS/ingress pods. Registry push/pull is
 opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
 
+`resource-budget`
+: Check the report-only resource policy in `infra/resource-budgets.yml` against
+Kubernetes pod requests/limits and Debian disk/memory signals.
+
 `workers <command>`
 : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
 `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`.

infra/resource-budgets.yml

@@ -0,0 +1,29 @@
+# Report-only resource budgets for the current homelab hardware.
+# Tune these as the HP laptop, Pimox workers, and Raspberry Pi roles change.
+cluster:
+  max_total_cpu_millicores: 9000
+  max_total_memory_mib: 22000
+  require_requests: true
+  require_limits: true
+namespaces:
+  website-production:
+    max_cpu_millicores: 1000
+    max_memory_mib: 1024
+  demos-static:
+    max_cpu_millicores: 1000
+    max_memory_mib: 1024
+  argocd:
+    max_cpu_millicores: 2000
+    max_memory_mib: 3072
+  traefik:
+    max_cpu_millicores: 1000
+    max_memory_mib: 1024
+hosts:
+  debian:
+    min_root_free_percent: 15
+    min_data_free_percent: 20
+    max_memory_used_percent: 80
+  rpi4:
+    min_docker_free_percent: 15
+  pimox:
+    min_worker_storage_free_percent: 15

jeannie

@@ -5997,6 +5997,11 @@ synthetic_checks() {
     "${REPO_ROOT}/scripts/synthetic-checks"
 }
 
+resource_budget() {
+    require_debian_server "resource-budget"
+    "${REPO_ROOT}/scripts/resource-budget"
+}
+
 workers_manage() {
     require_debian_server "workers"
     "${REPO_ROOT}/scripts/workers" "${@:2}"
@@ -6062,6 +6067,9 @@ case "${1:-}" in
     synthetic-checks)
         synthetic_checks
         ;;
+    resource-budget)
+        resource_budget
+        ;;
     workers)
         workers_manage "$@"
         ;;
@@ -6219,7 +6227,7 @@ case "${1:-}" in
         echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     *)
-        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
+        echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|capacity|recover-plan|gitops-status|cert-check|release-snapshot|backup-status|synthetic-checks|resource-budget|workers <list|start|stop|drain|uncordon|recreate-plan|rebalance>|change-journal {list|path}|map [--dot]|validate|access-audit|kubeconfig-readonly|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-restore|drill-gitea-restore|drill-pihole-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|security-scan|security-prepare|security-zap|security-k8s|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-runtime|security-attack-path|openwrt|nuke}"
         exit 1
         ;;
 esac

scripts/resource-budget

@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+BUDGET_FILE="${HOMELAB_RESOURCE_BUDGET_FILE:-${REPO_ROOT}/infra/resource-budgets.yml}"
+KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
+
+failures=0
+warnings=0
+
+section() {
+    printf '\n== %s ==\n' "$1"
+}
+
+warn() {
+    warnings=$((warnings + 1))
+    printf 'warn - %s\n' "$1"
+}
+
+fail() {
+    failures=$((failures + 1))
+    printf 'fail - %s\n' "$1"
+}
+
+have() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+host_budget() {
+    section "Host Budget"
+
+    if have free; then
+        free -m | awk '/^Mem:/ {
+          used_pct = int(($3 / $2) * 100)
+          printf "Debian memory used        %s%%\n", used_pct
+          if (used_pct > 80) exit 2
+        }' || warn "Debian memory usage is above the default 80% budget"
+    else
+        warn "free command unavailable"
+    fi
+
+    if have df; then
+        df -P / /data 2>/dev/null | awk 'NR > 1 { gsub("%", "", $5); free = 100 - $5; printf "%-25s free=%s%%\n", $6, free }' || true
+    else
+        warn "df command unavailable"
+    fi
+}
+
+kubernetes_budget() {
+    section "Kubernetes Budget"
+
+    if ! have kubectl || [ ! -s "$KUBECONFIG_PATH" ]; then
+        warn "kubectl or kubeconfig unavailable; skipping Kubernetes budget checks"
+        return 0
+    fi
+    if ! kubectl --kubeconfig "$KUBECONFIG_PATH" get --raw=/readyz >/dev/null 2>&1; then
+        warn "Kubernetes API unavailable; skipping Kubernetes budget checks"
+        return 0
+    fi
+
+    python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" <<'PY'
+import json
+import subprocess
+import sys
+
+budget_file, kubeconfig = sys.argv[1:3]
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+def load_budget(path):
+    if yaml:
+        with open(path, encoding="utf-8") as handle:
+            return yaml.safe_load(handle) or {}
+    return {
+        "cluster": {
+            "max_total_cpu_millicores": 9000,
+            "max_total_memory_mib": 22000,
+            "require_requests": True,
+            "require_limits": True,
+        },
+        "namespaces": {},
+    }
+
+def cpu_to_m(value):
+    if not value:
+        return 0
+    text = str(value)
+    if text.endswith("m"):
+        return int(text[:-1])
+    return int(float(text) * 1000)
+
+def mem_to_mib(value):
+    if not value:
+        return 0
+    text = str(value)
+    units = {
+        "Ki": 1 / 1024,
+        "Mi": 1,
+        "Gi": 1024,
+        "Ti": 1024 * 1024,
+        "K": 1 / 1000,
+        "M": 1000 / 1024,
+        "G": 1000 * 1000 / 1024,
+    }
+    for suffix, multiplier in units.items():
+        if text.endswith(suffix):
+            return int(float(text[: -len(suffix)]) * multiplier)
+    return int(float(text) / (1024 * 1024))
+
+budget = load_budget(budget_file)
+cluster_budget = budget.get("cluster", {})
+namespace_budgets = budget.get("namespaces", {})
+raw = subprocess.check_output(
+    ["kubectl", "--kubeconfig", kubeconfig, "get", "pods", "-A", "-o", "json"],
+    text=True,
+)
+pods = json.loads(raw).get("items", [])
+namespace_totals = {}
+missing = []
+
+for pod in pods:
+    ns = pod["metadata"]["namespace"]
+    pod_name = pod["metadata"]["name"]
+    namespace_totals.setdefault(ns, {"cpu": 0, "memory": 0})
+    for container in pod.get("spec", {}).get("containers", []):
+        name = container["name"]
+        resources = container.get("resources", {})
+        requests = resources.get("requests", {})
+        limits = resources.get("limits", {})
+        namespace_totals[ns]["cpu"] += cpu_to_m(requests.get("cpu"))
+        namespace_totals[ns]["memory"] += mem_to_mib(requests.get("memory"))
+        if cluster_budget.get("require_requests", True) and ("cpu" not in requests or "memory" not in requests):
+            missing.append(f"{ns}/{pod_name}:{name} missing requests")
+        if cluster_budget.get("require_limits", True) and ("cpu" not in limits or "memory" not in limits):
+            missing.append(f"{ns}/{pod_name}:{name} missing limits")
+
+total_cpu = sum(item["cpu"] for item in namespace_totals.values())
+total_memory = sum(item["memory"] for item in namespace_totals.values())
+print(f"cluster_requests_cpu_m={total_cpu}")
+print(f"cluster_requests_memory_mib={total_memory}")
+
+exit_code = 0
+if total_cpu > int(cluster_budget.get("max_total_cpu_millicores", 9000)):
+    print("fail - cluster CPU requests exceed budget")
+    exit_code = 1
+if total_memory > int(cluster_budget.get("max_total_memory_mib", 22000)):
+    print("fail - cluster memory requests exceed budget")
+    exit_code = 1
+
+for ns, totals in sorted(namespace_totals.items()):
+    ns_budget = namespace_budgets.get(ns)
+    if not ns_budget:
+        continue
+    print(f"{ns}: cpu_m={totals['cpu']} memory_mib={totals['memory']}")
+    if totals["cpu"] > int(ns_budget.get("max_cpu_millicores", 999999)):
+        print(f"fail - {ns} CPU requests exceed budget")
+        exit_code = 1
+    if totals["memory"] > int(ns_budget.get("max_memory_mib", 999999)):
+        print(f"fail - {ns} memory requests exceed budget")
+        exit_code = 1
+
+if missing:
+    print("missing_resource_policy:")
+    for item in missing[:50]:
+        print(f"  {item}")
+    if len(missing) > 50:
+        print(f"  ... {len(missing) - 50} more")
+    exit_code = 1
+
+raise SystemExit(exit_code)
+PY
+}
+
+section "Policy"
+if [ -s "$BUDGET_FILE" ]; then
+    printf 'budget_file=%s\n' "$BUDGET_FILE"
+else
+    fail "budget file is missing: $BUDGET_FILE"
+fi
+
+host_budget
+kubernetes_budget || failures=$((failures + 1))
+
+section "Summary"
+printf 'failures=%s warnings=%s\n' "$failures" "$warnings"
+
+if [ "$failures" -gt 0 ]; then
+    exit 1
+fi