diff --git a/README.md b/README.md index 548b792..c5014c0 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,7 @@ Run a read-only health snapshot from the Debian server with: ./jeannie backup-status ./jeannie synthetic-checks ./jeannie resource-budget +./jeannie control-plane status ./jeannie artifact-cache status ./jeannie blockchain-devnet status ./jeannie blockchain-test @@ -368,7 +369,14 @@ registry push/pull smoke test is opt-in with `resource-budget` checks the report-only policy in `infra/resource-budgets.yml` against Kubernetes pod requests/limits and Debian host disk/memory signals. Use -it before adding heavier workloads so the HP laptop remains usable. +it before adding heavier workloads so the HP laptop remains usable. By default +it summarizes missing requests/limits by namespace; use +`./jeannie resource-budget --details` for every pod/container. + +`control-plane` shows or manages the Debian Kubernetes control-plane scheduling +taint. After Pimox workers are stable, use `./jeannie control-plane taint` +to keep normal app/data-plane pods off the Debian host while control-plane and +DaemonSet workloads continue using their normal tolerations. `artifact-cache` manages an optional Debian-host cache stack in `infra/artifact-cache` with apt-cacher-ng and a Docker Hub registry pull-through diff --git a/README.md.tmpl b/README.md.tmpl index a9fd629..906e2e2 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -282,6 +282,7 @@ Run a read-only health snapshot from the Debian server with: ./{{ main_script }} backup-status ./{{ main_script }} synthetic-checks ./{{ main_script }} resource-budget +./{{ main_script }} control-plane status ./{{ main_script }} artifact-cache status ./{{ main_script }} blockchain-devnet status ./{{ main_script }} blockchain-test @@ -368,7 +369,14 @@ registry push/pull smoke test is opt-in with `resource-budget` checks the report-only policy in `infra/resource-budgets.yml` against Kubernetes pod requests/limits and Debian host disk/memory signals. Use -it before adding heavier workloads so the HP laptop remains usable. +it before adding heavier workloads so the HP laptop remains usable. By default +it summarizes missing requests/limits by namespace; use +`./{{ main_script }} resource-budget --details` for every pod/container. + +`control-plane` shows or manages the Debian Kubernetes control-plane scheduling +taint. After Pimox workers are stable, use `./{{ main_script }} control-plane taint` +to keep normal app/data-plane pods off the Debian host while control-plane and +DaemonSet workloads continue using their normal tolerations. `artifact-cache` manages an optional Debian-host cache stack in `infra/artifact-cache` with apt-cacher-ng and a Docker Hub registry pull-through diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 4807bc4..59d82d7 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -116,7 +116,13 @@ opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`. `resource-budget` : Check the report-only resource policy in `infra/resource-budgets.yml` against -Kubernetes pod requests/limits and Debian disk/memory signals. +Kubernetes pod requests/limits and Debian disk/memory signals. Use +`resource-budget --details` for the full pod/container list. + +`control-plane {status|taint|untaint|pods}` +: Show or manage the Debian Kubernetes control-plane scheduling taint. Use +`control-plane taint` after Pimox workers are stable to keep normal app/data +plane pods off the Debian host. `artifact-cache {status|up|down|instructions}` : Manage the optional Debian-host artifact cache stack for apt packages and diff --git a/jeannie b/jeannie index 44bce14..e58a2a8 100755 --- a/jeannie +++ b/jeannie @@ -5999,7 +5999,12 @@ synthetic_checks() { resource_budget() { require_debian_server "resource-budget" - "${REPO_ROOT}/scripts/resource-budget" + "${REPO_ROOT}/scripts/resource-budget" "${@:2}" +} + +control_plane() { + require_debian_server "control-plane" + "${REPO_ROOT}/scripts/control-plane" "${@:2}" } artifact_cache() { @@ -6120,6 +6125,8 @@ Observability And GitOps synthetic-checks Run end-to-end service probes. resource-budget Report Kubernetes resource request/limit gaps. route-inventory Report routes and Uptime Kuma coverage. + control-plane {status|taint|untaint|pods} + Keep normal pods off the Debian control plane. Recovery And State backup-gitea Back up the Debian-hosted Gitea data. @@ -6221,7 +6228,10 @@ case "${1:-}" in synthetic_checks ;; resource-budget) - resource_budget + resource_budget "$@" + ;; + control-plane) + control_plane "$@" ;; artifact-cache) artifact_cache "$@" diff --git a/scripts/control-plane b/scripts/control-plane new file mode 100755 index 0000000..de23daf --- /dev/null +++ b/scripts/control-plane @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -euo pipefail + +KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" +CONTROL_PLANE_NODE="${LAB_CONTROL_PLANE_NODE:-debian}" +TAINT_KEY="${LAB_CONTROL_PLANE_TAINT_KEY:-node-role.kubernetes.io/control-plane}" +TAINT_EFFECT="${LAB_CONTROL_PLANE_TAINT_EFFECT:-NoSchedule}" + +usage() { + cat </dev/null 2>&1; then + echo "kubectl is required." >&2 + exit 1 + fi + if [ ! -s "$KUBECONFIG_PATH" ]; then + echo "kubeconfig is missing: $KUBECONFIG_PATH" >&2 + exit 1 + fi + if ! kubectl --kubeconfig "$KUBECONFIG_PATH" get node "$CONTROL_PLANE_NODE" >/dev/null 2>&1; then + echo "control-plane node not found: ${CONTROL_PLANE_NODE}" >&2 + echo "Set LAB_CONTROL_PLANE_NODE if the node has a different name." >&2 + exit 1 + fi +} + +show_taints() { + kubectl --kubeconfig "$KUBECONFIG_PATH" get node "$CONTROL_PLANE_NODE" \ + -o jsonpath='{range .spec.taints[*]}{.key}{"="}{.value}{":"}{.effect}{"\n"}{end}' | + sed '/^$/d' || true +} + +show_non_system_pods() { + kubectl --kubeconfig "$KUBECONFIG_PATH" get pods -A \ + --field-selector "spec.nodeName=${CONTROL_PLANE_NODE}" \ + -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers | + awk '$1 !~ /^(kube-system|calico-system|tigera-operator)$/ { print }' +} + +case "${1:-status}" in + status) + require_kubectl + echo "Node: ${CONTROL_PLANE_NODE}" + echo + echo "Taints:" + if ! show_taints | sed 's/^/ /'; then + true + fi + echo + echo "Non-system pods currently on control plane:" + if ! show_non_system_pods | sed 's/^/ /'; then + true + fi + ;; + taint) + require_kubectl + kubectl --kubeconfig "$KUBECONFIG_PATH" taint node "$CONTROL_PLANE_NODE" "${TAINT_KEY}:${TAINT_EFFECT}" --overwrite + echo + echo "Current taints:" + show_taints | sed 's/^/ /' + ;; + untaint) + require_kubectl + kubectl --kubeconfig "$KUBECONFIG_PATH" taint node "$CONTROL_PLANE_NODE" "${TAINT_KEY}:${TAINT_EFFECT}-" || true + echo + echo "Current taints:" + show_taints | sed 's/^/ /' + ;; + pods) + require_kubectl + show_non_system_pods + ;; + -h|--help|help) + usage + ;; + *) + usage >&2 + exit 1 + ;; +esac diff --git a/scripts/resource-budget b/scripts/resource-budget index a260912..3625ef4 100755 --- a/scripts/resource-budget +++ b/scripts/resource-budget @@ -4,10 +4,40 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" BUDGET_FILE="${HOMELAB_RESOURCE_BUDGET_FILE:-${REPO_ROOT}/infra/resource-budgets.yml}" KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}" +DETAILS=false +MAX_EXAMPLES="${LAB_RESOURCE_BUDGET_EXAMPLES:-12}" failures=0 warnings=0 +usage() { + cat <<'EOF' +Usage: ./jeannie resource-budget [--details] + +Print host and Kubernetes resource budget checks. + +Options: + --details Show every pod/container missing requests or limits. +EOF +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --details|--verbose) + DETAILS=true + ;; + -h|--help|help) + usage + exit 0 + ;; + *) + usage >&2 + exit 2 + ;; + esac + shift +done + section() { printf '\n== %s ==\n' "$1" } @@ -58,12 +88,15 @@ kubernetes_budget() { return 0 fi - python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" <<'PY' + python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" "$DETAILS" "$MAX_EXAMPLES" <<'PY' import json import subprocess import sys +from collections import Counter, defaultdict -budget_file, kubeconfig = sys.argv[1:3] +budget_file, kubeconfig, details_arg, max_examples_arg = sys.argv[1:5] +show_details = details_arg.lower() == "true" +max_examples = int(max_examples_arg) try: import yaml @@ -120,6 +153,7 @@ raw = subprocess.check_output( pods = json.loads(raw).get("items", []) namespace_totals = {} missing = [] +missing_by_namespace = defaultdict(Counter) for pod in pods: ns = pod["metadata"]["namespace"] @@ -134,8 +168,10 @@ for pod in pods: namespace_totals[ns]["memory"] += mem_to_mib(requests.get("memory")) if cluster_budget.get("require_requests", True) and ("cpu" not in requests or "memory" not in requests): missing.append(f"{ns}/{pod_name}:{name} missing requests") + missing_by_namespace[ns]["requests"] += 1 if cluster_budget.get("require_limits", True) and ("cpu" not in limits or "memory" not in limits): missing.append(f"{ns}/{pod_name}:{name} missing limits") + missing_by_namespace[ns]["limits"] += 1 total_cpu = sum(item["cpu"] for item in namespace_totals.values()) total_memory = sum(item["memory"] for item in namespace_totals.values()) @@ -163,11 +199,30 @@ for ns, totals in sorted(namespace_totals.items()): exit_code = 1 if missing: - print("missing_resource_policy:") - for item in missing[:50]: - print(f" {item}") - if len(missing) > 50: - print(f" ... {len(missing) - 50} more") + print(f"missing_resource_policy: total={len(missing)}") + print("by_namespace:") + for ns, counts in sorted( + missing_by_namespace.items(), + key=lambda item: (item[1]["requests"] + item[1]["limits"], item[0]), + reverse=True, + ): + print(f" {ns}: requests_missing={counts['requests']} limits_missing={counts['limits']}") + + if show_details: + print("details:") + for item in missing: + print(f" {item}") + else: + print("examples:") + for item in missing[:max_examples]: + print(f" {item}") + if len(missing) > max_examples: + print(f" ... {len(missing) - max_examples} more") + print(" run ./jeannie resource-budget --details for the full list") + print("next_steps:") + print(" 1. Add requests before limits for app workloads you own.") + print(" 2. Leave kube-system/control-plane static pods for a separate pass.") + print(" 3. Use ./jeannie control-plane taint after Pimox workers are Ready.") exit_code = 1 raise SystemExit(exit_code)