Add control plane taint helper
This commit is contained in:
juvdiaz
parent
709881be86
commit
0a166f3a5d
10
README.md
10
README.md
|
|
@ -282,6 +282,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./jeannie backup-status
|
||||
./jeannie synthetic-checks
|
||||
./jeannie resource-budget
|
||||
./jeannie control-plane status
|
||||
./jeannie artifact-cache status
|
||||
./jeannie blockchain-devnet status
|
||||
./jeannie blockchain-test
|
||||
|
|
@ -368,7 +369,14 @@ registry push/pull smoke test is opt-in with
|
|||
|
||||
`resource-budget` checks the report-only policy in `infra/resource-budgets.yml`
|
||||
against Kubernetes pod requests/limits and Debian host disk/memory signals. Use
|
||||
it before adding heavier workloads so the HP laptop remains usable.
|
||||
it before adding heavier workloads so the HP laptop remains usable. By default
|
||||
it summarizes missing requests/limits by namespace; use
|
||||
`./jeannie resource-budget --details` for every pod/container.
|
||||
|
||||
`control-plane` shows or manages the Debian Kubernetes control-plane scheduling
|
||||
taint. After Pimox workers are stable, use `./jeannie control-plane taint`
|
||||
to keep normal app/data-plane pods off the Debian host while control-plane and
|
||||
DaemonSet workloads continue using their normal tolerations.
|
||||
|
||||
`artifact-cache` manages an optional Debian-host cache stack in
|
||||
`infra/artifact-cache` with apt-cacher-ng and a Docker Hub registry pull-through
|
||||
|
|
|
|||
|
|
@ -282,6 +282,7 @@ Run a read-only health snapshot from the Debian server with:
|
|||
./{{ main_script }} backup-status
|
||||
./{{ main_script }} synthetic-checks
|
||||
./{{ main_script }} resource-budget
|
||||
./{{ main_script }} control-plane status
|
||||
./{{ main_script }} artifact-cache status
|
||||
./{{ main_script }} blockchain-devnet status
|
||||
./{{ main_script }} blockchain-test
|
||||
|
|
@ -368,7 +369,14 @@ registry push/pull smoke test is opt-in with
|
|||
|
||||
`resource-budget` checks the report-only policy in `infra/resource-budgets.yml`
|
||||
against Kubernetes pod requests/limits and Debian host disk/memory signals. Use
|
||||
it before adding heavier workloads so the HP laptop remains usable.
|
||||
it before adding heavier workloads so the HP laptop remains usable. By default
|
||||
it summarizes missing requests/limits by namespace; use
|
||||
`./{{ main_script }} resource-budget --details` for every pod/container.
|
||||
|
||||
`control-plane` shows or manages the Debian Kubernetes control-plane scheduling
|
||||
taint. After Pimox workers are stable, use `./{{ main_script }} control-plane taint`
|
||||
to keep normal app/data-plane pods off the Debian host while control-plane and
|
||||
DaemonSet workloads continue using their normal tolerations.
|
||||
|
||||
`artifact-cache` manages an optional Debian-host cache stack in
|
||||
`infra/artifact-cache` with apt-cacher-ng and a Docker Hub registry pull-through
|
||||
|
|
|
|||
|
|
@ -116,7 +116,13 @@ opt-in through `LAB_SYNTHETIC_REGISTRY_PUSH=true`.
|
|||
|
||||
`resource-budget`
|
||||
: Check the report-only resource policy in `infra/resource-budgets.yml` against
|
||||
Kubernetes pod requests/limits and Debian disk/memory signals.
|
||||
Kubernetes pod requests/limits and Debian disk/memory signals. Use
|
||||
`resource-budget --details` for the full pod/container list.
|
||||
|
||||
`control-plane {status|taint|untaint|pods}`
|
||||
: Show or manage the Debian Kubernetes control-plane scheduling taint. Use
|
||||
`control-plane taint` after Pimox workers are stable to keep normal app/data
|
||||
plane pods off the Debian host.
|
||||
|
||||
`artifact-cache {status|up|down|instructions}`
|
||||
: Manage the optional Debian-host artifact cache stack for apt packages and
|
||||
|
|
|
|||
14
jeannie
14
jeannie
|
|
@ -5999,7 +5999,12 @@ synthetic_checks() {
|
|||
|
||||
resource_budget() {
|
||||
require_debian_server "resource-budget"
|
||||
"${REPO_ROOT}/scripts/resource-budget"
|
||||
"${REPO_ROOT}/scripts/resource-budget" "${@:2}"
|
||||
}
|
||||
|
||||
control_plane() {
|
||||
require_debian_server "control-plane"
|
||||
"${REPO_ROOT}/scripts/control-plane" "${@:2}"
|
||||
}
|
||||
|
||||
artifact_cache() {
|
||||
|
|
@ -6120,6 +6125,8 @@ Observability And GitOps
|
|||
synthetic-checks Run end-to-end service probes.
|
||||
resource-budget Report Kubernetes resource request/limit gaps.
|
||||
route-inventory Report routes and Uptime Kuma coverage.
|
||||
control-plane {status|taint|untaint|pods}
|
||||
Keep normal pods off the Debian control plane.
|
||||
|
||||
Recovery And State
|
||||
backup-gitea Back up the Debian-hosted Gitea data.
|
||||
|
|
@ -6221,7 +6228,10 @@ case "${1:-}" in
|
|||
synthetic_checks
|
||||
;;
|
||||
resource-budget)
|
||||
resource_budget
|
||||
resource_budget "$@"
|
||||
;;
|
||||
control-plane)
|
||||
control_plane "$@"
|
||||
;;
|
||||
artifact-cache)
|
||||
artifact_cache "$@"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,92 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
|
||||
CONTROL_PLANE_NODE="${LAB_CONTROL_PLANE_NODE:-debian}"
|
||||
TAINT_KEY="${LAB_CONTROL_PLANE_TAINT_KEY:-node-role.kubernetes.io/control-plane}"
|
||||
TAINT_EFFECT="${LAB_CONTROL_PLANE_TAINT_EFFECT:-NoSchedule}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: ./jeannie control-plane {status|taint|untaint|pods}
|
||||
|
||||
status Show node taints and non-system pods currently scheduled on ${CONTROL_PLANE_NODE}.
|
||||
taint Add ${TAINT_KEY}:${TAINT_EFFECT} to keep normal app/data-plane pods off the control plane.
|
||||
untaint Remove ${TAINT_KEY}:${TAINT_EFFECT} from the control plane.
|
||||
pods List non-system pods currently scheduled on ${CONTROL_PLANE_NODE}.
|
||||
|
||||
Environment:
|
||||
LAB_CONTROL_PLANE_NODE=${CONTROL_PLANE_NODE}
|
||||
EOF
|
||||
}
|
||||
|
||||
require_kubectl() {
|
||||
if ! command -v kubectl >/dev/null 2>&1; then
|
||||
echo "kubectl is required." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -s "$KUBECONFIG_PATH" ]; then
|
||||
echo "kubeconfig is missing: $KUBECONFIG_PATH" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! kubectl --kubeconfig "$KUBECONFIG_PATH" get node "$CONTROL_PLANE_NODE" >/dev/null 2>&1; then
|
||||
echo "control-plane node not found: ${CONTROL_PLANE_NODE}" >&2
|
||||
echo "Set LAB_CONTROL_PLANE_NODE if the node has a different name." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
show_taints() {
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" get node "$CONTROL_PLANE_NODE" \
|
||||
-o jsonpath='{range .spec.taints[*]}{.key}{"="}{.value}{":"}{.effect}{"\n"}{end}' |
|
||||
sed '/^$/d' || true
|
||||
}
|
||||
|
||||
show_non_system_pods() {
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" get pods -A \
|
||||
--field-selector "spec.nodeName=${CONTROL_PLANE_NODE}" \
|
||||
-o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase' --no-headers |
|
||||
awk '$1 !~ /^(kube-system|calico-system|tigera-operator)$/ { print }'
|
||||
}
|
||||
|
||||
case "${1:-status}" in
|
||||
status)
|
||||
require_kubectl
|
||||
echo "Node: ${CONTROL_PLANE_NODE}"
|
||||
echo
|
||||
echo "Taints:"
|
||||
if ! show_taints | sed 's/^/ /'; then
|
||||
true
|
||||
fi
|
||||
echo
|
||||
echo "Non-system pods currently on control plane:"
|
||||
if ! show_non_system_pods | sed 's/^/ /'; then
|
||||
true
|
||||
fi
|
||||
;;
|
||||
taint)
|
||||
require_kubectl
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" taint node "$CONTROL_PLANE_NODE" "${TAINT_KEY}:${TAINT_EFFECT}" --overwrite
|
||||
echo
|
||||
echo "Current taints:"
|
||||
show_taints | sed 's/^/ /'
|
||||
;;
|
||||
untaint)
|
||||
require_kubectl
|
||||
kubectl --kubeconfig "$KUBECONFIG_PATH" taint node "$CONTROL_PLANE_NODE" "${TAINT_KEY}:${TAINT_EFFECT}-" || true
|
||||
echo
|
||||
echo "Current taints:"
|
||||
show_taints | sed 's/^/ /'
|
||||
;;
|
||||
pods)
|
||||
require_kubectl
|
||||
show_non_system_pods
|
||||
;;
|
||||
-h|--help|help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
@ -4,10 +4,40 @@ set -euo pipefail
|
|||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
BUDGET_FILE="${HOMELAB_RESOURCE_BUDGET_FILE:-${REPO_ROOT}/infra/resource-budgets.yml}"
|
||||
KUBECONFIG_PATH="${KUBECONFIG:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}"
|
||||
DETAILS=false
|
||||
MAX_EXAMPLES="${LAB_RESOURCE_BUDGET_EXAMPLES:-12}"
|
||||
|
||||
failures=0
|
||||
warnings=0
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage: ./jeannie resource-budget [--details]
|
||||
|
||||
Print host and Kubernetes resource budget checks.
|
||||
|
||||
Options:
|
||||
--details Show every pod/container missing requests or limits.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--details|--verbose)
|
||||
DETAILS=true
|
||||
;;
|
||||
-h|--help|help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
section() {
|
||||
printf '\n== %s ==\n' "$1"
|
||||
}
|
||||
|
|
@ -58,12 +88,15 @@ kubernetes_budget() {
|
|||
return 0
|
||||
fi
|
||||
|
||||
python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" <<'PY'
|
||||
python3 - "$BUDGET_FILE" "$KUBECONFIG_PATH" "$DETAILS" "$MAX_EXAMPLES" <<'PY'
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
budget_file, kubeconfig = sys.argv[1:3]
|
||||
budget_file, kubeconfig, details_arg, max_examples_arg = sys.argv[1:5]
|
||||
show_details = details_arg.lower() == "true"
|
||||
max_examples = int(max_examples_arg)
|
||||
|
||||
try:
|
||||
import yaml
|
||||
|
|
@ -120,6 +153,7 @@ raw = subprocess.check_output(
|
|||
pods = json.loads(raw).get("items", [])
|
||||
namespace_totals = {}
|
||||
missing = []
|
||||
missing_by_namespace = defaultdict(Counter)
|
||||
|
||||
for pod in pods:
|
||||
ns = pod["metadata"]["namespace"]
|
||||
|
|
@ -134,8 +168,10 @@ for pod in pods:
|
|||
namespace_totals[ns]["memory"] += mem_to_mib(requests.get("memory"))
|
||||
if cluster_budget.get("require_requests", True) and ("cpu" not in requests or "memory" not in requests):
|
||||
missing.append(f"{ns}/{pod_name}:{name} missing requests")
|
||||
missing_by_namespace[ns]["requests"] += 1
|
||||
if cluster_budget.get("require_limits", True) and ("cpu" not in limits or "memory" not in limits):
|
||||
missing.append(f"{ns}/{pod_name}:{name} missing limits")
|
||||
missing_by_namespace[ns]["limits"] += 1
|
||||
|
||||
total_cpu = sum(item["cpu"] for item in namespace_totals.values())
|
||||
total_memory = sum(item["memory"] for item in namespace_totals.values())
|
||||
|
|
@ -163,11 +199,30 @@ for ns, totals in sorted(namespace_totals.items()):
|
|||
exit_code = 1
|
||||
|
||||
if missing:
|
||||
print("missing_resource_policy:")
|
||||
for item in missing[:50]:
|
||||
print(f" {item}")
|
||||
if len(missing) > 50:
|
||||
print(f" ... {len(missing) - 50} more")
|
||||
print(f"missing_resource_policy: total={len(missing)}")
|
||||
print("by_namespace:")
|
||||
for ns, counts in sorted(
|
||||
missing_by_namespace.items(),
|
||||
key=lambda item: (item[1]["requests"] + item[1]["limits"], item[0]),
|
||||
reverse=True,
|
||||
):
|
||||
print(f" {ns}: requests_missing={counts['requests']} limits_missing={counts['limits']}")
|
||||
|
||||
if show_details:
|
||||
print("details:")
|
||||
for item in missing:
|
||||
print(f" {item}")
|
||||
else:
|
||||
print("examples:")
|
||||
for item in missing[:max_examples]:
|
||||
print(f" {item}")
|
||||
if len(missing) > max_examples:
|
||||
print(f" ... {len(missing) - max_examples} more")
|
||||
print(" run ./jeannie resource-budget --details for the full list")
|
||||
print("next_steps:")
|
||||
print(" 1. Add requests before limits for app workloads you own.")
|
||||
print(" 2. Leave kube-system/control-plane static pods for a separate pass.")
|
||||
print(" 3. Use ./jeannie control-plane taint after Pimox workers are Ready.")
|
||||
exit_code = 1
|
||||
|
||||
raise SystemExit(exit_code)
|
||||
|
|
|
|||
Loading…
Reference in New Issue