Add AI workload scheduler simulator
This commit is contained in:
parent
0b886295c8
commit
edb36592a7
14
README.md
14
README.md
|
|
@ -218,6 +218,20 @@ policy starts with a read-only command classifier:
|
|||
|
||||
Policy is stored in `infra/agent-sandbox/policy.tsv`.
|
||||
|
||||
## AI Workload Scheduling
|
||||
|
||||
For AI-ish workloads, Jeannie can simulate placement across Debian, Pimox
|
||||
workers, and the RPi based on CPU, memory, architecture, tailnet access, and
|
||||
control-plane policy:
|
||||
|
||||
```bash
|
||||
./jeannie ai-scheduler recommend small-rag
|
||||
./jeannie ai-scheduler simulate
|
||||
```
|
||||
|
||||
The simulator lives under `infra/ai-scheduler` and is a stepping stone toward a
|
||||
real Kubernetes controller.
|
||||
|
||||
## Deploying
|
||||
|
||||
From the Debian server:
|
||||
|
|
|
|||
|
|
@ -218,6 +218,20 @@ policy starts with a read-only command classifier:
|
|||
|
||||
Policy is stored in `infra/agent-sandbox/policy.tsv`.
|
||||
|
||||
## AI Workload Scheduling
|
||||
|
||||
For AI-ish workloads, Jeannie can simulate placement across Debian, Pimox
|
||||
workers, and the RPi based on CPU, memory, architecture, tailnet access, and
|
||||
control-plane policy:
|
||||
|
||||
```bash
|
||||
./{{ main_script }} ai-scheduler recommend small-rag
|
||||
./{{ main_script }} ai-scheduler simulate
|
||||
```
|
||||
|
||||
The simulator lives under `infra/ai-scheduler` and is a stepping stone toward a
|
||||
real Kubernetes controller.
|
||||
|
||||
## Deploying
|
||||
|
||||
From the Debian server:
|
||||
|
|
|
|||
|
|
@ -326,6 +326,10 @@ and commands. Set `LAB_AI_ASK_LLM=true` to ask Ollama after retrieval.
|
|||
`infra/ai-evals`. The harness is read-only and uses fixtures plus allowlisted
|
||||
commands so it can run before CI or homelab changes.
|
||||
|
||||
`ai-scheduler {simulate|recommend NAME}`
|
||||
: Simulate AI workload placement against repo-managed node and workload policy,
|
||||
including CPU, memory, architecture, tailnet access, and control-plane penalty.
|
||||
|
||||
### Defensive Security
|
||||
|
||||
`security-scan`
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
# Kubernetes AI Workload Scheduler
|
||||
|
||||
This is a lightweight scheduler simulator for AI-ish workloads in the homelab.
|
||||
It scores candidate nodes by architecture, CPU, memory, tailnet access, and
|
||||
control-plane taints.
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
./jeannie ai-scheduler simulate
|
||||
./jeannie ai-scheduler recommend small-rag
|
||||
```
|
||||
|
||||
The first version is read-only and uses `infra/ai-scheduler/workloads.tsv` plus
|
||||
`infra/ai-scheduler/nodes.tsv`. It is meant to inform placement policy before
|
||||
building a real Kubernetes controller.
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
node arch cpu_m memory_mib tailnet control_plane power_class
|
||||
debian amd64 6000 16000 true true high
|
||||
pimox-worker-01 arm64 2000 4096 true false low
|
||||
pimox-worker-02 arm64 2000 4096 true false low
|
||||
pimox-worker-03 arm64 2000 4096 true false low
|
||||
rpi4 arm64 1500 2048 false false low
|
||||
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
workload cpu_m memory_mib requires_tailnet prefer_arch allow_control_plane
|
||||
small-rag 750 1536 true amd64 false
|
||||
model-observer 300 512 false any false
|
||||
eval-runner 1000 2048 false amd64 true
|
||||
website-ai-helper 500 768 true any false
|
||||
|
||||
|
8
jeannie
8
jeannie
|
|
@ -6338,6 +6338,10 @@ agent_sandbox() {
|
|||
"${REPO_ROOT}/scripts/agent-sandbox" "${@:2}"
|
||||
}
|
||||
|
||||
ai_scheduler() {
|
||||
"${REPO_ROOT}/scripts/ai-scheduler" "${@:2}"
|
||||
}
|
||||
|
||||
validate_homelab() {
|
||||
"${REPO_ROOT}/scripts/validate-homelab"
|
||||
}
|
||||
|
|
@ -6604,6 +6608,7 @@ AI And Indexing
|
|||
ai-check Query/check the local AI index.
|
||||
ask QUESTION... Find relevant docs/runbooks/commands.
|
||||
ai-evals {list|run|show} Run deterministic Jeannie/RAG eval cases.
|
||||
ai-scheduler {simulate|recommend NAME} Simulate AI workload placement policy.
|
||||
|
||||
Destructive
|
||||
nuke Guarded cluster state destruction path.
|
||||
|
|
@ -6829,6 +6834,9 @@ case "${1:-}" in
|
|||
ai-evals)
|
||||
ai_evals "$@"
|
||||
;;
|
||||
ai-scheduler)
|
||||
ai_scheduler "$@"
|
||||
;;
|
||||
impact)
|
||||
impact "$@"
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,124 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Simulate placement for AI-ish homelab workloads."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
SCHED_DIR = REPO_ROOT / "infra" / "ai-scheduler"
|
||||
NODES_FILE = SCHED_DIR / "nodes.tsv"
|
||||
WORKLOADS_FILE = SCHED_DIR / "workloads.tsv"
|
||||
|
||||
|
||||
def read_tsv(path: Path) -> list[dict[str, str]]:
|
||||
with path.open(encoding="utf-8", newline="") as handle:
|
||||
return list(csv.DictReader(handle, delimiter="\t"))
|
||||
|
||||
|
||||
def as_bool(value: str) -> bool:
|
||||
return value.strip().lower() in {"true", "yes", "1"}
|
||||
|
||||
|
||||
def score_node(workload: dict[str, str], node: dict[str, str]) -> tuple[int, list[str], list[str]]:
|
||||
score = 100
|
||||
reasons: list[str] = []
|
||||
blockers: list[str] = []
|
||||
cpu_needed = int(workload["cpu_m"])
|
||||
mem_needed = int(workload["memory_mib"])
|
||||
cpu_have = int(node["cpu_m"])
|
||||
mem_have = int(node["memory_mib"])
|
||||
|
||||
if cpu_have < cpu_needed:
|
||||
blockers.append(f"cpu {cpu_have}m < {cpu_needed}m")
|
||||
else:
|
||||
reasons.append("cpu fits")
|
||||
score += min((cpu_have - cpu_needed) // 250, 10)
|
||||
if mem_have < mem_needed:
|
||||
blockers.append(f"memory {mem_have}Mi < {mem_needed}Mi")
|
||||
else:
|
||||
reasons.append("memory fits")
|
||||
score += min((mem_have - mem_needed) // 512, 10)
|
||||
if as_bool(workload["requires_tailnet"]) and not as_bool(node["tailnet"]):
|
||||
blockers.append("tailnet required")
|
||||
elif as_bool(node["tailnet"]):
|
||||
reasons.append("tailnet available")
|
||||
score += 8
|
||||
if workload["prefer_arch"] != "any" and workload["prefer_arch"] != node["arch"]:
|
||||
score -= 15
|
||||
reasons.append(f"non-preferred arch {node['arch']}")
|
||||
else:
|
||||
reasons.append(f"arch {node['arch']} ok")
|
||||
if as_bool(node["control_plane"]) and not as_bool(workload["allow_control_plane"]):
|
||||
score -= 30
|
||||
reasons.append("control-plane penalty")
|
||||
if node["power_class"] == "low":
|
||||
score += 5
|
||||
reasons.append("low-power node")
|
||||
return score, reasons, blockers
|
||||
|
||||
|
||||
def recommend(workload_name: str) -> int:
|
||||
workloads = {row["workload"]: row for row in read_tsv(WORKLOADS_FILE)}
|
||||
nodes = read_tsv(NODES_FILE)
|
||||
if workload_name not in workloads:
|
||||
print(f"Unknown workload: {workload_name}")
|
||||
print("Known workloads:")
|
||||
for name in workloads:
|
||||
print(f" {name}")
|
||||
return 2
|
||||
workload = workloads[workload_name]
|
||||
rows = []
|
||||
for node in nodes:
|
||||
score, reasons, blockers = score_node(workload, node)
|
||||
rows.append((score, node["node"], reasons, blockers))
|
||||
rows.sort(reverse=True)
|
||||
|
||||
print("AI Workload Scheduler")
|
||||
print("=====================")
|
||||
print(f"workload: {workload_name}")
|
||||
print()
|
||||
for score, node_name, reasons, blockers in rows:
|
||||
status = "blocked" if blockers else "candidate"
|
||||
print(f"{status:9} {node_name:18} score={score}")
|
||||
if blockers:
|
||||
print(f" blockers: {', '.join(blockers)}")
|
||||
print(f" reasons: {', '.join(reasons)}")
|
||||
best = next((row for row in rows if not row[3]), None)
|
||||
if best:
|
||||
print()
|
||||
print(f"Recommendation: {best[1]}")
|
||||
return 0
|
||||
print()
|
||||
print("Recommendation: add capacity or relax workload requirements")
|
||||
return 1
|
||||
|
||||
|
||||
def simulate() -> int:
|
||||
failures = 0
|
||||
for workload in read_tsv(WORKLOADS_FILE):
|
||||
print()
|
||||
rc = recommend(workload["workload"])
|
||||
failures += 1 if rc else 0
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
subparsers.add_parser("simulate")
|
||||
rec = subparsers.add_parser("recommend")
|
||||
rec.add_argument("workload")
|
||||
args = parser.parse_args()
|
||||
if args.command == "simulate":
|
||||
return simulate()
|
||||
if args.command == "recommend":
|
||||
return recommend(args.workload)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -34,7 +34,7 @@ EOF
|
|||
|
||||
is_read_only_command() {
|
||||
case "$1" in
|
||||
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane)
|
||||
status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
|
|
|
|||
Loading…
Reference in New Issue