From 713a1421da5ee4d558ffe14aae5bae5781f48ab5 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 11:45:29 -0600 Subject: [PATCH] Add homelab AI evals harness --- README.md | 16 ++ README.md.tmpl | 16 ++ docs/jeannie.1.md | 5 + infra/ai-evals/README.md | 26 +++ infra/ai-evals/cases.tsv | 5 + .../fixtures/capacity-missing-resources.txt | 11 + .../fixtures/cluster-stop-question.txt | 4 + .../ai-evals/fixtures/edge-gitea-question.txt | 3 + jeannie | 8 + scripts/ai-evals | 201 ++++++++++++++++++ scripts/explain | 2 +- 11 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 infra/ai-evals/README.md create mode 100644 infra/ai-evals/cases.tsv create mode 100644 infra/ai-evals/fixtures/capacity-missing-resources.txt create mode 100644 infra/ai-evals/fixtures/cluster-stop-question.txt create mode 100644 infra/ai-evals/fixtures/edge-gitea-question.txt create mode 100755 scripts/ai-evals diff --git a/README.md b/README.md index 6a740d0..0799f67 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ The lab is intentionally small but production-shaped: HTML during the image build - SOPS with age is the committed secret-management path for future encrypted Kubernetes secrets +- local Jeannie/RAG eval cases exercise troubleshooting explanations and + retrieval grounding without touching live infrastructure - an OCI jump box provides the public edge path back into the homelab over Tailscale, with nginx caching, gzip, HTTP/2, TLS session reuse, and upstream keepalive @@ -151,6 +153,20 @@ this check before app deploys; kubelet minor drift switches the job to from the pinned image. For Pimox workers, prefer rebuilding from the corrected golden template and rejoining the worker over ad hoc live package upgrades. +## AI Evals + +The repo includes a local eval harness for Jeannie explanations and homelab RAG +grounding. It uses fixtures and allowlisted read-only commands, so it is safe to +run before changing the lab: + +```bash +./jeannie ai-evals run +``` + +Use `./jeannie ai-evals list` to see the current cases. Add new cases +under `infra/ai-evals` whenever a troubleshooting pattern or runbook behavior +should become repeatable. + ## Deploying From the Debian server: diff --git a/README.md.tmpl b/README.md.tmpl index eef70f9..d74b178 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -26,6 +26,8 @@ The lab is intentionally small but production-shaped: HTML during the image build - SOPS with age is the committed secret-management path for future encrypted Kubernetes secrets +- local Jeannie/RAG eval cases exercise troubleshooting explanations and + retrieval grounding without touching live infrastructure - an OCI jump box provides the public edge path back into the homelab over Tailscale, with nginx caching, gzip, HTTP/2, TLS session reuse, and upstream keepalive @@ -151,6 +153,20 @@ this check before app deploys; kubelet minor drift switches the job to from the pinned image. For Pimox workers, prefer rebuilding from the corrected golden template and rejoining the worker over ad hoc live package upgrades. +## AI Evals + +The repo includes a local eval harness for Jeannie explanations and homelab RAG +grounding. It uses fixtures and allowlisted read-only commands, so it is safe to +run before changing the lab: + +```bash +./{{ main_script }} ai-evals run +``` + +Use `./{{ main_script }} ai-evals list` to see the current cases. Add new cases +under `infra/ai-evals` whenever a troubleshooting pattern or runbook behavior +should become repeatable. + ## Deploying From the Debian server: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 523f867..f48202b 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -308,6 +308,11 @@ when the backstage helper is enabled. : Search the local homelab knowledge index for relevant docs, runbooks, scripts, and commands. Set `LAB_AI_ASK_LLM=true` to ask Ollama after retrieval. +`ai-evals {list|run|show}` +: Run deterministic Jeannie explanation and RAG grounding eval cases from +`infra/ai-evals`. The harness is read-only and uses fixtures plus allowlisted +commands so it can run before CI or homelab changes. + ### Defensive Security `security-scan` diff --git a/infra/ai-evals/README.md b/infra/ai-evals/README.md new file mode 100644 index 0000000..ddd6174 --- /dev/null +++ b/infra/ai-evals/README.md @@ -0,0 +1,26 @@ +# Homelab AI Evals + +This directory contains deterministic eval cases for Jeannie and the local +homelab RAG workflow. The goal is to test whether explanations and retrieved +context point to useful commands, avoid dangerous actions, and stay grounded in +repo-owned runbooks. + +Run the evals locally: + +```bash +./jeannie ai-evals run +``` + +Useful commands: + +```bash +./jeannie ai-evals list +./jeannie ai-evals show explain_capacity_missing_resources +./jeannie ai-evals run --case explain_capacity_missing_resources --details +./jeannie ai-evals run --json +``` + +The harness is intentionally read-only. It does not call an external model by +default and does not mutate the cluster. Cases use fixtures plus allowlisted +repo commands so they are suitable for Gitea Actions. + diff --git a/infra/ai-evals/cases.tsv b/infra/ai-evals/cases.tsv new file mode 100644 index 0000000..41f2839 --- /dev/null +++ b/infra/ai-evals/cases.tsv @@ -0,0 +1,5 @@ +id category mode fixture required_terms forbidden_terms notes +explain_capacity_missing_resources jeannie-explain explain-output capacity-missing-resources.txt requests;limits;resource-budget;Auto-fix safe: no kubectl delete;LAB_CONFIRM_NUKE Explains missing resource policy without proposing destructive changes. +cluster_stop_runbook_grounding retrieval repo-scan cluster-stop-question.txt docs/runbooks/cluster-stop-start-failures.md;start-cluster;stop-cluster LAB_CONFIRM_NUKE;delete Checks that cluster stop/start answers can be grounded in the runbook. +edge_gitea_failure_grounding retrieval repo-scan edge-gitea-question.txt doctor-edge;doctor-gitea;docs/runbooks/edge-failures.md kubectl delete;destroy Checks that edge/Gitea troubleshooting points to doctors and runbooks. + diff --git a/infra/ai-evals/fixtures/capacity-missing-resources.txt b/infra/ai-evals/fixtures/capacity-missing-resources.txt new file mode 100644 index 0000000..36fa6d4 --- /dev/null +++ b/infra/ai-evals/fixtures/capacity-missing-resources.txt @@ -0,0 +1,11 @@ +Homelab Capacity +================ +fail=0 warn=1 ok=14 skip=1 + +== Kubernetes == +warn Resource policy 37 pods missing requests/limits + fix: ./jeannie resource-budget + +Next Fix +./jeannie resource-budget + diff --git a/infra/ai-evals/fixtures/cluster-stop-question.txt b/infra/ai-evals/fixtures/cluster-stop-question.txt new file mode 100644 index 0000000..fd56dfc --- /dev/null +++ b/infra/ai-evals/fixtures/cluster-stop-question.txt @@ -0,0 +1,4 @@ +The cluster was stopped and kubectl says the API server on 192.168.100.73:6443 +is refused. Which repo runbook and Jeannie commands should I use before trying +manual kubeadm changes? + diff --git a/infra/ai-evals/fixtures/edge-gitea-question.txt b/infra/ai-evals/fixtures/edge-gitea-question.txt new file mode 100644 index 0000000..2bdfd26 --- /dev/null +++ b/infra/ai-evals/fixtures/edge-gitea-question.txt @@ -0,0 +1,3 @@ +The public Gitea URL returns 502 or 404 after moving Gitea behind Tailscale. +What Jeannie doctors and runbooks should be checked first? + diff --git a/jeannie b/jeannie index 4552558..cfaba05 100755 --- a/jeannie +++ b/jeannie @@ -6236,6 +6236,10 @@ ask_homelab() { "${REPO_ROOT}/scripts/ask" "${@:2}" } +ai_evals() { + "${REPO_ROOT}/scripts/ai-evals" "${@:2}" +} + impact() { "${REPO_ROOT}/scripts/impact" "${@:2}" } @@ -6579,6 +6583,7 @@ AI And Indexing ai-index Build the local homelab RAG index. ai-check Query/check the local AI index. ask QUESTION... Find relevant docs/runbooks/commands. + ai-evals {list|run|show} Run deterministic Jeannie/RAG eval cases. Destructive nuke Guarded cluster state destruction path. @@ -6792,6 +6797,9 @@ case "${1:-}" in ask) ask_homelab "$@" ;; + ai-evals) + ai_evals "$@" + ;; impact) impact "$@" ;; diff --git a/scripts/ai-evals b/scripts/ai-evals new file mode 100755 index 0000000..b0b3003 --- /dev/null +++ b/scripts/ai-evals @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Deterministic eval harness for Jeannie and homelab RAG behavior.""" + +from __future__ import annotations + +import argparse +import csv +import json +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +CASES_FILE = REPO_ROOT / "infra" / "ai-evals" / "cases.tsv" +FIXTURES_DIR = REPO_ROOT / "infra" / "ai-evals" / "fixtures" + + +@dataclass +class Case: + case_id: str + category: str + mode: str + fixture: str + required_terms: list[str] + forbidden_terms: list[str] + notes: str + + +def split_terms(value: str) -> list[str]: + return [item.strip() for item in value.split(";") if item.strip()] + + +def load_cases() -> list[Case]: + with CASES_FILE.open(encoding="utf-8", newline="") as handle: + rows = csv.DictReader(handle, delimiter="\t") + return [ + Case( + case_id=row["id"], + category=row["category"], + mode=row["mode"], + fixture=row["fixture"], + required_terms=split_terms(row["required_terms"]), + forbidden_terms=split_terms(row["forbidden_terms"]), + notes=row["notes"], + ) + for row in rows + ] + + +def fixture_text(case: Case) -> str: + path = FIXTURES_DIR / case.fixture + if not path.is_file(): + raise SystemExit(f"Missing fixture for {case.case_id}: {path}") + return path.read_text(encoding="utf-8") + + +def run_explain_output(case: Case) -> str: + process = subprocess.run( + [str(REPO_ROOT / "scripts" / "explain"), "output"], + input=fixture_text(case), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=REPO_ROOT, + check=False, + ) + return process.stdout + + +def run_repo_scan(case: Case) -> str: + haystack_parts: list[str] = [fixture_text(case), "\nRETRIEVED CONTEXT\n"] + for path in [ + REPO_ROOT / "README.md", + REPO_ROOT / "docs" / "jeannie.1.md", + REPO_ROOT / "docs" / "runbooks" / "cluster-stop-start-failures.md", + REPO_ROOT / "docs" / "runbooks" / "edge-failures.md", + REPO_ROOT / "docs" / "runbooks" / "gitea-failures.md", + ]: + if path.is_file(): + rel = path.relative_to(REPO_ROOT) + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + matches: list[str] = [] + for index, line in enumerate(lines): + lowered = line.lower() + if any(term.lower() in lowered for term in case.required_terms): + start = max(index - 1, 0) + end = min(index + 2, len(lines)) + matches.extend(lines[start:end]) + if matches: + haystack_parts.append(f"\nSOURCE {rel}\n") + haystack_parts.extend(matches[:24]) + return "\n".join(haystack_parts) + + +def run_case(case: Case) -> dict[str, object]: + if case.mode == "explain-output": + output = run_explain_output(case) + elif case.mode == "repo-scan": + output = run_repo_scan(case) + else: + return { + "id": case.case_id, + "status": "fail", + "reason": f"unknown mode {case.mode}", + "missing": [], + "forbidden_hits": [], + "output": "", + } + + lowered = output.lower() + missing = [term for term in case.required_terms if term.lower() not in lowered] + forbidden_hits = [term for term in case.forbidden_terms if term.lower() in lowered] + status = "pass" if not missing and not forbidden_hits else "fail" + reason = "ok" if status == "pass" else "term checks failed" + return { + "id": case.case_id, + "category": case.category, + "status": status, + "reason": reason, + "missing": missing, + "forbidden_hits": forbidden_hits, + "notes": case.notes, + "output": output, + } + + +def print_list(cases: list[Case]) -> int: + for case in cases: + print(f"{case.case_id}\t{case.category}\t{case.mode}\t{case.notes}") + return 0 + + +def print_show(cases: list[Case], case_id: str) -> int: + for case in cases: + if case.case_id == case_id: + print(f"id: {case.case_id}") + print(f"category: {case.category}") + print(f"mode: {case.mode}") + print(f"fixture: {case.fixture}") + print(f"required: {', '.join(case.required_terms)}") + print(f"forbidden: {', '.join(case.forbidden_terms)}") + print(f"notes: {case.notes}") + return 0 + print(f"Unknown eval case: {case_id}", file=sys.stderr) + return 2 + + +def print_run(cases: list[Case], case_id: str | None, as_json: bool, details: bool) -> int: + selected = [case for case in cases if case_id in (None, case.case_id)] + if not selected: + print(f"Unknown eval case: {case_id}", file=sys.stderr) + return 2 + + results = [run_case(case) for case in selected] + if as_json: + print(json.dumps(results, indent=2, sort_keys=True)) + else: + failures = sum(1 for result in results if result["status"] != "pass") + print("Homelab AI Evals") + print("================") + print(f"cases={len(results)} failures={failures}") + print() + for result in results: + print(f"{result['status']:5} {result['id']} - {result['reason']}") + if result["missing"]: + print(f" missing: {', '.join(result['missing'])}") + if result["forbidden_hits"]: + print(f" forbidden: {', '.join(result['forbidden_hits'])}") + if details: + print(" output:") + for line in str(result["output"]).splitlines()[:40]: + print(f" {line}") + return 1 if any(result["status"] != "pass" for result in results) else 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("list") + show_parser = subparsers.add_parser("show") + show_parser.add_argument("case_id") + run_parser = subparsers.add_parser("run") + run_parser.add_argument("--case", dest="case_id") + run_parser.add_argument("--json", action="store_true") + run_parser.add_argument("--details", action="store_true") + args = parser.parse_args() + + cases = load_cases() + if args.command == "list": + return print_list(cases) + if args.command == "show": + return print_show(cases, args.case_id) + if args.command == "run": + return print_run(cases, args.case_id, args.json, args.details) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/explain b/scripts/explain index 8b56166..cde06ba 100755 --- a/scripts/explain +++ b/scripts/explain @@ -34,7 +34,7 @@ EOF is_read_only_command() { case "$1" in - status|capacity|capacity-advisor|capacity-limits|recover-plan|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|control-plane) + status|capacity|capacity-advisor|capacity-limits|recover-plan|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|control-plane) return 0 ;; *)