From 0af1257d863a5ad2f73919c1f1a5591d761a244a Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 13:19:49 -0600 Subject: [PATCH] Add Jeannie memory provenance checks --- README.md | 12 +++++++ README.md.tmpl | 12 +++++++ docs/jeannie.1.md | 8 +++++ infra/ai/provenance.md | 15 ++++++++ jeannie | 10 +++++- scripts/ai-memory | 66 ++++++++++++++++++++++++++++++++++ scripts/ask | 11 +++++- scripts/build-homelab-ai-index | 6 ++++ scripts/explain | 2 +- scripts/query-homelab-ai-index | 30 ++++++++++++++++ 10 files changed, 169 insertions(+), 3 deletions(-) create mode 100644 infra/ai/provenance.md create mode 100755 scripts/ai-memory diff --git a/README.md b/README.md index 192fe3e..9d3000a 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,18 @@ control-plane policy: The simulator lives under `infra/ai-scheduler` and is a stepping stone toward a real Kubernetes controller. +## Memory With Provenance + +Jeannie can return cited retrieval context and check whether the local RAG index +is stale: + +```bash +./jeannie ask --citations "how do I recover the cluster?" +./jeannie ai-memory check +``` + +This keeps local AI answers grounded in repo files and runbooks. + ## Deploying From the Debian server: diff --git a/README.md.tmpl b/README.md.tmpl index 2aa6f4c..6317e6d 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -232,6 +232,18 @@ control-plane policy: The simulator lives under `infra/ai-scheduler` and is a stepping stone toward a real Kubernetes controller. +## Memory With Provenance + +Jeannie can return cited retrieval context and check whether the local RAG index +is stale: + +```bash +./{{ main_script }} ask --citations "how do I recover the cluster?" +./{{ main_script }} ai-memory check +``` + +This keeps local AI answers grounded in repo files and runbooks. + ## Deploying From the Debian server: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 8890ad0..22f1a04 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -321,6 +321,10 @@ when the backstage helper is enabled. : Search the local homelab knowledge index for relevant docs, runbooks, scripts, and commands. Set `LAB_AI_ASK_LLM=true` to ask Ollama after retrieval. +`ask --citations QUESTION...` +: Return cited retrieval context with source paths and line ranges, without +calling a model. + `ai-evals {list|run|show}` : Run deterministic Jeannie explanation and RAG grounding eval cases from `infra/ai-evals`. The harness is read-only and uses fixtures plus allowlisted @@ -330,6 +334,10 @@ commands so it can run before CI or homelab changes. : Simulate AI workload placement against repo-managed node and workload policy, including CPU, memory, architecture, tailnet access, and control-plane penalty. +`ai-memory {check}` +: Check whether the local homelab RAG index manifest is present and whether +indexed source files changed after indexing. + ### Defensive Security `security-scan` diff --git a/infra/ai/provenance.md b/infra/ai/provenance.md new file mode 100644 index 0000000..f33cf89 --- /dev/null +++ b/infra/ai/provenance.md @@ -0,0 +1,15 @@ +# Jeannie Memory With Provenance + +The homelab RAG path must stay grounded: + +- every answer should cite repo files, runbooks, metrics, or command outputs +- stale index data should be detected before relying on retrieved context +- missing context should produce an inspection command, not a guess + +Commands: + +```bash +./jeannie ask --citations "how do I recover the cluster?" +./jeannie ai-memory check +``` + diff --git a/jeannie b/jeannie index ab6c6f2..a64431e 100755 --- a/jeannie +++ b/jeannie @@ -6342,6 +6342,10 @@ ai_scheduler() { "${REPO_ROOT}/scripts/ai-scheduler" "${@:2}" } +ai_memory() { + "${REPO_ROOT}/scripts/ai-memory" "${@:2}" +} + validate_homelab() { "${REPO_ROOT}/scripts/validate-homelab" } @@ -6606,9 +6610,10 @@ Security Learning AI And Indexing ai-index Build the local homelab RAG index. ai-check Query/check the local AI index. - ask QUESTION... Find relevant docs/runbooks/commands. + ask [--citations] QUESTION... Find relevant docs/runbooks/commands. ai-evals {list|run|show} Run deterministic Jeannie/RAG eval cases. ai-scheduler {simulate|recommend NAME} Simulate AI workload placement policy. + ai-memory {check} Check local RAG index provenance freshness. Destructive nuke Guarded cluster state destruction path. @@ -6837,6 +6842,9 @@ case "${1:-}" in ai-scheduler) ai_scheduler "$@" ;; + ai-memory) + ai_memory "$@" + ;; impact) impact "$@" ;; diff --git a/scripts/ai-memory b/scripts/ai-memory new file mode 100755 index 0000000..71ab4a2 --- /dev/null +++ b/scripts/ai-memory @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Check local homelab AI memory provenance and freshness.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_INDEX_DIR = Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index")) + + +def check(index_dir: Path) -> int: + manifest_path = index_dir / "manifest.json" + if not manifest_path.is_file(): + print(f"fail index manifest missing: {manifest_path}") + print("fix: ./jeannie ai-index") + return 1 + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + stale: list[str] = [] + missing: list[str] = [] + for rel_path, indexed_mtime in manifest.get("files", {}).items(): + path = REPO_ROOT / rel_path + if not path.is_file(): + missing.append(rel_path) + continue + if int(path.stat().st_mtime) > int(indexed_mtime): + stale.append(rel_path) + + print("Jeannie Memory") + print("==============") + print(f"index: {index_dir}") + print(f"chunks: {manifest.get('chunk_count', 'unknown')}") + print(f"sources: {manifest.get('file_count', 'unknown')}") + if stale: + print(f"status: stale ({len(stale)} source file(s) changed)") + for rel_path in stale[:20]: + print(f" stale: {rel_path}") + print("fix: ./jeannie ai-index") + return 1 + if missing: + print(f"status: warn ({len(missing)} indexed source file(s) missing)") + for rel_path in missing[:20]: + print(f" missing: {rel_path}") + print("fix: ./jeannie ai-index") + return 1 + print("status: ok") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + check_parser = subparsers.add_parser("check") + check_parser.add_argument("--index-dir", type=Path, default=DEFAULT_INDEX_DIR) + args = parser.parse_args() + if args.command == "check": + return check(args.index_dir) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ask b/scripts/ask index 9fcbd29..a6c8c79 100755 --- a/scripts/ask +++ b/scripts/ask @@ -5,20 +5,29 @@ REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" usage() { cat <<'EOF' -Usage: ./jeannie ask QUESTION... +Usage: ./jeannie ask [--citations] QUESTION... Search the homelab knowledge index for the closest docs, runbooks, scripts, and Jeannie commands. Set LAB_AI_ASK_LLM=true to ask Ollama after retrieval. EOF } +citations=false case "${1:-}" in + --citations) + citations=true + shift + ;; ""|-h|--help|help) usage exit 0 ;; esac +if [ "$citations" = "true" ]; then + exec "${REPO_ROOT}/scripts/query-homelab-ai-index" --citations "$@" +fi + if [ "${LAB_AI_ASK_LLM:-false}" = "true" ]; then exec "${REPO_ROOT}/scripts/query-homelab-ai-index" --ask "$@" fi diff --git a/scripts/build-homelab-ai-index b/scripts/build-homelab-ai-index index 1bcafff..1bb827e 100755 --- a/scripts/build-homelab-ai-index +++ b/scripts/build-homelab-ai-index @@ -11,6 +11,7 @@ import os import pathlib import re import sys +import time from collections import Counter @@ -185,11 +186,16 @@ def main() -> int: manifest_path.write_text( json.dumps( { + "generated_at_epoch": int(time.time()), "sources_file": str(args.sources_file.relative_to(REPO_ROOT)), "excludes_file": str(args.excludes_file.relative_to(REPO_ROOT)) if args.excludes_file.is_file() else None, "file_count": len(files), "chunk_count": index["chunk_count"], "repo_root": str(REPO_ROOT), + "files": { + path.relative_to(REPO_ROOT).as_posix(): int(path.stat().st_mtime) + for path in files + }, }, indent=2, sort_keys=True, diff --git a/scripts/explain b/scripts/explain index 26bf1e3..58716c9 100755 --- a/scripts/explain +++ b/scripts/explain @@ -34,7 +34,7 @@ EOF is_read_only_command() { case "$1" in - status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane) + status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane) return 0 ;; *) diff --git a/scripts/query-homelab-ai-index b/scripts/query-homelab-ai-index index 63f729a..a345821 100755 --- a/scripts/query-homelab-ai-index +++ b/scripts/query-homelab-ai-index @@ -86,6 +86,31 @@ def render_context(results: list[tuple[float, dict[str, object]]], max_chars: in return "\n\n---\n\n".join(sections) +def render_citations(results: list[tuple[float, dict[str, object]]], max_chars: int) -> str: + sections = ["Jeannie Memory With Provenance", "=============================", ""] + remaining = max_chars + for index, (score, chunk) in enumerate(results, start=1): + path = chunk.get("path") + start_line = chunk.get("start_line") + end_line = chunk.get("end_line") + text = str(chunk.get("text", "")).strip() + excerpt = text[:900].rstrip() + section = [ + f"[{index}] {path}:{start_line}-{end_line} score={score:.3f}", + excerpt, + ] + rendered = "\n".join(section) + if len(rendered) > remaining: + rendered = rendered[: max(0, remaining - 20)].rstrip() + "\n[truncated]" + sections.append(rendered) + sections.append("") + remaining -= len(rendered) + if remaining <= 0: + break + sections.append("Rule: answers must cite these source paths or say the context is insufficient.") + return "\n".join(sections) + + def ask_ollama(question: str, context: str, endpoint: str, model: str, timeout: int) -> str: prompt = f"""You are helping operate a personal homelab. Use only the context below. If the context is insufficient, say what to inspect next. @@ -124,6 +149,7 @@ def main() -> int: parser.add_argument("--limit", type=int, default=5) parser.add_argument("--max-chars", type=int, default=9000) parser.add_argument("--context-only", action="store_true") + parser.add_argument("--citations", action="store_true") parser.add_argument("--ask", action="store_true") parser.add_argument("--ollama-url", default=os.environ.get("LAB_AI_GATEWAY_URL", "http://127.0.0.1:11434")) parser.add_argument("--model", default=os.environ.get("LAB_AI_GATEWAY_MODEL", "qwen2.5:0.5b")) @@ -143,6 +169,10 @@ def main() -> int: return 1 context = render_context(results, args.max_chars) + if args.citations: + print(render_citations(results, args.max_chars)) + return 0 + if args.context_only: print(context) return 0