diff --git a/README.md b/README.md index 762bb8a..6bd5345 100644 --- a/README.md +++ b/README.md @@ -868,6 +868,18 @@ Disable it for low-CPU sessions with: LAB_BACKSTAGE_BRAIN_ENABLED=false ./jeannie doctor-edge ``` +Build the local homelab knowledge index separately from the main deployment: + +```bash +./jeannie ai-index +./jeannie ai-check +``` + +The index is built from the non-secret source list in +`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`. +Doctor commands use it as extra context when it exists, but infrastructure +deployment does not depend on it. + The CV page has two client-side presentation modes: - `Elegant`: dark, minimal, terminal-inspired styling with a square profile diff --git a/README.md.tmpl b/README.md.tmpl index 443e8f2..e9a574d 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -868,6 +868,18 @@ Disable it for low-CPU sessions with: LAB_BACKSTAGE_BRAIN_ENABLED=false ./{{ main_script }} doctor-edge ``` +Build the local homelab knowledge index separately from the main deployment: + +```bash +./{{ main_script }} ai-index +./{{ main_script }} ai-check +``` + +The index is built from the non-secret source list in +`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`. +Doctor commands use it as extra context when it exists, but infrastructure +deployment does not depend on it. + The CV page has two client-side presentation modes: - `Elegant`: dark, minimal, terminal-inspired styling with a square profile diff --git a/homelab.yml b/homelab.yml index 10ce5c0..7ab93b2 100644 --- a/homelab.yml +++ b/homelab.yml @@ -124,3 +124,4 @@ ai_gateway: url: http://192.168.100.73:11434 model: qwen2.5:0.5b timeout_seconds: 20 + knowledge_index_dir: /data/homelab-ai/index diff --git a/infra/ai/README.md b/infra/ai/README.md new file mode 100644 index 0000000..647f77b --- /dev/null +++ b/infra/ai/README.md @@ -0,0 +1,49 @@ +# Homelab AI Knowledge Index + +This directory defines the optional local knowledge layer for the backstage +`jeannie` helper. + +The index is not model training. It is retrieval over repo files, so current +inventory, runbooks, scripts, Terraform, compose files, and app manifests remain +normal source-controlled text. + +## Build + +Run on the Debian homelab server: + +```bash +./jeannie ai-index +``` + +The default index path comes from `homelab.yml`: + +```text +/data/homelab-ai/index +``` + +Override it with `LAB_AI_KNOWLEDGE_INDEX_DIR` when testing. + +## Check + +```bash +./jeannie ai-check +``` + +This validates: + +- Python is available +- the index exists +- a simple retrieval query returns context +- Ollama is reachable when the backstage helper is enabled +- the configured model is pulled + +## Query + +The query script can be used directly: + +```bash +scripts/query-homelab-ai-index "gitea public url is failing" +scripts/query-homelab-ai-index --ask "why is lab2025 returning 502?" +``` + +`--ask` sends retrieved context to the configured Ollama endpoint and model. diff --git a/infra/ai/knowledge-sources.txt b/infra/ai/knowledge-sources.txt new file mode 100644 index 0000000..3334025 --- /dev/null +++ b/infra/ai/knowledge-sources.txt @@ -0,0 +1,25 @@ +# Files and globs indexed for the local homelab AI helper. +# Keep this non-secret. The index is built from repository content only. +homelab.yml +README.md +jeannie +docs/**/*.md +infra/**/*.md +infra/**/*.yml +infra/**/*.yaml +infra/**/*.json +infra/**/*.txt +infra/**/*.sh +infra/**/*.py +bootstrap/**/*.md +bootstrap/**/*.tf +bootstrap/**/*.tftpl +bootstrap/**/*.yml +bootstrap/**/*.yaml +apps/**/*.md +apps/**/*.yaml +apps/**/*.yml +apps/**/*.tf +apps/**/*.php +apps/**/*.sh +scripts/* diff --git a/jeannie b/jeannie index c9b5e1c..1992da6 100755 --- a/jeannie +++ b/jeannie @@ -65,6 +65,7 @@ mapping = { "ai_gateway.url": "LAB_AI_GATEWAY_URL", "ai_gateway.model": "LAB_AI_GATEWAY_MODEL", "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS", + "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR", "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", } @@ -3839,7 +3840,9 @@ backstage_brain_note() { local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}" + local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" local context + local retrieved_context="" if ! backstage_brain_enabled; then return 0 @@ -3849,6 +3852,15 @@ backstage_brain_note() { fi context="$(cat)" + if [[ -s "${index_dir}/index.json" ]]; then + retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)" + fi + if [[ -n "${retrieved_context}" ]]; then + context="${context} + +Retrieved homelab knowledge: +${retrieved_context}" + fi BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true import json import os @@ -4614,6 +4626,83 @@ tailnet_policy_check() { "${REPO_ROOT}/scripts/validate-tailnet-policy" } +ai_index() { + local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" + + require_debian_server "ai-index" + ensure_python3 + if [[ "${index_dir}" == /data/* ]]; then + sudo mkdir -p "${index_dir}" + sudo chown "${USER}:$(id -gn)" "${index_dir}" + fi + "${REPO_ROOT}/scripts/build-homelab-ai-index" --index-dir "${index_dir}" +} + +ai_check() { + local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" + local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" + local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" + local failures=0 + + require_debian_server "ai-check" + + status_section "AI Knowledge" + if command -v python3 >/dev/null 2>&1; then + printf '%-28s ok\n' "python3" + else + printf '%-28s missing\n' "python3" + failures=$((failures + 1)) + fi + + if [[ -s "${index_dir}/index.json" ]]; then + printf '%-28s %s\n' "knowledge index" "${index_dir}/index.json" + else + printf '%-28s missing - run ./jeannie ai-index\n' "knowledge index" + failures=$((failures + 1)) + fi + + if "${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 2 "gitea edge traefik" >/dev/null 2>&1; then + printf '%-28s ok\n' "retrieval smoke test" + else + printf '%-28s failed\n' "retrieval smoke test" + failures=$((failures + 1)) + fi + + if backstage_brain_enabled; then + if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then + printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}" + if python3 - "${endpoint}" "${model}" <<'PY' +import json +import sys +import urllib.request + +endpoint, model = sys.argv[1:3] +with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response: + tags = json.loads(response.read().decode("utf-8")) +models = {item.get("name", "") for item in tags.get("models", [])} +raise SystemExit(0 if model in models or f"{model}:latest" in models else 1) +PY + then + printf '%-28s ok - %s\n' "ollama model" "${model}" + else + printf '%-28s missing - run: ollama pull %s\n' "ollama model" "${model}" + failures=$((failures + 1)) + fi + else + printf '%-28s unreachable - %s\n' "ollama endpoint" "${endpoint}" + failures=$((failures + 1)) + fi + else + printf '%-28s disabled\n' "backstage helper" + fi + + if ((failures > 0)); then + echo "AI check failed with ${failures} issue(s)." >&2 + exit 1 + fi + echo "AI check passed." +} + case "${1:-}" in up) up @@ -4687,6 +4776,12 @@ case "${1:-}" in tailnet-policy-check) tailnet_policy_check ;; + ai-index) + ai_index + ;; + ai-check) + ai_check + ;; openwrt) openwrt ;; @@ -4694,7 +4789,7 @@ case "${1:-}" in nuke ;; *) - echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|secrets-init|secrets-check|tailnet-policy-check|openwrt|nuke}" + echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}" exit 1 ;; esac diff --git a/scripts/build-homelab-ai-index b/scripts/build-homelab-ai-index new file mode 100755 index 0000000..ff58c2b --- /dev/null +++ b/scripts/build-homelab-ai-index @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Build a small local retrieval index from non-secret homelab repo files.""" + +from __future__ import annotations + +import argparse +import fnmatch +import json +import math +import os +import pathlib +import re +import sys +from collections import Counter + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_SOURCES_FILE = REPO_ROOT / "infra" / "ai" / "knowledge-sources.txt" +DEFAULT_INDEX_DIR = pathlib.Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index")) +TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]{2,}") +MAX_FILE_BYTES = 512 * 1024 +CHUNK_TARGET_LINES = 80 +CHUNK_OVERLAP_LINES = 12 + + +def tokenise(text: str) -> list[str]: + return [token.lower() for token in TOKEN_RE.findall(text)] + + +def load_patterns(path: pathlib.Path) -> list[str]: + patterns: list[str] = [] + for raw_line in path.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + patterns.append(line) + return patterns + + +def path_is_secret(path: pathlib.Path) -> bool: + lowered = str(path).lower() + secret_markers = ( + ".git/", + ".terraform/", + ".lab/", + "__pycache__/", + "terraform.tfstate", + ".secret.", + ".enc.", + "id_ed25519", + "cosign.key", + "keys.txt", + ) + return any(marker in lowered for marker in secret_markers) + + +def candidate_files(patterns: list[str]) -> list[pathlib.Path]: + files: set[pathlib.Path] = set() + all_files = [path for path in REPO_ROOT.rglob("*") if path.is_file()] + + for pattern in patterns: + if any(char in pattern for char in "*?["): + for path in all_files: + rel = path.relative_to(REPO_ROOT).as_posix() + if fnmatch.fnmatch(rel, pattern): + files.add(path) + else: + path = REPO_ROOT / pattern + if path.is_file(): + files.add(path) + + return sorted(path for path in files if not path_is_secret(path)) + + +def read_text(path: pathlib.Path) -> str | None: + try: + if path.stat().st_size > MAX_FILE_BYTES: + return None + return path.read_text(encoding="utf-8") + except UnicodeDecodeError: + return None + + +def chunk_file(path: pathlib.Path, text: str) -> list[dict[str, object]]: + rel = path.relative_to(REPO_ROOT).as_posix() + lines = text.splitlines() + chunks: list[dict[str, object]] = [] + start = 0 + + while start < len(lines): + end = min(start + CHUNK_TARGET_LINES, len(lines)) + body = "\n".join(lines[start:end]).strip() + if body: + chunks.append( + { + "id": f"{rel}:{start + 1}", + "path": rel, + "start_line": start + 1, + "end_line": end, + "text": body, + "tokens": tokenise(body), + } + ) + if end == len(lines): + break + start = max(end - CHUNK_OVERLAP_LINES, start + 1) + + if not chunks and text.strip(): + chunks.append( + { + "id": f"{rel}:1", + "path": rel, + "start_line": 1, + "end_line": 1, + "text": text.strip(), + "tokens": tokenise(text), + } + ) + + return chunks + + +def build_index(files: list[pathlib.Path]) -> dict[str, object]: + chunks: list[dict[str, object]] = [] + document_frequency: Counter[str] = Counter() + + for path in files: + text = read_text(path) + if text is None: + continue + for chunk in chunk_file(path, text): + tokens = chunk["tokens"] + if not isinstance(tokens, list) or not tokens: + continue + chunks.append(chunk) + document_frequency.update(set(tokens)) + + chunk_count = len(chunks) + average_length = sum(len(chunk["tokens"]) for chunk in chunks) / chunk_count if chunk_count else 0 + idf = { + token: math.log(1 + (chunk_count - freq + 0.5) / (freq + 0.5)) + for token, freq in document_frequency.items() + } + + return { + "version": 1, + "repo_root": str(REPO_ROOT), + "chunk_count": chunk_count, + "average_length": average_length, + "chunks": chunks, + "idf": idf, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sources-file", type=pathlib.Path, default=DEFAULT_SOURCES_FILE) + parser.add_argument("--index-dir", type=pathlib.Path, default=DEFAULT_INDEX_DIR) + args = parser.parse_args() + + if not args.sources_file.is_file(): + print(f"missing sources file: {args.sources_file}", file=sys.stderr) + return 1 + + patterns = load_patterns(args.sources_file) + files = candidate_files(patterns) + index = build_index(files) + + args.index_dir.mkdir(parents=True, exist_ok=True) + index_path = args.index_dir / "index.json" + manifest_path = args.index_dir / "manifest.json" + index_path.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="utf-8") + manifest_path.write_text( + json.dumps( + { + "sources_file": str(args.sources_file.relative_to(REPO_ROOT)), + "file_count": len(files), + "chunk_count": index["chunk_count"], + "repo_root": str(REPO_ROOT), + }, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + + print(f"indexed {len(files)} file(s), {index['chunk_count']} chunk(s) into {index_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/query-homelab-ai-index b/scripts/query-homelab-ai-index new file mode 100755 index 0000000..63f729a --- /dev/null +++ b/scripts/query-homelab-ai-index @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Query the local homelab retrieval index and optionally ask Ollama.""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import pathlib +import re +import sys +import urllib.error +import urllib.request +from collections import Counter + + +DEFAULT_INDEX_DIR = pathlib.Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index")) +TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]{2,}") + + +def tokenise(text: str) -> list[str]: + return [token.lower() for token in TOKEN_RE.findall(text)] + + +def load_index(index_dir: pathlib.Path) -> dict[str, object]: + index_path = index_dir / "index.json" + if not index_path.is_file(): + raise FileNotFoundError(f"missing index: {index_path}") + return json.loads(index_path.read_text(encoding="utf-8")) + + +def score_chunks(index: dict[str, object], query: str, limit: int) -> list[tuple[float, dict[str, object]]]: + query_terms = tokenise(query) + if not query_terms: + return [] + + query_counts = Counter(query_terms) + idf = index.get("idf", {}) + average_length = float(index.get("average_length") or 1.0) + chunks = index.get("chunks", []) + results: list[tuple[float, dict[str, object]]] = [] + k1 = 1.5 + b = 0.75 + + if not isinstance(idf, dict) or not isinstance(chunks, list): + return [] + + for chunk in chunks: + if not isinstance(chunk, dict): + continue + tokens = chunk.get("tokens", []) + if not isinstance(tokens, list) or not tokens: + continue + term_counts = Counter(str(token) for token in tokens) + doc_len = len(tokens) + score = 0.0 + for term, query_count in query_counts.items(): + frequency = term_counts.get(term, 0) + if frequency == 0: + continue + term_idf = float(idf.get(term, 0.0)) + denominator = frequency + k1 * (1 - b + b * doc_len / average_length) + score += query_count * term_idf * (frequency * (k1 + 1)) / denominator + if score > 0: + results.append((score, chunk)) + + return sorted(results, key=lambda item: item[0], reverse=True)[:limit] + + +def render_context(results: list[tuple[float, dict[str, object]]], max_chars: int) -> str: + sections: list[str] = [] + remaining = max_chars + + for score, chunk in results: + header = f"[{chunk.get('path')}:{chunk.get('start_line')}] score={score:.3f}" + text = str(chunk.get("text", "")).strip() + section = f"{header}\n{text}" + if len(section) > remaining: + section = section[: max(0, remaining - 20)].rstrip() + "\n[truncated]" + sections.append(section) + remaining -= len(section) + if remaining <= 0: + break + + return "\n\n---\n\n".join(sections) + + +def ask_ollama(question: str, context: str, endpoint: str, model: str, timeout: int) -> str: + prompt = f"""You are helping operate a personal homelab. +Use only the context below. If the context is insufficient, say what to inspect next. +Be concise, factual, and command-oriented. + +Question: +{question} + +Context: +{context} +""" + payload = { + "model": model, + "prompt": prompt, + "stream": False, + "options": { + "temperature": 0.1, + "num_predict": 360, + }, + } + request = urllib.request.Request( + f"{endpoint.rstrip('/')}/api/generate", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + result = json.loads(response.read().decode("utf-8")) + return str(result.get("response", "")).strip() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("query", nargs="+", help="Question or search text") + parser.add_argument("--index-dir", type=pathlib.Path, default=DEFAULT_INDEX_DIR) + parser.add_argument("--limit", type=int, default=5) + parser.add_argument("--max-chars", type=int, default=9000) + parser.add_argument("--context-only", action="store_true") + parser.add_argument("--ask", action="store_true") + parser.add_argument("--ollama-url", default=os.environ.get("LAB_AI_GATEWAY_URL", "http://127.0.0.1:11434")) + parser.add_argument("--model", default=os.environ.get("LAB_AI_GATEWAY_MODEL", "qwen2.5:0.5b")) + parser.add_argument("--timeout", type=int, default=int(os.environ.get("LAB_AI_GATEWAY_TIMEOUT_SECONDS", "20"))) + args = parser.parse_args() + + query = " ".join(args.query) + try: + index = load_index(args.index_dir) + except FileNotFoundError as exc: + print(str(exc), file=sys.stderr) + return 1 + + results = score_chunks(index, query, args.limit) + if not results: + print("no matching homelab knowledge chunks found", file=sys.stderr) + return 1 + + context = render_context(results, args.max_chars) + if args.context_only: + print(context) + return 0 + + if args.ask: + try: + answer = ask_ollama(query, context, args.ollama_url, args.model, args.timeout) + except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError) as exc: + print(f"ollama request failed: {exc}", file=sys.stderr) + return 1 + print(answer) + return 0 + + print(context) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())