jv
/
my-homelab-configs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add optional homelab AI knowledge index

This commit is contained in:
juvdiaz 2026-06-27 23:14:27 -06:00
parent 8488c9523e
commit 172bcca975
8 changed files with 552 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
README.md
View File

@ -868,6 +868,18 @@ Disable it for low-CPU sessions with:
LAB_BACKSTAGE_BRAIN_ENABLED=false ./jeannie doctor-edge
```
Build the local homelab knowledge index separately from the main deployment:
```bash
./jeannie ai-index
./jeannie ai-check
```
The index is built from the non-secret source list in
`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`.
Doctor commands use it as extra context when it exists, but infrastructure
deployment does not depend on it.
The CV page has two client-side presentation modes:
- `Elegant`: dark, minimal, terminal-inspired styling with a square profile

12
README.md.tmpl
View File

@ -868,6 +868,18 @@ Disable it for low-CPU sessions with:
LAB_BACKSTAGE_BRAIN_ENABLED=false ./{{ main_script }} doctor-edge
```
Build the local homelab knowledge index separately from the main deployment:
```bash
./{{ main_script }} ai-index
./{{ main_script }} ai-check
```
The index is built from the non-secret source list in
`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`.
Doctor commands use it as extra context when it exists, but infrastructure
deployment does not depend on it.
The CV page has two client-side presentation modes:
- `Elegant`: dark, minimal, terminal-inspired styling with a square profile

1
homelab.yml
View File

@ -124,3 +124,4 @@ ai_gateway:
url: http://192.168.100.73:11434
model: qwen2.5:0.5b
timeout_seconds: 20
knowledge_index_dir: /data/homelab-ai/index

49
infra/ai/README.md Normal file
View File

@ -0,0 +1,49 @@
# Homelab AI Knowledge Index
This directory defines the optional local knowledge layer for the backstage
`jeannie` helper.
The index is not model training. It is retrieval over repo files, so current
inventory, runbooks, scripts, Terraform, compose files, and app manifests remain
normal source-controlled text.
## Build
Run on the Debian homelab server:
```bash
./jeannie ai-index
```
The default index path comes from `homelab.yml`:
```text
/data/homelab-ai/index
```
Override it with `LAB_AI_KNOWLEDGE_INDEX_DIR` when testing.
## Check
```bash
./jeannie ai-check
```
This validates:
- Python is available
- the index exists
- a simple retrieval query returns context
- Ollama is reachable when the backstage helper is enabled
- the configured model is pulled
## Query
The query script can be used directly:
```bash
scripts/query-homelab-ai-index "gitea public url is failing"
scripts/query-homelab-ai-index --ask "why is lab2025 returning 502?"
```
`--ask` sends retrieved context to the configured Ollama endpoint and model.

25
infra/ai/knowledge-sources.txt Normal file
View File

@ -0,0 +1,25 @@
# Files and globs indexed for the local homelab AI helper.
# Keep this non-secret. The index is built from repository content only.
homelab.yml
README.md
jeannie
docs/**/*.md
infra/**/*.md
infra/**/*.yml
infra/**/*.yaml
infra/**/*.json
infra/**/*.txt
infra/**/*.sh
infra/**/*.py
bootstrap/**/*.md
bootstrap/**/*.tf
bootstrap/**/*.tftpl
bootstrap/**/*.yml
bootstrap/**/*.yaml
apps/**/*.md
apps/**/*.yaml
apps/**/*.yml
apps/**/*.tf
apps/**/*.php
apps/**/*.sh
scripts/*

97
jeannie
View File

@ -65,6 +65,7 @@ mapping = {
"ai_gateway.url": "LAB_AI_GATEWAY_URL",
"ai_gateway.model": "LAB_AI_GATEWAY_MODEL",
"ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS",
"ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR",
"pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID",
"pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT",
}
@ -3839,7 +3840,9 @@ backstage_brain_note() {
local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}"
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}"
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
local context
local retrieved_context=""
if ! backstage_brain_enabled; then
return 0
@ -3849,6 +3852,15 @@ backstage_brain_note() {
fi
context="$(cat)"
if [[ -s "${index_dir}/index.json" ]]; then
retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)"
fi
if [[ -n "${retrieved_context}" ]]; then
context="${context}
Retrieved homelab knowledge:
${retrieved_context}"
fi
BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true
import json
import os
@ -4614,6 +4626,83 @@ tailnet_policy_check() {
"${REPO_ROOT}/scripts/validate-tailnet-policy"
}
ai_index() {
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
require_debian_server "ai-index"
ensure_python3
if [[ "${index_dir}" == /data/* ]]; then
sudo mkdir -p "${index_dir}"
sudo chown "${USER}:$(id -gn)" "${index_dir}"
fi
"${REPO_ROOT}/scripts/build-homelab-ai-index" --index-dir "${index_dir}"
}
ai_check() {
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}"
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
local failures=0
require_debian_server "ai-check"
status_section "AI Knowledge"
if command -v python3 >/dev/null 2>&1; then
printf '%-28s ok\n' "python3"
else
printf '%-28s missing\n' "python3"
failures=$((failures + 1))
fi
if [[ -s "${index_dir}/index.json" ]]; then
printf '%-28s %s\n' "knowledge index" "${index_dir}/index.json"
else
printf '%-28s missing - run ./jeannie ai-index\n' "knowledge index"
failures=$((failures + 1))
fi
if "${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 2 "gitea edge traefik" >/dev/null 2>&1; then
printf '%-28s ok\n' "retrieval smoke test"
else
printf '%-28s failed\n' "retrieval smoke test"
failures=$((failures + 1))
fi
if backstage_brain_enabled; then
if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then
printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}"
if python3 - "${endpoint}" "${model}" <<'PY'
import json
import sys
import urllib.request
endpoint, model = sys.argv[1:3]
with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response:
tags = json.loads(response.read().decode("utf-8"))
models = {item.get("name", "") for item in tags.get("models", [])}
raise SystemExit(0 if model in models or f"{model}:latest" in models else 1)
PY
then
printf '%-28s ok - %s\n' "ollama model" "${model}"
else
printf '%-28s missing - run: ollama pull %s\n' "ollama model" "${model}"
failures=$((failures + 1))
fi
else
printf '%-28s unreachable - %s\n' "ollama endpoint" "${endpoint}"
failures=$((failures + 1))
fi
else
printf '%-28s disabled\n' "backstage helper"
fi
if ((failures > 0)); then
echo "AI check failed with ${failures} issue(s)." >&2
exit 1
fi
echo "AI check passed."
}
case "${1:-}" in
up)
up
@ -4687,6 +4776,12 @@ case "${1:-}" in
tailnet-policy-check)
tailnet_policy_check
;;
ai-index)
ai_index
;;
ai-check)
ai_check
;;
openwrt)
openwrt
;;
@ -4694,7 +4789,7 @@ case "${1:-}" in
nuke
;;
*)
echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|secrets-init|secrets-check|tailnet-policy-check|openwrt|nuke}"
echo "Usage: $0 {up|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}"
exit 1
;;
esac

193
scripts/build-homelab-ai-index Executable file
View File

@ -0,0 +1,193 @@
#!/usr/bin/env python3
"""Build a small local retrieval index from non-secret homelab repo files."""
from __future__ import annotations
import argparse
import fnmatch
import json
import math
import os
import pathlib
import re
import sys
from collections import Counter
REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
DEFAULT_SOURCES_FILE = REPO_ROOT / "infra" / "ai" / "knowledge-sources.txt"
DEFAULT_INDEX_DIR = pathlib.Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index"))
TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]{2,}")
MAX_FILE_BYTES = 512 * 1024
CHUNK_TARGET_LINES = 80
CHUNK_OVERLAP_LINES = 12
def tokenise(text: str) -> list[str]:
return [token.lower() for token in TOKEN_RE.findall(text)]
def load_patterns(path: pathlib.Path) -> list[str]:
patterns: list[str] = []
for raw_line in path.read_text(encoding="utf-8").splitlines():
line = raw_line.strip()
if not line or line.startswith("#"):
continue
patterns.append(line)
return patterns
def path_is_secret(path: pathlib.Path) -> bool:
lowered = str(path).lower()
secret_markers = (
".git/",
".terraform/",
".lab/",
"__pycache__/",
"terraform.tfstate",
".secret.",
".enc.",
"id_ed25519",
"cosign.key",
"keys.txt",
)
return any(marker in lowered for marker in secret_markers)
def candidate_files(patterns: list[str]) -> list[pathlib.Path]:
files: set[pathlib.Path] = set()
all_files = [path for path in REPO_ROOT.rglob("*") if path.is_file()]
for pattern in patterns:
if any(char in pattern for char in "*?["):
for path in all_files:
rel = path.relative_to(REPO_ROOT).as_posix()
if fnmatch.fnmatch(rel, pattern):
files.add(path)
else:
path = REPO_ROOT / pattern
if path.is_file():
files.add(path)
return sorted(path for path in files if not path_is_secret(path))
def read_text(path: pathlib.Path) -> str | None:
try:
if path.stat().st_size > MAX_FILE_BYTES:
return None
return path.read_text(encoding="utf-8")
except UnicodeDecodeError:
return None
def chunk_file(path: pathlib.Path, text: str) -> list[dict[str, object]]:
rel = path.relative_to(REPO_ROOT).as_posix()
lines = text.splitlines()
chunks: list[dict[str, object]] = []
start = 0
while start < len(lines):
end = min(start + CHUNK_TARGET_LINES, len(lines))
body = "\n".join(lines[start:end]).strip()
if body:
chunks.append(
{
"id": f"{rel}:{start + 1}",
"path": rel,
"start_line": start + 1,
"end_line": end,
"text": body,
"tokens": tokenise(body),
}
)
if end == len(lines):
break
start = max(end - CHUNK_OVERLAP_LINES, start + 1)
if not chunks and text.strip():
chunks.append(
{
"id": f"{rel}:1",
"path": rel,
"start_line": 1,
"end_line": 1,
"text": text.strip(),
"tokens": tokenise(text),
}
)
return chunks
def build_index(files: list[pathlib.Path]) -> dict[str, object]:
chunks: list[dict[str, object]] = []
document_frequency: Counter[str] = Counter()
for path in files:
text = read_text(path)
if text is None:
continue
for chunk in chunk_file(path, text):
tokens = chunk["tokens"]
if not isinstance(tokens, list) or not tokens:
continue
chunks.append(chunk)
document_frequency.update(set(tokens))
chunk_count = len(chunks)
average_length = sum(len(chunk["tokens"]) for chunk in chunks) / chunk_count if chunk_count else 0
idf = {
token: math.log(1 + (chunk_count - freq + 0.5) / (freq + 0.5))
for token, freq in document_frequency.items()
}
return {
"version": 1,
"repo_root": str(REPO_ROOT),
"chunk_count": chunk_count,
"average_length": average_length,
"chunks": chunks,
"idf": idf,
}
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--sources-file", type=pathlib.Path, default=DEFAULT_SOURCES_FILE)
parser.add_argument("--index-dir", type=pathlib.Path, default=DEFAULT_INDEX_DIR)
args = parser.parse_args()
if not args.sources_file.is_file():
print(f"missing sources file: {args.sources_file}", file=sys.stderr)
return 1
patterns = load_patterns(args.sources_file)
files = candidate_files(patterns)
index = build_index(files)
args.index_dir.mkdir(parents=True, exist_ok=True)
index_path = args.index_dir / "index.json"
manifest_path = args.index_dir / "manifest.json"
index_path.write_text(json.dumps(index, indent=2, sort_keys=True) + "\n", encoding="utf-8")
manifest_path.write_text(
json.dumps(
{
"sources_file": str(args.sources_file.relative_to(REPO_ROOT)),
"file_count": len(files),
"chunk_count": index["chunk_count"],
"repo_root": str(REPO_ROOT),
},
indent=2,
sort_keys=True,
)
+ "\n",
encoding="utf-8",
)
print(f"indexed {len(files)} file(s), {index['chunk_count']} chunk(s) into {index_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())

164
scripts/query-homelab-ai-index Executable file
View File

@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""Query the local homelab retrieval index and optionally ask Ollama."""
from __future__ import annotations
import argparse
import json
import math
import os
import pathlib
import re
import sys
import urllib.error
import urllib.request
from collections import Counter
DEFAULT_INDEX_DIR = pathlib.Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index"))
TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]{2,}")
def tokenise(text: str) -> list[str]:
return [token.lower() for token in TOKEN_RE.findall(text)]
def load_index(index_dir: pathlib.Path) -> dict[str, object]:
index_path = index_dir / "index.json"
if not index_path.is_file():
raise FileNotFoundError(f"missing index: {index_path}")
return json.loads(index_path.read_text(encoding="utf-8"))
def score_chunks(index: dict[str, object], query: str, limit: int) -> list[tuple[float, dict[str, object]]]:
query_terms = tokenise(query)
if not query_terms:
return []
query_counts = Counter(query_terms)
idf = index.get("idf", {})
average_length = float(index.get("average_length") or 1.0)
chunks = index.get("chunks", [])
results: list[tuple[float, dict[str, object]]] = []
k1 = 1.5
b = 0.75
if not isinstance(idf, dict) or not isinstance(chunks, list):
return []
for chunk in chunks:
if not isinstance(chunk, dict):
continue
tokens = chunk.get("tokens", [])
if not isinstance(tokens, list) or not tokens:
continue
term_counts = Counter(str(token) for token in tokens)
doc_len = len(tokens)
score = 0.0
for term, query_count in query_counts.items():
frequency = term_counts.get(term, 0)
if frequency == 0:
continue
term_idf = float(idf.get(term, 0.0))
denominator = frequency + k1 * (1 - b + b * doc_len / average_length)
score += query_count * term_idf * (frequency * (k1 + 1)) / denominator
if score > 0:
results.append((score, chunk))
return sorted(results, key=lambda item: item[0], reverse=True)[:limit]
def render_context(results: list[tuple[float, dict[str, object]]], max_chars: int) -> str:
sections: list[str] = []
remaining = max_chars
for score, chunk in results:
header = f"[{chunk.get('path')}:{chunk.get('start_line')}] score={score:.3f}"
text = str(chunk.get("text", "")).strip()
section = f"{header}\n{text}"
if len(section) > remaining:
section = section[: max(0, remaining - 20)].rstrip() + "\n[truncated]"
sections.append(section)
remaining -= len(section)
if remaining <= 0:
break
return "\n\n---\n\n".join(sections)
def ask_ollama(question: str, context: str, endpoint: str, model: str, timeout: int) -> str:
prompt = f"""You are helping operate a personal homelab.
Use only the context below. If the context is insufficient, say what to inspect next.
Be concise, factual, and command-oriented.
Question:
{question}
Context:
{context}
"""
payload = {
"model": model,
"prompt": prompt,
"stream": False,
"options": {
"temperature": 0.1,
"num_predict": 360,
},
}
request = urllib.request.Request(
f"{endpoint.rstrip('/')}/api/generate",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=timeout) as response:
result = json.loads(response.read().decode("utf-8"))
return str(result.get("response", "")).strip()
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("query", nargs="+", help="Question or search text")
parser.add_argument("--index-dir", type=pathlib.Path, default=DEFAULT_INDEX_DIR)
parser.add_argument("--limit", type=int, default=5)
parser.add_argument("--max-chars", type=int, default=9000)
parser.add_argument("--context-only", action="store_true")
parser.add_argument("--ask", action="store_true")
parser.add_argument("--ollama-url", default=os.environ.get("LAB_AI_GATEWAY_URL", "http://127.0.0.1:11434"))
parser.add_argument("--model", default=os.environ.get("LAB_AI_GATEWAY_MODEL", "qwen2.5:0.5b"))
parser.add_argument("--timeout", type=int, default=int(os.environ.get("LAB_AI_GATEWAY_TIMEOUT_SECONDS", "20")))
args = parser.parse_args()
query = " ".join(args.query)
try:
index = load_index(args.index_dir)
except FileNotFoundError as exc:
print(str(exc), file=sys.stderr)
return 1
results = score_chunks(index, query, args.limit)
if not results:
print("no matching homelab knowledge chunks found", file=sys.stderr)
return 1
context = render_context(results, args.max_chars)
if args.context_only:
print(context)
return 0
if args.ask:
try:
answer = ask_ollama(query, context, args.ollama_url, args.model, args.timeout)
except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError) as exc:
print(f"ollama request failed: {exc}", file=sys.stderr)
return 1
print(answer)
return 0
print(context)
return 0
if __name__ == "__main__":
raise SystemExit(main())