From 3c24b7fbc44f9142c2bea756fc84180210132b56 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Mon, 29 Jun 2026 21:05:30 -0600 Subject: [PATCH] Add AI knowledge exclude list --- README.md | 3 ++- README.md.tmpl | 3 ++- infra/ai/knowledge-excludes.txt | 6 ++++++ scripts/build-homelab-ai-index | 19 ++++++++++++++++--- 4 files changed, 26 insertions(+), 5 deletions(-) create mode 100644 infra/ai/knowledge-excludes.txt diff --git a/README.md b/README.md index 6b902d0..29e25bb 100644 --- a/README.md +++ b/README.md @@ -1113,7 +1113,8 @@ Build the local homelab knowledge index separately from the main deployment: ``` The index is built from the non-secret source list in -`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`. +`infra/ai/knowledge-sources.txt`, skips paths in +`infra/ai/knowledge-excludes.txt`, and defaults to `/data/homelab-ai/index`. Doctor commands use it as extra context when it exists, but infrastructure deployment does not depend on it. Use `ask` as the runbook and command finder. It returns the closest indexed diff --git a/README.md.tmpl b/README.md.tmpl index f587757..ff48975 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -1113,7 +1113,8 @@ Build the local homelab knowledge index separately from the main deployment: ``` The index is built from the non-secret source list in -`infra/ai/knowledge-sources.txt` and defaults to `/data/homelab-ai/index`. +`infra/ai/knowledge-sources.txt`, skips paths in +`infra/ai/knowledge-excludes.txt`, and defaults to `/data/homelab-ai/index`. Doctor commands use it as extra context when it exists, but infrastructure deployment does not depend on it. Use `ask` as the runbook and command finder. It returns the closest indexed diff --git a/infra/ai/knowledge-excludes.txt b/infra/ai/knowledge-excludes.txt new file mode 100644 index 0000000..24426d8 --- /dev/null +++ b/infra/ai/knowledge-excludes.txt @@ -0,0 +1,6 @@ +# Files and globs excluded from the local homelab AI helper. +# Keep inactive reference material out of answers about the active lab. +archive/** +labs/** +apps/arr-stack/** +apps/mlops-platform/** diff --git a/scripts/build-homelab-ai-index b/scripts/build-homelab-ai-index index ff58c2b..1bcafff 100755 --- a/scripts/build-homelab-ai-index +++ b/scripts/build-homelab-ai-index @@ -16,6 +16,7 @@ from collections import Counter REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] DEFAULT_SOURCES_FILE = REPO_ROOT / "infra" / "ai" / "knowledge-sources.txt" +DEFAULT_EXCLUDES_FILE = REPO_ROOT / "infra" / "ai" / "knowledge-excludes.txt" DEFAULT_INDEX_DIR = pathlib.Path(os.environ.get("LAB_AI_KNOWLEDGE_INDEX_DIR", "/data/homelab-ai/index")) TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]{2,}") MAX_FILE_BYTES = 512 * 1024 @@ -54,7 +55,11 @@ def path_is_secret(path: pathlib.Path) -> bool: return any(marker in lowered for marker in secret_markers) -def candidate_files(patterns: list[str]) -> list[pathlib.Path]: +def path_matches_any(rel_path: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(rel_path, pattern) for pattern in patterns) + + +def candidate_files(patterns: list[str], exclude_patterns: list[str]) -> list[pathlib.Path]: files: set[pathlib.Path] = set() all_files = [path for path in REPO_ROOT.rglob("*") if path.is_file()] @@ -69,7 +74,12 @@ def candidate_files(patterns: list[str]) -> list[pathlib.Path]: if path.is_file(): files.add(path) - return sorted(path for path in files if not path_is_secret(path)) + return sorted( + path + for path in files + if not path_is_secret(path) + and not path_matches_any(path.relative_to(REPO_ROOT).as_posix(), exclude_patterns) + ) def read_text(path: pathlib.Path) -> str | None: @@ -155,6 +165,7 @@ def build_index(files: list[pathlib.Path]) -> dict[str, object]: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--sources-file", type=pathlib.Path, default=DEFAULT_SOURCES_FILE) + parser.add_argument("--excludes-file", type=pathlib.Path, default=DEFAULT_EXCLUDES_FILE) parser.add_argument("--index-dir", type=pathlib.Path, default=DEFAULT_INDEX_DIR) args = parser.parse_args() @@ -163,7 +174,8 @@ def main() -> int: return 1 patterns = load_patterns(args.sources_file) - files = candidate_files(patterns) + exclude_patterns = load_patterns(args.excludes_file) if args.excludes_file.is_file() else [] + files = candidate_files(patterns, exclude_patterns) index = build_index(files) args.index_dir.mkdir(parents=True, exist_ok=True) @@ -174,6 +186,7 @@ def main() -> int: json.dumps( { "sources_file": str(args.sources_file.relative_to(REPO_ROOT)), + "excludes_file": str(args.excludes_file.relative_to(REPO_ROOT)) if args.excludes_file.is_file() else None, "file_count": len(files), "chunk_count": index["chunk_count"], "repo_root": str(REPO_ROOT),