From 14aa2555c1a236958836c95a06f43592c5095a28 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 11:50:10 -0600 Subject: [PATCH] Add AI incident commander --- README.md | 14 ++ README.md.tmpl | 14 ++ docs/jeannie.1.md | 5 + infra/incident-commander/README.md | 25 +++ .../fixtures/cluster-api-down.txt | 5 + .../incident-commander/fixtures/edge-502.txt | 7 + .../fixtures/gitea-backend.txt | 5 + .../fixtures/rpi-dns-timeout.txt | 5 + infra/incident-commander/incidents.tsv | 6 + jeannie | 8 + scripts/explain | 2 +- scripts/incident-commander | 208 ++++++++++++++++++ 12 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 infra/incident-commander/README.md create mode 100644 infra/incident-commander/fixtures/cluster-api-down.txt create mode 100644 infra/incident-commander/fixtures/edge-502.txt create mode 100644 infra/incident-commander/fixtures/gitea-backend.txt create mode 100644 infra/incident-commander/fixtures/rpi-dns-timeout.txt create mode 100644 infra/incident-commander/incidents.tsv create mode 100755 scripts/incident-commander diff --git a/README.md b/README.md index 0f0e2d7..d1c9470 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,20 @@ Run it without touching live infrastructure: Cases live under `security/prompt-injection-lab` and are meant to become regression tests for future Jeannie AI features. +## Incident Commander + +The repo also includes a read-only incident commander trainer. It classifies +pasted failure text, prints a likely hypothesis, links the right runbook, and +lists the next read-only Jeannie commands: + +```bash +./jeannie incident replay +./jeannie incident triage --text "502 Bad Gateway from nginx" +``` + +Incident fixtures live under `infra/incident-commander`. Use them to turn +repeat outages into repeatable triage behavior before adding any self-healing. + ## Deploying From the Debian server: diff --git a/README.md.tmpl b/README.md.tmpl index 1db0977..edc817a 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -180,6 +180,20 @@ Run it without touching live infrastructure: Cases live under `security/prompt-injection-lab` and are meant to become regression tests for future Jeannie AI features. +## Incident Commander + +The repo also includes a read-only incident commander trainer. It classifies +pasted failure text, prints a likely hypothesis, links the right runbook, and +lists the next read-only Jeannie commands: + +```bash +./{{ main_script }} incident replay +./{{ main_script }} incident triage --text "502 Bad Gateway from nginx" +``` + +Incident fixtures live under `infra/incident-commander`. Use them to turn +repeat outages into repeatable triage behavior before adding any self-healing. + ## Deploying From the Debian server: diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index f4ec0e1..1778b9e 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -87,6 +87,11 @@ verification. : Run the post-outage recovery sequence in dependency order: Debian runtime, Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks. +`incident {list|replay|triage}` +: Classify incident text, replay known fixtures, and print the likely +hypothesis, runbook, next read-only commands, and actions blocked until +explicitly approved. + `scorecard` : Roll up backup readiness, GitOps health, DNS/RPi health, cluster health, edge health, capacity pressure, security posture, public certificates, inventory, and diff --git a/infra/incident-commander/README.md b/infra/incident-commander/README.md new file mode 100644 index 0000000..fc4c652 --- /dev/null +++ b/infra/incident-commander/README.md @@ -0,0 +1,25 @@ +# AI Incident Commander + +This is a deterministic incident-triage trainer for the homelab. It is the +first step toward a Jeannie incident assistant: classify symptoms, map evidence +to a likely failure class, print read-only next commands, and make unsafe +actions explicit. + +Run fixture replays: + +```bash +./jeannie incident replay +``` + +Useful commands: + +```bash +./jeannie incident list +./jeannie incident replay edge_502 +./jeannie incident triage --from-file /tmp/incident-output.txt +./jeannie incident triage --text "curl shows 502 Bad Gateway from nginx" +``` + +The command is report-only. It does not run Kubernetes, SSH, OpenTofu, or Docker +commands for you. + diff --git a/infra/incident-commander/fixtures/cluster-api-down.txt b/infra/incident-commander/fixtures/cluster-api-down.txt new file mode 100644 index 0000000..7f02717 --- /dev/null +++ b/infra/incident-commander/fixtures/cluster-api-down.txt @@ -0,0 +1,5 @@ +kubectl get nodes + +The connection to the server 192.168.100.73:6443 was refused - did you specify +the right host or port? + diff --git a/infra/incident-commander/fixtures/edge-502.txt b/infra/incident-commander/fixtures/edge-502.txt new file mode 100644 index 0000000..3bf3379 --- /dev/null +++ b/infra/incident-commander/fixtures/edge-502.txt @@ -0,0 +1,7 @@ +Browser shows: + +502 Bad Gateway +nginx/1.31.2 + +The public website URL was working earlier. + diff --git a/infra/incident-commander/fixtures/gitea-backend.txt b/infra/incident-commander/fixtures/gitea-backend.txt new file mode 100644 index 0000000..f71839f --- /dev/null +++ b/infra/incident-commander/fixtures/gitea-backend.txt @@ -0,0 +1,5 @@ +OpenTofu edge apply failed: + +Cannot reach Gitea backend http://100.85.138.30:3000/git/ from the edge host. +Check the Debian host Tailscale IP, Gitea container, and port publishing. + diff --git a/infra/incident-commander/fixtures/rpi-dns-timeout.txt b/infra/incident-commander/fixtures/rpi-dns-timeout.txt new file mode 100644 index 0000000..b38a7b2 --- /dev/null +++ b/infra/incident-commander/fixtures/rpi-dns-timeout.txt @@ -0,0 +1,5 @@ +dig @192.168.100.89 cloudflare.com A +time=3 +tries=1 +short + +;; communications error to 192.168.100.89#53: timed out +;; no servers could be reached + diff --git a/infra/incident-commander/incidents.tsv b/infra/incident-commander/incidents.tsv new file mode 100644 index 0000000..3f98fdd --- /dev/null +++ b/infra/incident-commander/incidents.tsv @@ -0,0 +1,6 @@ +id title fixture expected_class runbook next_commands forbidden_actions +edge_502 Public edge returns 502 edge-502.txt edge_bad_gateway docs/runbooks/edge-failures.md ./jeannie doctor-edge;./jeannie doctor-cluster terraform destroy;./jeannie nuke +gitea_backend_unreachable Edge cannot reach Gitea backend gitea-backend.txt gitea_edge_backend docs/runbooks/gitea-failures.md ./jeannie doctor-gitea;./jeannie doctor-edge docker rm;reset gitea database +cluster_api_down Kubernetes API is refused cluster-api-down.txt cluster_api_down docs/runbooks/cluster-stop-start-failures.md ./jeannie start-cluster;./jeannie doctor-cluster kubeadm reset;./jeannie nuke +rpi_dns_timeout Pi-hole DNS timeout rpi-dns-timeout.txt dns_failure docs/runbooks/edge-failures.md ./jeannie doctor-rpi;./jeannie status docker system prune;disable firewall + diff --git a/jeannie b/jeannie index 62b5cd2..5594f6d 100755 --- a/jeannie +++ b/jeannie @@ -6326,6 +6326,10 @@ prompt_injection_lab() { "${REPO_ROOT}/scripts/prompt-injection-lab" "${@:2}" } +incident_commander() { + "${REPO_ROOT}/scripts/incident-commander" "${@:2}" +} + validate_homelab() { "${REPO_ROOT}/scripts/validate-homelab" } @@ -6519,6 +6523,7 @@ Operator View And Reports capacity-limits [namespace] Recommend requests/limits YAML snippets. recover-plan Print disaster recovery order and prerequisites. recover-power [--dry-run] Run or preview post-outage recovery. + incident {list|replay|triage} Classify incident text and suggest read-only next steps. impact [--since REF] [PATH...] Explain affected lab areas before changes. review-last-change [REF] Summarize changed files, impact, validation. map [--dot] Print the homelab dependency map. @@ -6684,6 +6689,9 @@ case "${1:-}" in recover-power) recover_power "$@" ;; + incident) + incident_commander "$@" + ;; scorecard) scorecard ;; diff --git a/scripts/explain b/scripts/explain index 209f351..fdf236f 100755 --- a/scripts/explain +++ b/scripts/explain @@ -34,7 +34,7 @@ EOF is_read_only_command() { case "$1" in - status|capacity|capacity-advisor|capacity-limits|recover-plan|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane) + status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane) return 0 ;; *) diff --git a/scripts/incident-commander b/scripts/incident-commander new file mode 100755 index 0000000..99b1672 --- /dev/null +++ b/scripts/incident-commander @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Read-only incident classification and triage for homelab failures.""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +INCIDENT_DIR = REPO_ROOT / "infra" / "incident-commander" +INCIDENTS_FILE = INCIDENT_DIR / "incidents.tsv" +FIXTURES_DIR = INCIDENT_DIR / "fixtures" + + +@dataclass +class Incident: + incident_id: str + title: str + fixture: str + expected_class: str + runbook: str + next_commands: list[str] + forbidden_actions: list[str] + + +RULES: list[tuple[str, str, str]] = [ + ("edge_bad_gateway", r"\b502\b|bad gateway|nginx/1\.31", "Public edge proxy cannot get a healthy upstream response."), + ("gitea_edge_backend", r"cannot reach gitea backend|100\.85\.138\.30:3000|/git/", "Edge-to-Gitea backend path is broken or Gitea is unhealthy."), + ("cluster_api_down", r"6443.*refused|api server.*refused|connection to the server .* was refused", "Kubernetes API is not accepting connections."), + ("dns_failure", r"no servers could be reached|communications error .*#53|timed out", "RPi/Pi-hole DNS path is unavailable or blocked."), +] + + +def split_items(value: str) -> list[str]: + return [item.strip() for item in value.split(";") if item.strip()] + + +def load_incidents() -> list[Incident]: + with INCIDENTS_FILE.open(encoding="utf-8", newline="") as handle: + rows = csv.DictReader(handle, delimiter="\t") + return [ + Incident( + incident_id=row["id"], + title=row["title"], + fixture=row["fixture"], + expected_class=row["expected_class"], + runbook=row["runbook"], + next_commands=split_items(row["next_commands"]), + forbidden_actions=split_items(row["forbidden_actions"]), + ) + for row in rows + ] + + +def fixture_text(incident: Incident) -> str: + path = FIXTURES_DIR / incident.fixture + if not path.is_file(): + raise SystemExit(f"Missing fixture for {incident.incident_id}: {path}") + return path.read_text(encoding="utf-8") + + +def classify(text: str) -> tuple[str, str]: + lowered = text.lower() + for incident_class, pattern, hypothesis in RULES: + if re.search(pattern, lowered, re.IGNORECASE | re.DOTALL): + return incident_class, hypothesis + return "unknown", "No known incident pattern matched. Preserve evidence and run read-only status checks." + + +def incident_for_class(incidents: list[Incident], incident_class: str) -> Incident | None: + for incident in incidents: + if incident.expected_class == incident_class: + return incident + return None + + +def render_triage(text: str, incidents: list[Incident]) -> dict[str, object]: + incident_class, hypothesis = classify(text) + incident = incident_for_class(incidents, incident_class) + if incident is None: + return { + "class": incident_class, + "hypothesis": hypothesis, + "runbook": "none", + "next_commands": ["./jeannie status", "./jeannie scorecard"], + "forbidden_actions": ["destructive commands", "secret exposure", "unverified manual fixes"], + "evidence": first_evidence_lines(text), + } + return { + "class": incident_class, + "hypothesis": hypothesis, + "runbook": incident.runbook, + "next_commands": incident.next_commands, + "forbidden_actions": incident.forbidden_actions, + "evidence": first_evidence_lines(text), + } + + +def first_evidence_lines(text: str) -> list[str]: + lines = [line.strip() for line in text.splitlines() if line.strip()] + return lines[:6] + + +def print_triage(result: dict[str, object]) -> None: + print("AI Incident Commander") + print("=====================") + print("mode: read-only") + print() + print(f"Class: {result['class']}") + print(f"Hypothesis: {result['hypothesis']}") + print(f"Runbook: {result['runbook']}") + print() + print("Evidence:") + for line in result["evidence"]: + print(f" {line}") + print() + print("Next read-only commands:") + for command in result["next_commands"]: + print(f" {command}") + print() + print("Blocked until explicitly approved:") + for action in result["forbidden_actions"]: + print(f" {action}") + + +def print_list(incidents: list[Incident]) -> int: + for incident in incidents: + print(f"{incident.incident_id}\t{incident.expected_class}\t{incident.title}") + return 0 + + +def replay(incidents: list[Incident], incident_id: str | None, as_json: bool, details: bool) -> int: + selected = [incident for incident in incidents if incident_id in (None, incident.incident_id)] + if not selected: + print(f"Unknown incident fixture: {incident_id}", file=sys.stderr) + return 2 + + results = [] + for incident in selected: + text = fixture_text(incident) + result = render_triage(text, incidents) + status = "pass" if result["class"] == incident.expected_class else "fail" + results.append({"id": incident.incident_id, "status": status, "expected": incident.expected_class, **result}) + + if as_json: + print(json.dumps(results, indent=2, sort_keys=True)) + else: + failures = sum(1 for result in results if result["status"] != "pass") + print("Incident Replay") + print("===============") + print(f"cases={len(results)} failures={failures}") + print() + for result in results: + print(f"{result['status']:5} {result['id']} - {result['class']}") + print(f" next: {result['next_commands'][0]}") + if details: + print(f" runbook: {result['runbook']}") + print(f" hypothesis: {result['hypothesis']}") + return 1 if any(result["status"] != "pass" for result in results) else 0 + + +def read_triage_input(args: argparse.Namespace) -> str: + if args.text: + return args.text + if args.from_file: + return Path(args.from_file).read_text(encoding="utf-8", errors="replace") + if not sys.stdin.isatty(): + return sys.stdin.read() + raise SystemExit("Provide --text, --from-file, or stdin for incident triage.") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("list") + replay_parser = subparsers.add_parser("replay") + replay_parser.add_argument("incident_id", nargs="?") + replay_parser.add_argument("--json", action="store_true") + replay_parser.add_argument("--details", action="store_true") + triage_parser = subparsers.add_parser("triage") + triage_parser.add_argument("--from-file") + triage_parser.add_argument("--text") + triage_parser.add_argument("--json", action="store_true") + args = parser.parse_args() + + incidents = load_incidents() + if args.command == "list": + return print_list(incidents) + if args.command == "replay": + return replay(incidents, args.incident_id, args.json, args.details) + if args.command == "triage": + result = render_triage(read_triage_input(args), incidents) + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + else: + print_triage(result) + return 0 + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())