Add AI incident commander

README.md

@@ -180,6 +180,20 @@ Run it without touching live infrastructure:
 Cases live under `security/prompt-injection-lab` and are meant to become
 regression tests for future Jeannie AI features.
 
+## Incident Commander
+
+The repo also includes a read-only incident commander trainer. It classifies
+pasted failure text, prints a likely hypothesis, links the right runbook, and
+lists the next read-only Jeannie commands:
+
+```bash
+./jeannie incident replay
+./jeannie incident triage --text "502 Bad Gateway from nginx"
+```
+
+Incident fixtures live under `infra/incident-commander`. Use them to turn
+repeat outages into repeatable triage behavior before adding any self-healing.
+
 ## Deploying
 
 From the Debian server:

README.md.tmpl

@@ -180,6 +180,20 @@ Run it without touching live infrastructure:
 Cases live under `security/prompt-injection-lab` and are meant to become
 regression tests for future Jeannie AI features.
 
+## Incident Commander
+
+The repo also includes a read-only incident commander trainer. It classifies
+pasted failure text, prints a likely hypothesis, links the right runbook, and
+lists the next read-only Jeannie commands:
+
+```bash
+./{{ main_script }} incident replay
+./{{ main_script }} incident triage --text "502 Bad Gateway from nginx"
+```
+
+Incident fixtures live under `infra/incident-commander`. Use them to turn
+repeat outages into repeatable triage behavior before adding any self-healing.
+
 ## Deploying
 
 From the Debian server:

docs/jeannie.1.md

@@ -87,6 +87,11 @@ verification.
 : Run the post-outage recovery sequence in dependency order: Debian runtime,
 Gitea, RPi DNS, Pimox workers, Kubernetes, GitOps/apps, and edge/public checks.
 
+`incident {list|replay|triage}`
+: Classify incident text, replay known fixtures, and print the likely
+hypothesis, runbook, next read-only commands, and actions blocked until
+explicitly approved.
+
 `scorecard`
 : Roll up backup readiness, GitOps health, DNS/RPi health, cluster health, edge
 health, capacity pressure, security posture, public certificates, inventory, and

infra/incident-commander/README.md

@@ -0,0 +1,25 @@
+# AI Incident Commander
+
+This is a deterministic incident-triage trainer for the homelab. It is the
+first step toward a Jeannie incident assistant: classify symptoms, map evidence
+to a likely failure class, print read-only next commands, and make unsafe
+actions explicit.
+
+Run fixture replays:
+
+```bash
+./jeannie incident replay
+```
+
+Useful commands:
+
+```bash
+./jeannie incident list
+./jeannie incident replay edge_502
+./jeannie incident triage --from-file /tmp/incident-output.txt
+./jeannie incident triage --text "curl shows 502 Bad Gateway from nginx"
+```
+
+The command is report-only. It does not run Kubernetes, SSH, OpenTofu, or Docker
+commands for you.
+

infra/incident-commander/fixtures/cluster-api-down.txt

@@ -0,0 +1,5 @@
+kubectl get nodes
+
+The connection to the server 192.168.100.73:6443 was refused - did you specify
+the right host or port?
+

infra/incident-commander/fixtures/edge-502.txt

@@ -0,0 +1,7 @@
+Browser shows:
+
+502 Bad Gateway
+nginx/1.31.2
+
+The public website URL was working earlier.
+

infra/incident-commander/fixtures/gitea-backend.txt

@@ -0,0 +1,5 @@
+OpenTofu edge apply failed:
+
+Cannot reach Gitea backend http://100.85.138.30:3000/git/ from the edge host.
+Check the Debian host Tailscale IP, Gitea container, and port publishing.
+

infra/incident-commander/fixtures/rpi-dns-timeout.txt

@@ -0,0 +1,5 @@
+dig @192.168.100.89 cloudflare.com A +time=3 +tries=1 +short
+
+;; communications error to 192.168.100.89#53: timed out
+;; no servers could be reached
+

infra/incident-commander/incidents.tsv

@@ -0,0 +1,6 @@
+id	title	fixture	expected_class	runbook	next_commands	forbidden_actions
+edge_502	Public edge returns 502	edge-502.txt	edge_bad_gateway	docs/runbooks/edge-failures.md	./jeannie doctor-edge;./jeannie doctor-cluster	terraform destroy;./jeannie nuke
+gitea_backend_unreachable	Edge cannot reach Gitea backend	gitea-backend.txt	gitea_edge_backend	docs/runbooks/gitea-failures.md	./jeannie doctor-gitea;./jeannie doctor-edge	docker rm;reset gitea database
+cluster_api_down	Kubernetes API is refused	cluster-api-down.txt	cluster_api_down	docs/runbooks/cluster-stop-start-failures.md	./jeannie start-cluster;./jeannie doctor-cluster	kubeadm reset;./jeannie nuke
+rpi_dns_timeout	Pi-hole DNS timeout	rpi-dns-timeout.txt	dns_failure	docs/runbooks/edge-failures.md	./jeannie doctor-rpi;./jeannie status	docker system prune;disable firewall
+

jeannie

@@ -6326,6 +6326,10 @@ prompt_injection_lab() {
     "${REPO_ROOT}/scripts/prompt-injection-lab" "${@:2}"
 }
 
+incident_commander() {
+    "${REPO_ROOT}/scripts/incident-commander" "${@:2}"
+}
+
 validate_homelab() {
     "${REPO_ROOT}/scripts/validate-homelab"
 }
@@ -6519,6 +6523,7 @@ Operator View And Reports
   capacity-limits [namespace]               Recommend requests/limits YAML snippets.
   recover-plan                               Print disaster recovery order and prerequisites.
   recover-power [--dry-run]                  Run or preview post-outage recovery.
+  incident {list|replay|triage}              Classify incident text and suggest read-only next steps.
   impact [--since REF] [PATH...]             Explain affected lab areas before changes.
   review-last-change [REF]                   Summarize changed files, impact, validation.
   map [--dot]                                Print the homelab dependency map.
@@ -6684,6 +6689,9 @@ case "${1:-}" in
     recover-power)
         recover_power "$@"
         ;;
+    incident)
+        incident_commander "$@"
+        ;;
     scorecard)
         scorecard
         ;;

scripts/explain

@@ -34,7 +34,7 @@ EOF
 
 is_read_only_command() {
     case "$1" in
-        status|capacity|capacity-advisor|capacity-limits|recover-plan|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane)
+        status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|control-plane)
             return 0
             ;;
         *)

scripts/incident-commander

@@ -0,0 +1,208 @@
+#!/usr/bin/env python3
+"""Read-only incident classification and triage for homelab failures."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+INCIDENT_DIR = REPO_ROOT / "infra" / "incident-commander"
+INCIDENTS_FILE = INCIDENT_DIR / "incidents.tsv"
+FIXTURES_DIR = INCIDENT_DIR / "fixtures"
+
+
+@dataclass
+class Incident:
+    incident_id: str
+    title: str
+    fixture: str
+    expected_class: str
+    runbook: str
+    next_commands: list[str]
+    forbidden_actions: list[str]
+
+
+RULES: list[tuple[str, str, str]] = [
+    ("edge_bad_gateway", r"\b502\b|bad gateway|nginx/1\.31", "Public edge proxy cannot get a healthy upstream response."),
+    ("gitea_edge_backend", r"cannot reach gitea backend|100\.85\.138\.30:3000|/git/", "Edge-to-Gitea backend path is broken or Gitea is unhealthy."),
+    ("cluster_api_down", r"6443.*refused|api server.*refused|connection to the server .* was refused", "Kubernetes API is not accepting connections."),
+    ("dns_failure", r"no servers could be reached|communications error .*#53|timed out", "RPi/Pi-hole DNS path is unavailable or blocked."),
+]
+
+
+def split_items(value: str) -> list[str]:
+    return [item.strip() for item in value.split(";") if item.strip()]
+
+
+def load_incidents() -> list[Incident]:
+    with INCIDENTS_FILE.open(encoding="utf-8", newline="") as handle:
+        rows = csv.DictReader(handle, delimiter="\t")
+        return [
+            Incident(
+                incident_id=row["id"],
+                title=row["title"],
+                fixture=row["fixture"],
+                expected_class=row["expected_class"],
+                runbook=row["runbook"],
+                next_commands=split_items(row["next_commands"]),
+                forbidden_actions=split_items(row["forbidden_actions"]),
+            )
+            for row in rows
+        ]
+
+
+def fixture_text(incident: Incident) -> str:
+    path = FIXTURES_DIR / incident.fixture
+    if not path.is_file():
+        raise SystemExit(f"Missing fixture for {incident.incident_id}: {path}")
+    return path.read_text(encoding="utf-8")
+
+
+def classify(text: str) -> tuple[str, str]:
+    lowered = text.lower()
+    for incident_class, pattern, hypothesis in RULES:
+        if re.search(pattern, lowered, re.IGNORECASE | re.DOTALL):
+            return incident_class, hypothesis
+    return "unknown", "No known incident pattern matched. Preserve evidence and run read-only status checks."
+
+
+def incident_for_class(incidents: list[Incident], incident_class: str) -> Incident | None:
+    for incident in incidents:
+        if incident.expected_class == incident_class:
+            return incident
+    return None
+
+
+def render_triage(text: str, incidents: list[Incident]) -> dict[str, object]:
+    incident_class, hypothesis = classify(text)
+    incident = incident_for_class(incidents, incident_class)
+    if incident is None:
+        return {
+            "class": incident_class,
+            "hypothesis": hypothesis,
+            "runbook": "none",
+            "next_commands": ["./jeannie status", "./jeannie scorecard"],
+            "forbidden_actions": ["destructive commands", "secret exposure", "unverified manual fixes"],
+            "evidence": first_evidence_lines(text),
+        }
+    return {
+        "class": incident_class,
+        "hypothesis": hypothesis,
+        "runbook": incident.runbook,
+        "next_commands": incident.next_commands,
+        "forbidden_actions": incident.forbidden_actions,
+        "evidence": first_evidence_lines(text),
+    }
+
+
+def first_evidence_lines(text: str) -> list[str]:
+    lines = [line.strip() for line in text.splitlines() if line.strip()]
+    return lines[:6]
+
+
+def print_triage(result: dict[str, object]) -> None:
+    print("AI Incident Commander")
+    print("=====================")
+    print("mode: read-only")
+    print()
+    print(f"Class: {result['class']}")
+    print(f"Hypothesis: {result['hypothesis']}")
+    print(f"Runbook: {result['runbook']}")
+    print()
+    print("Evidence:")
+    for line in result["evidence"]:
+        print(f"  {line}")
+    print()
+    print("Next read-only commands:")
+    for command in result["next_commands"]:
+        print(f"  {command}")
+    print()
+    print("Blocked until explicitly approved:")
+    for action in result["forbidden_actions"]:
+        print(f"  {action}")
+
+
+def print_list(incidents: list[Incident]) -> int:
+    for incident in incidents:
+        print(f"{incident.incident_id}\t{incident.expected_class}\t{incident.title}")
+    return 0
+
+
+def replay(incidents: list[Incident], incident_id: str | None, as_json: bool, details: bool) -> int:
+    selected = [incident for incident in incidents if incident_id in (None, incident.incident_id)]
+    if not selected:
+        print(f"Unknown incident fixture: {incident_id}", file=sys.stderr)
+        return 2
+
+    results = []
+    for incident in selected:
+        text = fixture_text(incident)
+        result = render_triage(text, incidents)
+        status = "pass" if result["class"] == incident.expected_class else "fail"
+        results.append({"id": incident.incident_id, "status": status, "expected": incident.expected_class, **result})
+
+    if as_json:
+        print(json.dumps(results, indent=2, sort_keys=True))
+    else:
+        failures = sum(1 for result in results if result["status"] != "pass")
+        print("Incident Replay")
+        print("===============")
+        print(f"cases={len(results)} failures={failures}")
+        print()
+        for result in results:
+            print(f"{result['status']:5} {result['id']} - {result['class']}")
+            print(f"      next: {result['next_commands'][0]}")
+            if details:
+                print(f"      runbook: {result['runbook']}")
+                print(f"      hypothesis: {result['hypothesis']}")
+    return 1 if any(result["status"] != "pass" for result in results) else 0
+
+
+def read_triage_input(args: argparse.Namespace) -> str:
+    if args.text:
+        return args.text
+    if args.from_file:
+        return Path(args.from_file).read_text(encoding="utf-8", errors="replace")
+    if not sys.stdin.isatty():
+        return sys.stdin.read()
+    raise SystemExit("Provide --text, --from-file, or stdin for incident triage.")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    subparsers = parser.add_subparsers(dest="command", required=True)
+    subparsers.add_parser("list")
+    replay_parser = subparsers.add_parser("replay")
+    replay_parser.add_argument("incident_id", nargs="?")
+    replay_parser.add_argument("--json", action="store_true")
+    replay_parser.add_argument("--details", action="store_true")
+    triage_parser = subparsers.add_parser("triage")
+    triage_parser.add_argument("--from-file")
+    triage_parser.add_argument("--text")
+    triage_parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    incidents = load_incidents()
+    if args.command == "list":
+        return print_list(incidents)
+    if args.command == "replay":
+        return replay(incidents, args.incident_id, args.json, args.details)
+    if args.command == "triage":
+        result = render_triage(read_triage_input(args), incidents)
+        if args.json:
+            print(json.dumps(result, indent=2, sort_keys=True))
+        else:
+            print_triage(result)
+        return 0
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())