From 91df63af969d702a340629ab7075c72f5118a448 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 13:50:00 -0600 Subject: [PATCH] Heal one highest impact finding at a time --- README.md | 8 +- README.md.tmpl | 8 +- docs/jeannie.1.md | 11 +-- infra/agent-sandbox/policy.tsv | 2 + scripts/heal | 139 ++++++++++++++++++--------------- 5 files changed, 94 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index ff2dabd..698621f 100644 --- a/README.md +++ b/README.md @@ -281,10 +281,10 @@ Jeannie can turn status findings into an experimental self-heal plan: ``` Normal `status` remains read-only. The heal planner maps known findings to -guarded actions, checks proposed commands against `infra/agent-sandbox/policy.tsv`, -and only auto-runs low-risk actions such as `./jeannie start-cluster`. -Medium-risk repairs like redeploying Gitea or RPi DNS are listed for review -instead of being run automatically. Destructive actions remain blocked. +guarded actions, chooses one highest-impact auto-eligible finding, checks the +proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that +one action. Re-run `./jeannie status` after every heal cycle so Jeannie +can reassess before touching the next issue. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/README.md.tmpl b/README.md.tmpl index a719b05..4c6270f 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -281,10 +281,10 @@ Jeannie can turn status findings into an experimental self-heal plan: ``` Normal `status` remains read-only. The heal planner maps known findings to -guarded actions, checks proposed commands against `infra/agent-sandbox/policy.tsv`, -and only auto-runs low-risk actions such as `./{{ main_script }} start-cluster`. -Medium-risk repairs like redeploying Gitea or RPi DNS are listed for review -instead of being run automatically. Destructive actions remain blocked. +guarded actions, chooses one highest-impact auto-eligible finding, checks the +proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that +one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie +can reassess before touching the next issue. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 34f81f3..3ebb286 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -68,13 +68,14 @@ Traefik 5xx/404 log evidence, and recent Gitea errors. read-only and may include cited RAG context when the local index is available. `status --heal` -: Build the same plan and dry-run auto-eligible actions. Use `heal apply --yes` -to actually run low-risk actions. +: Build the same plan and dry-run the single highest-impact auto-eligible + action. Use `heal apply --yes` to actually run it. `heal {plan|apply}` -: Plan or apply guarded self-healing actions. `apply` only executes actions -marked auto-eligible and allowed by `infra/agent-sandbox/policy.tsv`; destructive -actions remain blocked. +: Plan or apply guarded self-healing actions. `apply` executes at most one +highest-impact auto-eligible action allowed by +`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run +`status` before the next heal cycle. `capacity` : Print a compact read-only placement report covering Debian, Kubernetes, diff --git a/infra/agent-sandbox/policy.tsv b/infra/agent-sandbox/policy.tsv index 967eb42..8e7c6d6 100644 --- a/infra/agent-sandbox/policy.tsv +++ b/infra/agent-sandbox/policy.tsv @@ -8,6 +8,8 @@ pattern decision reason ./jeannie ai-evals allow Read-only eval harness. ./jeannie prompt-injection-lab allow Read-only security lab harness. ./jeannie start-cluster allow Idempotent recovery path that starts saved Kubernetes services and worker VMs without destroying state. +./jeannie deploy-gitea allow Idempotent Debian-hosted Gitea Compose repair path for this homelab. +./jeannie rpi-services allow Idempotent RPi Pi-hole/Unbound/Uptime Kuma Compose repair path for this homelab. ./jeannie up approval High-blast-radius deployment. ./jeannie apps approval Application deployment can mutate cluster state. ./jeannie rpi-services approval Mutates RPi Docker services. diff --git a/scripts/heal b/scripts/heal index 62bd4a5..59399eb 100755 --- a/scripts/heal +++ b/scripts/heal @@ -22,26 +22,27 @@ class HealRule: risk: str reason: str auto: bool + priority: int RULES = [ - HealRule("Pimox workers running", "./jeannie start-cluster", "low", "Worker VMs are expected cluster capacity and start-cluster is idempotent.", True), - HealRule("Kubernetes API", "./jeannie start-cluster", "low", "API is down; start-cluster starts kubelet/containerd and worker VMs without destroying state.", True), - HealRule("Kubernetes nodes Ready", "./jeannie start-cluster", "low", "NotReady worker nodes commonly recover by starting the saved cluster runtime.", True), - HealRule("Gitea container", "./jeannie deploy-gitea", "medium", "Reapplies the Debian-hosted Gitea Compose service.", False), - HealRule("Gitea local HTTP", "./jeannie deploy-gitea", "medium", "Local Gitea HTTP is down; redeploying Compose is usually safe but still mutates Git service runtime.", False), - HealRule("Pi-hole DNS", "./jeannie rpi-services", "medium", "Reapplies RPi DNS services; safe for the lab but can briefly disrupt DNS.", False), - HealRule("Uptime Kuma HTTP", "./jeannie rpi-services", "medium", "Reapplies RPi service Compose stack.", False), - HealRule("RPi Docker root state", "./jeannie rpi-services", "medium", "May repair service runtime but should be reviewed if storage is failing.", False), - HealRule("Traefik deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False), - HealRule("Website deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False), - HealRule("Traefik LoadBalancer HTTP", "./jeannie doctor-edge", "diagnostic", "Edge/LB failures need diagnosis after cluster nodes are healthy.", False), - HealRule("Website public URL", "./jeannie doctor-edge", "diagnostic", "Public URL failures need edge and cluster diagnosis after local services recover.", False), - HealRule("Gitea public route", "./jeannie doctor-gitea", "diagnostic", "Public Gitea route needs Gitea and edge diagnosis.", False), - HealRule("No problem pods", "./jeannie explain status", "diagnostic", "Pod states need root-cause context; avoid blind deletes.", False), - HealRule("Recent deployments healthy", "./jeannie explain status", "diagnostic", "Deployment drift can be a symptom of node health, image pulls, or scheduling.", False), - HealRule("Pod restart pressure", "./jeannie explain status", "diagnostic", "Restart pressure needs workload-specific diagnosis before mutation.", False), - HealRule("Traefik 5xx/404 signals", "./jeannie doctor-edge", "diagnostic", "Use edge logs and route checks before changing config.", False), + HealRule("Kubernetes API", "./jeannie start-cluster", "low", "API is down; start-cluster starts kubelet/containerd and worker VMs without destroying state.", True, 100), + HealRule("Pimox workers running", "./jeannie start-cluster", "low", "Worker VMs are expected cluster capacity and start-cluster is idempotent.", True, 95), + HealRule("Kubernetes nodes Ready", "./jeannie start-cluster", "low", "NotReady worker nodes commonly recover by starting the saved cluster runtime.", True, 90), + HealRule("Pi-hole DNS", "./jeannie rpi-services", "medium", "Reapplies RPi DNS services; safe for the lab but can briefly disrupt DNS.", True, 80), + HealRule("RPi Docker root state", "./jeannie rpi-services", "medium", "May repair service runtime but should be reviewed if storage is failing.", True, 78), + HealRule("Uptime Kuma HTTP", "./jeannie rpi-services", "medium", "Reapplies RPi service Compose stack.", True, 70), + HealRule("Gitea container", "./jeannie deploy-gitea", "medium", "Reapplies the Debian-hosted Gitea Compose service.", True, 75), + HealRule("Gitea local HTTP", "./jeannie deploy-gitea", "medium", "Local Gitea HTTP is down; redeploying Compose is usually safe but still mutates Git service runtime.", True, 74), + HealRule("Traefik deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False, 65), + HealRule("Website deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False, 64), + HealRule("Traefik LoadBalancer HTTP", "./jeannie doctor-edge", "diagnostic", "Edge/LB failures need diagnosis after cluster nodes are healthy.", False, 60), + HealRule("Website public URL", "./jeannie doctor-edge", "diagnostic", "Public URL failures need edge and cluster diagnosis after local services recover.", False, 58), + HealRule("Gitea public route", "./jeannie doctor-gitea", "diagnostic", "Public Gitea route needs Gitea and edge diagnosis.", False, 55), + HealRule("No problem pods", "./jeannie explain status", "diagnostic", "Pod states need root-cause context; avoid blind deletes.", False, 50), + HealRule("Recent deployments healthy", "./jeannie explain status", "diagnostic", "Deployment drift can be a symptom of node health, image pulls, or scheduling.", False, 45), + HealRule("Pod restart pressure", "./jeannie explain status", "diagnostic", "Restart pressure needs workload-specific diagnosis before mutation.", False, 40), + HealRule("Traefik 5xx/404 signals", "./jeannie doctor-edge", "diagnostic", "Use edge logs and route checks before changing config.", False, 35), ] @@ -92,6 +93,7 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: "command": "./jeannie explain status", "risk": "diagnostic", "auto": False, + "priority": 1, "reason": "No specific heal rule exists yet.", "summary": row.get("summary", ""), } @@ -108,6 +110,7 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: "command": rule.command, "risk": rule.risk, "auto": rule.auto, + "priority": rule.priority, "reason": rule.reason, "summary": row.get("summary", ""), } @@ -115,31 +118,46 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: return planned +def impact_sort_key(item: dict[str, object]) -> tuple[int, int, str]: + status_score = 10 if item.get("status") == "fail" else 0 + return (int(item.get("priority") or 0) + status_score, 1 if item.get("auto") else 0, str(item.get("check") or "")) + + +def highest_impact(plan: list[dict[str, object]], *, auto_only: bool) -> dict[str, object] | None: + candidates = [item for item in plan if item.get("auto")] if auto_only else list(plan) + if not candidates: + return None + return sorted(candidates, key=impact_sort_key, reverse=True)[0] + + def print_plan(plan: list[dict[str, object]], ai: bool) -> None: - auto = [item for item in plan if item["auto"]] - review = [item for item in plan if not item["auto"]] + target = highest_impact(plan, auto_only=True) + top_overall = highest_impact(plan, auto_only=False) + deferred = [item for item in sorted(plan, key=impact_sort_key, reverse=True) if item is not target] print("Jeannie Heal Plan") print("=================") - print("mode: guarded") - print(f"auto_actions={len(auto)} review_actions={len(review)}") + print("mode: one-at-a-time") + print(f"findings={len(plan)}") print() - if auto: - print("Auto-eligible:") - for item in auto: - print(f" - {item['command']}") - print(f" fixes: {item['area']} / {item['check']}") - print(f" why: {item['reason']}") + if target: + print("Next heal target:") + print(f" command: {target['command']}") + print(f" finding: {target['area']} / {target['check']} ({target['status']})") + print(f" impact: {target['priority']}") + print(f" why: {target['reason']}") else: - print("Auto-eligible: none") + print("Next heal target: none") + if top_overall: + print(f"highest finding is diagnostic-only: {top_overall['area']} / {top_overall['check']}") + print(f"next diagnostic command: {top_overall['command']}") print() - if review: - print("Needs review or diagnostic only:") - for item in review: - print(f" - {item['command']}") - print(f" finding: {item['area']} / {item['check']} ({item['status']})") - print(f" why: {item['reason']}") + if deferred: + print("Deferred until next status/heal cycle:") + for item in deferred[:8]: + auto_marker = "auto" if item.get("auto") else "diagnostic" + print(f" - {item['area']} / {item['check']} ({auto_marker}, impact={item['priority']})") if ai: - print_ai_context(plan) + print_ai_context([target] if target else ([top_overall] if top_overall else [])) def print_ai_context(plan: list[dict[str, object]]) -> None: @@ -162,37 +180,33 @@ def print_ai_context(plan: list[dict[str, object]]) -> None: def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: - commands: list[str] = [] - for item in plan: - if item["auto"]: - command = str(item["command"]) - if command not in commands: - commands.append(command) - if not commands: + target = highest_impact(plan, auto_only=True) + if not target: print("No auto-eligible heal actions.") return 0 print("Jeannie Heal Apply") print("==================") - for command in commands: - sandbox = subprocess.run( - [str(REPO_ROOT / "scripts" / "agent-sandbox"), "check", command, "--json"], - cwd=REPO_ROOT, - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - check=False, - ) - try: - decision = json.loads(sandbox.stdout).get("decision") - except json.JSONDecodeError: - decision = "approval" - if decision != "allow": - print(f"skip: {command} blocked by agent sandbox decision={decision}") - continue - if not yes: - print(f"would run: {command}") - continue + command = str(target["command"]) + sandbox = subprocess.run( + [str(REPO_ROOT / "scripts" / "agent-sandbox"), "check", command, "--json"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + try: + decision = json.loads(sandbox.stdout).get("decision") + except json.JSONDecodeError: + decision = "approval" + if decision != "allow": + print(f"skip: {command} blocked by agent sandbox decision={decision}") + return 1 + print(f"target: {target['area']} / {target['check']}") + if not yes: + print(f"would run: {command}") + else: print(f"run: {command}") process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False) if process.returncode != 0: @@ -200,6 +214,9 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: if not yes: print() print("Dry run only. Re-run with: ./jeannie heal apply --yes") + print("After it runs, use ./jeannie status again before healing the next finding.") + else: + print("Run ./jeannie status again before healing the next finding.") return 0