From 0d887acd24f29ba8c95c551473e478bdc603169b Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 13:46:47 -0600 Subject: [PATCH] Add guarded Jeannie self healing --- README.md | 16 +++ README.md.tmpl | 16 +++ docs/jeannie.1.md | 13 ++ infra/agent-sandbox/policy.tsv | 2 +- jeannie | 50 ++++++- scripts/explain | 5 +- scripts/heal | 231 +++++++++++++++++++++++++++++++++ 7 files changed, 330 insertions(+), 3 deletions(-) create mode 100755 scripts/heal diff --git a/README.md b/README.md index a44d33e..ff2dabd 100644 --- a/README.md +++ b/README.md @@ -270,6 +270,22 @@ The current version is intentionally conservative: it validates local gates and prints rollback steps while dev namespace automation remains explicit future work. +## Guarded Self-Healing + +Jeannie can turn status findings into an experimental self-heal plan: + +```bash +./jeannie status --heal-plan +./jeannie status --heal +./jeannie heal apply --yes +``` + +Normal `status` remains read-only. The heal planner maps known findings to +guarded actions, checks proposed commands against `infra/agent-sandbox/policy.tsv`, +and only auto-runs low-risk actions such as `./jeannie start-cluster`. +Medium-risk repairs like redeploying Gitea or RPi DNS are listed for review +instead of being run automatically. Destructive actions remain blocked. + ## Model Behavior Observatory Stable prompts track local model behavior, latency, refusal behavior, and diff --git a/README.md.tmpl b/README.md.tmpl index fffe576..a719b05 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -270,6 +270,22 @@ The current version is intentionally conservative: it validates local gates and prints rollback steps while dev namespace automation remains explicit future work. +## Guarded Self-Healing + +Jeannie can turn status findings into an experimental self-heal plan: + +```bash +./{{ main_script }} status --heal-plan +./{{ main_script }} status --heal +./{{ main_script }} heal apply --yes +``` + +Normal `status` remains read-only. The heal planner maps known findings to +guarded actions, checks proposed commands against `infra/agent-sandbox/policy.tsv`, +and only auto-runs low-risk actions such as `./{{ main_script }} start-cluster`. +Medium-risk repairs like redeploying Gitea or RPi DNS are listed for review +instead of being run automatically. Destructive actions remain blocked. + ## Model Behavior Observatory Stable prompts track local model behavior, latency, refusal behavior, and diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 32913ae..34f81f3 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -63,6 +63,19 @@ for the older detailed tables. The cascade includes "what broke" signals for recent deployment readiness, pod restarts, node pressure, disk pressure, Traefik 5xx/404 log evidence, and recent Gitea errors. +`status --heal-plan` +: Build a guarded self-heal plan from current status findings. This is +read-only and may include cited RAG context when the local index is available. + +`status --heal` +: Build the same plan and dry-run auto-eligible actions. Use `heal apply --yes` +to actually run low-risk actions. + +`heal {plan|apply}` +: Plan or apply guarded self-healing actions. `apply` only executes actions +marked auto-eligible and allowed by `infra/agent-sandbox/policy.tsv`; destructive +actions remain blocked. + `capacity` : Print a compact read-only placement report covering Debian, Kubernetes, Pimox, RPi, and placement signals. Use `capacity --details`, `--json`, `--only diff --git a/infra/agent-sandbox/policy.tsv b/infra/agent-sandbox/policy.tsv index 2bc74bf..967eb42 100644 --- a/infra/agent-sandbox/policy.tsv +++ b/infra/agent-sandbox/policy.tsv @@ -7,6 +7,7 @@ pattern decision reason ./jeannie incident allow Read-only incident triage/replay. ./jeannie ai-evals allow Read-only eval harness. ./jeannie prompt-injection-lab allow Read-only security lab harness. +./jeannie start-cluster allow Idempotent recovery path that starts saved Kubernetes services and worker VMs without destroying state. ./jeannie up approval High-blast-radius deployment. ./jeannie apps approval Application deployment can mutate cluster state. ./jeannie rpi-services approval Mutates RPi Docker services. @@ -16,4 +17,3 @@ terraform destroy deny Destructive infrastructure path. rm -rf deny Destructive file deletion pattern. kubectl delete approval Kubernetes deletion needs explicit scope and approval. docker rm approval Docker deletion needs explicit scope and approval. - diff --git a/jeannie b/jeannie index ce7eb23..d20915a 100755 --- a/jeannie +++ b/jeannie @@ -5519,13 +5519,50 @@ fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" status_report() { local cascade_status=0 + local heal_mode="" + local status_json + local -a status_args=() require_debian_server "status" - if ! report_ui_parse_args "$@"; then + while (($# > 0)); do + case "$1" in + --heal-plan) + heal_mode="plan" + shift + ;; + --heal) + heal_mode="apply" + shift + ;; + *) + status_args+=("$1") + shift + ;; + esac + done + + if ! report_ui_parse_args "${status_args[@]}"; then return 1 fi + if [[ -n "${heal_mode}" ]]; then + status_json="$(mktemp "${TMPDIR:-/tmp}/jeannie-status-json.XXXXXX")" + REPORT_UI_JSON=true + REPORT_UI_ONLY=problems + report_ui_begin + JEANNIE_REPORT_MODE=true status_cascade_report >/dev/null || cascade_status=$? + report_ui_render "Jeannie Status" >"${status_json}" + report_ui_cleanup + if [[ "${heal_mode}" == "plan" ]]; then + "${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}" --ai + else + "${REPO_ROOT}/scripts/heal" apply --status-json "${status_json}" --ai + fi + rm -f "${status_json}" + return "${cascade_status}" + fi + if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then report_ui_begin JEANNIE_REPORT_MODE=true status_cascade_report || cascade_status=$? @@ -6358,6 +6395,11 @@ model_observe() { "${REPO_ROOT}/scripts/model-observe" "${@:2}" } +heal() { + require_debian_server "heal" + "${REPO_ROOT}/scripts/heal" "${@:2}" +} + validate_homelab() { "${REPO_ROOT}/scripts/validate-homelab" } @@ -6545,12 +6587,15 @@ Cluster Lifecycle Operator View And Reports status [--all|--details|--verbose|--json] Problems-first cascade from host to public URLs. + status --heal-plan Build a guarded self-heal plan from status. + status --heal Dry-run auto-eligible self-heal actions. scorecard Compact pass/warn/fail operator scorecard. capacity Compact capacity and placement report. capacity-advisor Decide whether to add VMs or fix placement/resources. capacity-limits [namespace] Recommend requests/limits YAML snippets. recover-plan Print disaster recovery order and prerequisites. recover-power [--dry-run] Run or preview post-outage recovery. + heal {plan|apply} Plan/apply guarded self-healing actions. incident {list|replay|triage} Classify incident text and suggest read-only next steps. safety-case [--since REF] [PATH...] Generate risk, blast-radius, test, and rollback case. agent-sandbox {check|policy} Classify proposed commands against agent policy. @@ -6727,6 +6772,9 @@ case "${1:-}" in recover-power) recover_power "$@" ;; + heal) + heal "$@" + ;; incident) incident_commander "$@" ;; diff --git a/scripts/explain b/scripts/explain index e7460c9..dd08524 100755 --- a/scripts/explain +++ b/scripts/explain @@ -34,7 +34,7 @@ EOF is_read_only_command() { case "$1" in - status|capacity|capacity-advisor|capacity-limits|recover-plan|incident|safety-case|agent-sandbox|scorecard|gitops-status|promote|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|model-observe|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane) + status|capacity|capacity-advisor|capacity-limits|recover-plan|heal|incident|safety-case|agent-sandbox|scorecard|gitops-status|promote|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|ask|ai-evals|ai-scheduler|ai-memory|model-observe|impact|review-last-change|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|prompt-injection-lab|red-blue|control-plane) return 0 ;; *) @@ -57,6 +57,9 @@ is_read_only_subcommand() { change-journal) [ -z "$subcommand" ] || [ "$subcommand" = "list" ] || [ "$subcommand" = "path" ] ;; + heal) + [ -z "$subcommand" ] || [ "$subcommand" = "plan" ] + ;; control-plane) [ -z "$subcommand" ] || [ "$subcommand" = "status" ] || [ "$subcommand" = "pods" ] ;; diff --git a/scripts/heal b/scripts/heal new file mode 100755 index 0000000..62bd4a5 --- /dev/null +++ b/scripts/heal @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Plan and apply guarded self-healing actions from Jeannie status findings.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@dataclass(frozen=True) +class HealRule: + check: str + command: str + risk: str + reason: str + auto: bool + + +RULES = [ + HealRule("Pimox workers running", "./jeannie start-cluster", "low", "Worker VMs are expected cluster capacity and start-cluster is idempotent.", True), + HealRule("Kubernetes API", "./jeannie start-cluster", "low", "API is down; start-cluster starts kubelet/containerd and worker VMs without destroying state.", True), + HealRule("Kubernetes nodes Ready", "./jeannie start-cluster", "low", "NotReady worker nodes commonly recover by starting the saved cluster runtime.", True), + HealRule("Gitea container", "./jeannie deploy-gitea", "medium", "Reapplies the Debian-hosted Gitea Compose service.", False), + HealRule("Gitea local HTTP", "./jeannie deploy-gitea", "medium", "Local Gitea HTTP is down; redeploying Compose is usually safe but still mutates Git service runtime.", False), + HealRule("Pi-hole DNS", "./jeannie rpi-services", "medium", "Reapplies RPi DNS services; safe for the lab but can briefly disrupt DNS.", False), + HealRule("Uptime Kuma HTTP", "./jeannie rpi-services", "medium", "Reapplies RPi service Compose stack.", False), + HealRule("RPi Docker root state", "./jeannie rpi-services", "medium", "May repair service runtime but should be reviewed if storage is failing.", False), + HealRule("Traefik deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False), + HealRule("Website deployment", "./jeannie promote validate", "diagnostic", "Validate gates first; deployment rollout failures are usually downstream of cluster/node health.", False), + HealRule("Traefik LoadBalancer HTTP", "./jeannie doctor-edge", "diagnostic", "Edge/LB failures need diagnosis after cluster nodes are healthy.", False), + HealRule("Website public URL", "./jeannie doctor-edge", "diagnostic", "Public URL failures need edge and cluster diagnosis after local services recover.", False), + HealRule("Gitea public route", "./jeannie doctor-gitea", "diagnostic", "Public Gitea route needs Gitea and edge diagnosis.", False), + HealRule("No problem pods", "./jeannie explain status", "diagnostic", "Pod states need root-cause context; avoid blind deletes.", False), + HealRule("Recent deployments healthy", "./jeannie explain status", "diagnostic", "Deployment drift can be a symptom of node health, image pulls, or scheduling.", False), + HealRule("Pod restart pressure", "./jeannie explain status", "diagnostic", "Restart pressure needs workload-specific diagnosis before mutation.", False), + HealRule("Traefik 5xx/404 signals", "./jeannie doctor-edge", "diagnostic", "Use edge logs and route checks before changing config.", False), +] + + +def load_status(path: Path | None) -> dict[str, object]: + if path: + return json.loads(path.read_text(encoding="utf-8")) + + process = subprocess.run( + [str(REPO_ROOT / "jeannie"), "status", "--json"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if not process.stdout.strip(): + print(process.stderr.strip() or "status produced no JSON", file=sys.stderr) + raise SystemExit(1) + return json.loads(process.stdout) + + +def status_rows(status: dict[str, object]) -> list[dict[str, str]]: + rows = status.get("rows", []) + if not isinstance(rows, list): + return [] + return [row for row in rows if isinstance(row, dict) and row.get("status") in {"fail", "warn"}] + + +def rule_for(row: dict[str, str]) -> HealRule | None: + check = str(row.get("check", "")) + for rule in RULES: + if rule.check == check: + return rule + return None + + +def build_plan(status: dict[str, object]) -> list[dict[str, object]]: + planned: list[dict[str, object]] = [] + seen_commands: set[str] = set() + for row in status_rows(status): + rule = rule_for(row) + if rule is None: + planned.append( + { + "check": row.get("check", ""), + "area": row.get("area", ""), + "status": row.get("status", ""), + "command": "./jeannie explain status", + "risk": "diagnostic", + "auto": False, + "reason": "No specific heal rule exists yet.", + "summary": row.get("summary", ""), + } + ) + continue + if rule.command in seen_commands and rule.auto: + continue + seen_commands.add(rule.command) + planned.append( + { + "check": row.get("check", ""), + "area": row.get("area", ""), + "status": row.get("status", ""), + "command": rule.command, + "risk": rule.risk, + "auto": rule.auto, + "reason": rule.reason, + "summary": row.get("summary", ""), + } + ) + return planned + + +def print_plan(plan: list[dict[str, object]], ai: bool) -> None: + auto = [item for item in plan if item["auto"]] + review = [item for item in plan if not item["auto"]] + print("Jeannie Heal Plan") + print("=================") + print("mode: guarded") + print(f"auto_actions={len(auto)} review_actions={len(review)}") + print() + if auto: + print("Auto-eligible:") + for item in auto: + print(f" - {item['command']}") + print(f" fixes: {item['area']} / {item['check']}") + print(f" why: {item['reason']}") + else: + print("Auto-eligible: none") + print() + if review: + print("Needs review or diagnostic only:") + for item in review: + print(f" - {item['command']}") + print(f" finding: {item['area']} / {item['check']} ({item['status']})") + print(f" why: {item['reason']}") + if ai: + print_ai_context(plan) + + +def print_ai_context(plan: list[dict[str, object]]) -> None: + query = " ".join(str(item["check"]) for item in plan[:5]) + if not query: + return + process = subprocess.run( + [str(REPO_ROOT / "scripts" / "query-homelab-ai-index"), "--citations", "--limit", "3", query], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + if process.returncode == 0 and process.stdout.strip(): + print() + print("AI/RAG Context") + print("--------------") + print(process.stdout.strip()) + + +def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: + commands: list[str] = [] + for item in plan: + if item["auto"]: + command = str(item["command"]) + if command not in commands: + commands.append(command) + if not commands: + print("No auto-eligible heal actions.") + return 0 + + print("Jeannie Heal Apply") + print("==================") + for command in commands: + sandbox = subprocess.run( + [str(REPO_ROOT / "scripts" / "agent-sandbox"), "check", command, "--json"], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + try: + decision = json.loads(sandbox.stdout).get("decision") + except json.JSONDecodeError: + decision = "approval" + if decision != "allow": + print(f"skip: {command} blocked by agent sandbox decision={decision}") + continue + if not yes: + print(f"would run: {command}") + continue + print(f"run: {command}") + process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False) + if process.returncode != 0: + return process.returncode + if not yes: + print() + print("Dry run only. Re-run with: ./jeannie heal apply --yes") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + plan_parser = subparsers.add_parser("plan") + plan_parser.add_argument("--status-json", type=Path) + plan_parser.add_argument("--ai", action="store_true") + apply_parser = subparsers.add_parser("apply") + apply_parser.add_argument("--status-json", type=Path) + apply_parser.add_argument("--yes", action="store_true") + apply_parser.add_argument("--ai", action="store_true") + args = parser.parse_args() + + status = load_status(getattr(args, "status_json", None)) + plan = build_plan(status) + if args.command == "plan": + print_plan(plan, args.ai) + return 0 + if args.command == "apply": + print_plan(plan, args.ai) + print() + return apply_plan(plan, args.yes) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())