Cooldown failed heal actions

README.md

@@ -288,6 +288,8 @@ can reassess before touching the next issue. If a specific Pimox worker is
 NotReady, heal targets that worker with `./jeannie workers restart N`
 instead of repeating a broad cluster start. After an action runs, Jeannie waits
 for the target condition to clear and reports `healed` or `still failing`.
+Failed heal actions are placed on a short cooldown so Jeannie does not loop on
+the same VM restart; the next plan escalates to diagnostics instead.
 Destructive actions remain blocked.
 
 ## Model Behavior Observatory

README.md.tmpl

@@ -288,6 +288,8 @@ can reassess before touching the next issue. If a specific Pimox worker is
 NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
 instead of repeating a broad cluster start. After an action runs, Jeannie waits
 for the target condition to clear and reports `healed` or `still failing`.
+Failed heal actions are placed on a short cooldown so Jeannie does not loop on
+the same VM restart; the next plan escalates to diagnostics instead.
 Destructive actions remain blocked.
 
 ## Model Behavior Observatory

docs/jeannie.1.md

@@ -76,7 +76,10 @@ read-only and may include cited RAG context when the local index is available.
 highest-impact auto-eligible action allowed by
 `infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
 `status` before the next heal cycle. After an action runs, Jeannie waits for the
-target condition to clear and reports `healed` or `still failing`.
+target condition to clear and reports `healed` or `still failing`. Failed
+actions are recorded in the homelab state directory and placed on a short
+cooldown; during cooldown the next plan escalates to diagnostics instead of
+repeating the same restart.
 
 `capacity`
 : Print a compact read-only placement report covering Debian, Kubernetes,

scripts/heal

@@ -16,6 +16,9 @@ from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config")))
+STATE_DIR = Path(os.environ.get("HOMELAB_STATE_DIR", Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local/share")) / "homelab"))
+HEAL_STATE_FILE = Path(os.environ.get("LAB_HEAL_STATE_FILE", STATE_DIR / "heal-state.json"))
+HEAL_COOLDOWN_SECONDS = int(os.environ.get("LAB_HEAL_COOLDOWN_SECONDS", "1800"))
 
 
 @dataclass(frozen=True)
@@ -105,6 +108,7 @@ def command_for(row: dict[str, str], rule: HealRule) -> tuple[str, str, int]:
 def build_plan(status: dict[str, object]) -> list[dict[str, object]]:
     planned: list[dict[str, object]] = []
     seen_commands: set[str] = set()
+    blocked = recent_failed_commands()
     for row in status_rows(status):
         rule = rule_for(row)
         if rule is None:
@@ -123,7 +127,14 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]:
             )
             continue
         command, reason, priority = command_for(row, rule)
-        if command in seen_commands and rule.auto:
+        auto = rule.auto
+        if command in blocked:
+            failed_finding = blocked[command]
+            auto = False
+            command = "./jeannie doctor-cluster --details"
+            reason = f"Previous heal action did not clear this finding recently; escalate to diagnostics before retrying. Last failure: {failed_finding}"
+            priority = max(priority - 20, 1)
+        if command in seen_commands and auto:
             continue
         seen_commands.add(command)
         planned.append(
@@ -133,7 +144,7 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]:
                 "status": row.get("status", ""),
                 "command": command,
                 "risk": rule.risk,
-                "auto": rule.auto,
+                "auto": auto,
                 "priority": priority,
                 "reason": reason,
                 "summary": row.get("summary", ""),
@@ -234,12 +245,15 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
         print(f"run: {command}")
         process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
         if process.returncode != 0:
+            record_failed_action(command, target)
             return process.returncode
         if verify_healed(target):
             print(f"healed: {target['area']} / {target['check']}")
+            clear_failed_action(command)
         else:
             print(f"still failing: {target['area']} / {target['check']}")
             print("Run ./jeannie status --details for the current evidence.")
+            record_failed_action(command, target)
             return 1
     if not yes:
         print()
@@ -250,6 +264,69 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
     return 0
 
 
+def load_heal_state() -> dict[str, object]:
+    if not HEAL_STATE_FILE.is_file():
+        return {"failed_actions": {}}
+    try:
+        document = json.loads(HEAL_STATE_FILE.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {"failed_actions": {}}
+    if not isinstance(document, dict):
+        return {"failed_actions": {}}
+    if not isinstance(document.get("failed_actions"), dict):
+        document["failed_actions"] = {}
+    return document
+
+
+def write_heal_state(document: dict[str, object]) -> None:
+    HEAL_STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    HEAL_STATE_FILE.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+
+def recent_failed_commands() -> dict[str, str]:
+    now = int(time.time())
+    document = load_heal_state()
+    failed = document.get("failed_actions", {})
+    blocked: dict[str, str] = {}
+    changed = False
+    for command, info in list(failed.items()):
+        if not isinstance(info, dict):
+            failed.pop(command, None)
+            changed = True
+            continue
+        failed_at = int(info.get("failed_at", 0) or 0)
+        if now - failed_at > HEAL_COOLDOWN_SECONDS:
+            failed.pop(command, None)
+            changed = True
+            continue
+        blocked[command] = str(info.get("finding") or "unknown")
+    if changed:
+        document["failed_actions"] = failed
+        write_heal_state(document)
+    return blocked
+
+
+def record_failed_action(command: str, target: dict[str, object]) -> None:
+    document = load_heal_state()
+    failed = document.setdefault("failed_actions", {})
+    if isinstance(failed, dict):
+        failed[command] = {
+            "failed_at": int(time.time()),
+            "finding": f"{target.get('area')} / {target.get('check')}",
+            "summary": target.get("summary", ""),
+        }
+    write_heal_state(document)
+    print(f"recorded failed heal action cooldown: {command}")
+
+
+def clear_failed_action(command: str) -> None:
+    document = load_heal_state()
+    failed = document.get("failed_actions", {})
+    if isinstance(failed, dict) and command in failed:
+        failed.pop(command, None)
+        write_heal_state(document)
+
+
 def verify_healed(target: dict[str, object]) -> bool:
     timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180"))
     interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10"))

tests/jeannie-unit

@@ -269,6 +269,57 @@ PY
     assert_command_failure_contains "inventory validator rejects invalid Debian LAN IP" "hosts.debian.lan_ip is not a valid IP" "${REPO_ROOT}/scripts/validate-homelab-inventory" "${inventory}"
 }
 
+test_heal_cooldown_escalates_to_diagnostics() {
+    local status_json="${TMP_DIR}/heal-status.json"
+    local state_json="${TMP_DIR}/heal-state.json"
+    local output
+
+    cat >"${status_json}" <<'EOF'
+{
+  "rows": [
+    {
+      "area": "Pimox And Cluster",
+      "check": "Kubernetes nodes Ready",
+      "status": "fail",
+      "summary": "pimox-worker-01 NotReady",
+      "detail": "pimox-worker-01 NotReady"
+    },
+    {
+      "area": "Platform And Apps",
+      "check": "Traefik deployment",
+      "status": "fail",
+      "summary": "deployment unavailable"
+    }
+  ]
+}
+EOF
+    python3 - "${state_json}" <<'PY'
+import json
+import sys
+import time
+
+with open(sys.argv[1], "w", encoding="utf-8") as handle:
+    json.dump(
+        {
+            "failed_actions": {
+                "./jeannie workers restart 1": {
+                    "failed_at": int(time.time()),
+                    "finding": "Pimox And Cluster / Kubernetes nodes Ready",
+                }
+            }
+        },
+        handle,
+    )
+PY
+
+    output="$(
+        LAB_HEAL_STATE_FILE="${state_json}" LAB_HEAL_COOLDOWN_SECONDS=3600 \
+            "${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}"
+    )"
+    assert_contains "heal cooldown removes auto target" "Next heal target: none" "${output}"
+    assert_contains "heal cooldown escalates diagnostics" "next diagnostic command: ./jeannie doctor-cluster --details" "${output}"
+}
+
 test_pimox_worker_count_detects_generated_topology
 test_configured_worker_count_cannot_hide_existing_workers
 test_configured_worker_count_can_expand_target_set
@@ -280,6 +331,7 @@ test_report_renderer_problems_only_hides_ok_rows
 test_report_renderer_empty_problems_message
 test_inventory_validator_accepts_current_inventory
 test_inventory_validator_rejects_ip_drift
+test_heal_cooldown_escalates_to_diagnostics
 
 if ((FAILURES > 0)); then
     printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2