From 8a6ae40ab4c30dc60e623a026f56d79e56410212 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 30 Jun 2026 16:15:06 -0600 Subject: [PATCH] Cooldown failed heal actions --- README.md | 2 ++ README.md.tmpl | 2 ++ docs/jeannie.1.md | 5 ++- scripts/heal | 81 ++++++++++++++++++++++++++++++++++++++++++++-- tests/jeannie-unit | 52 +++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6e2b240..f42c0b7 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,8 @@ can reassess before touching the next issue. If a specific Pimox worker is NotReady, heal targets that worker with `./jeannie workers restart N` instead of repeating a broad cluster start. After an action runs, Jeannie waits for the target condition to clear and reports `healed` or `still failing`. +Failed heal actions are placed on a short cooldown so Jeannie does not loop on +the same VM restart; the next plan escalates to diagnostics instead. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/README.md.tmpl b/README.md.tmpl index 78161f2..9de04e8 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -288,6 +288,8 @@ can reassess before touching the next issue. If a specific Pimox worker is NotReady, heal targets that worker with `./{{ main_script }} workers restart N` instead of repeating a broad cluster start. After an action runs, Jeannie waits for the target condition to clear and reports `healed` or `still failing`. +Failed heal actions are placed on a short cooldown so Jeannie does not loop on +the same VM restart; the next plan escalates to diagnostics instead. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 25ec933..3fbda33 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -76,7 +76,10 @@ read-only and may include cited RAG context when the local index is available. highest-impact auto-eligible action allowed by `infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run `status` before the next heal cycle. After an action runs, Jeannie waits for the -target condition to clear and reports `healed` or `still failing`. +target condition to clear and reports `healed` or `still failing`. Failed +actions are recorded in the homelab state directory and placed on a short +cooldown; during cooldown the next plan escalates to diagnostics instead of +repeating the same restart. `capacity` : Print a compact read-only placement report covering Debian, Kubernetes, diff --git a/scripts/heal b/scripts/heal index 59885ee..764d1ff 100755 --- a/scripts/heal +++ b/scripts/heal @@ -16,6 +16,9 @@ from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config"))) +STATE_DIR = Path(os.environ.get("HOMELAB_STATE_DIR", Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local/share")) / "homelab")) +HEAL_STATE_FILE = Path(os.environ.get("LAB_HEAL_STATE_FILE", STATE_DIR / "heal-state.json")) +HEAL_COOLDOWN_SECONDS = int(os.environ.get("LAB_HEAL_COOLDOWN_SECONDS", "1800")) @dataclass(frozen=True) @@ -105,6 +108,7 @@ def command_for(row: dict[str, str], rule: HealRule) -> tuple[str, str, int]: def build_plan(status: dict[str, object]) -> list[dict[str, object]]: planned: list[dict[str, object]] = [] seen_commands: set[str] = set() + blocked = recent_failed_commands() for row in status_rows(status): rule = rule_for(row) if rule is None: @@ -123,7 +127,14 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: ) continue command, reason, priority = command_for(row, rule) - if command in seen_commands and rule.auto: + auto = rule.auto + if command in blocked: + failed_finding = blocked[command] + auto = False + command = "./jeannie doctor-cluster --details" + reason = f"Previous heal action did not clear this finding recently; escalate to diagnostics before retrying. Last failure: {failed_finding}" + priority = max(priority - 20, 1) + if command in seen_commands and auto: continue seen_commands.add(command) planned.append( @@ -133,7 +144,7 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: "status": row.get("status", ""), "command": command, "risk": rule.risk, - "auto": rule.auto, + "auto": auto, "priority": priority, "reason": reason, "summary": row.get("summary", ""), @@ -234,12 +245,15 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: print(f"run: {command}") process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False) if process.returncode != 0: + record_failed_action(command, target) return process.returncode if verify_healed(target): print(f"healed: {target['area']} / {target['check']}") + clear_failed_action(command) else: print(f"still failing: {target['area']} / {target['check']}") print("Run ./jeannie status --details for the current evidence.") + record_failed_action(command, target) return 1 if not yes: print() @@ -250,6 +264,69 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: return 0 +def load_heal_state() -> dict[str, object]: + if not HEAL_STATE_FILE.is_file(): + return {"failed_actions": {}} + try: + document = json.loads(HEAL_STATE_FILE.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return {"failed_actions": {}} + if not isinstance(document, dict): + return {"failed_actions": {}} + if not isinstance(document.get("failed_actions"), dict): + document["failed_actions"] = {} + return document + + +def write_heal_state(document: dict[str, object]) -> None: + HEAL_STATE_FILE.parent.mkdir(parents=True, exist_ok=True) + HEAL_STATE_FILE.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def recent_failed_commands() -> dict[str, str]: + now = int(time.time()) + document = load_heal_state() + failed = document.get("failed_actions", {}) + blocked: dict[str, str] = {} + changed = False + for command, info in list(failed.items()): + if not isinstance(info, dict): + failed.pop(command, None) + changed = True + continue + failed_at = int(info.get("failed_at", 0) or 0) + if now - failed_at > HEAL_COOLDOWN_SECONDS: + failed.pop(command, None) + changed = True + continue + blocked[command] = str(info.get("finding") or "unknown") + if changed: + document["failed_actions"] = failed + write_heal_state(document) + return blocked + + +def record_failed_action(command: str, target: dict[str, object]) -> None: + document = load_heal_state() + failed = document.setdefault("failed_actions", {}) + if isinstance(failed, dict): + failed[command] = { + "failed_at": int(time.time()), + "finding": f"{target.get('area')} / {target.get('check')}", + "summary": target.get("summary", ""), + } + write_heal_state(document) + print(f"recorded failed heal action cooldown: {command}") + + +def clear_failed_action(command: str) -> None: + document = load_heal_state() + failed = document.get("failed_actions", {}) + if isinstance(failed, dict) and command in failed: + failed.pop(command, None) + write_heal_state(document) + + def verify_healed(target: dict[str, object]) -> bool: timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180")) interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10")) diff --git a/tests/jeannie-unit b/tests/jeannie-unit index 6408129..82fce38 100755 --- a/tests/jeannie-unit +++ b/tests/jeannie-unit @@ -269,6 +269,57 @@ PY assert_command_failure_contains "inventory validator rejects invalid Debian LAN IP" "hosts.debian.lan_ip is not a valid IP" "${REPO_ROOT}/scripts/validate-homelab-inventory" "${inventory}" } +test_heal_cooldown_escalates_to_diagnostics() { + local status_json="${TMP_DIR}/heal-status.json" + local state_json="${TMP_DIR}/heal-state.json" + local output + + cat >"${status_json}" <<'EOF' +{ + "rows": [ + { + "area": "Pimox And Cluster", + "check": "Kubernetes nodes Ready", + "status": "fail", + "summary": "pimox-worker-01 NotReady", + "detail": "pimox-worker-01 NotReady" + }, + { + "area": "Platform And Apps", + "check": "Traefik deployment", + "status": "fail", + "summary": "deployment unavailable" + } + ] +} +EOF + python3 - "${state_json}" <<'PY' +import json +import sys +import time + +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump( + { + "failed_actions": { + "./jeannie workers restart 1": { + "failed_at": int(time.time()), + "finding": "Pimox And Cluster / Kubernetes nodes Ready", + } + } + }, + handle, + ) +PY + + output="$( + LAB_HEAL_STATE_FILE="${state_json}" LAB_HEAL_COOLDOWN_SECONDS=3600 \ + "${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}" + )" + assert_contains "heal cooldown removes auto target" "Next heal target: none" "${output}" + assert_contains "heal cooldown escalates diagnostics" "next diagnostic command: ./jeannie doctor-cluster --details" "${output}" +} + test_pimox_worker_count_detects_generated_topology test_configured_worker_count_cannot_hide_existing_workers test_configured_worker_count_can_expand_target_set @@ -280,6 +331,7 @@ test_report_renderer_problems_only_hides_ok_rows test_report_renderer_empty_problems_message test_inventory_validator_accepts_current_inventory test_inventory_validator_rejects_ip_drift +test_heal_cooldown_escalates_to_diagnostics if ((FAILURES > 0)); then printf '\n%s unit test(s) failed.\n' "${FAILURES}" >&2