diff --git a/README.md b/README.md index 3761b02..6e2b240 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that one action. Re-run `./jeannie status` after every heal cycle so Jeannie can reassess before touching the next issue. If a specific Pimox worker is NotReady, heal targets that worker with `./jeannie workers restart N` -instead of repeating a broad cluster start. Destructive actions remain blocked. +instead of repeating a broad cluster start. After an action runs, Jeannie waits +for the target condition to clear and reports `healed` or `still failing`. +Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/README.md.tmpl b/README.md.tmpl index abf4f9a..78161f2 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie can reassess before touching the next issue. If a specific Pimox worker is NotReady, heal targets that worker with `./{{ main_script }} workers restart N` -instead of repeating a broad cluster start. Destructive actions remain blocked. +instead of repeating a broad cluster start. After an action runs, Jeannie waits +for the target condition to clear and reports `healed` or `still failing`. +Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 3cfc13e..25ec933 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -75,7 +75,8 @@ read-only and may include cited RAG context when the local index is available. : Plan or apply guarded self-healing actions. `apply` executes at most one highest-impact auto-eligible action allowed by `infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run -`status` before the next heal cycle. +`status` before the next heal cycle. After an action runs, Jeannie waits for the +target condition to clear and reports `healed` or `still failing`. `capacity` : Print a compact read-only placement report covering Debian, Kubernetes, diff --git a/scripts/heal b/scripts/heal index 7811f1d..59885ee 100755 --- a/scripts/heal +++ b/scripts/heal @@ -9,11 +9,13 @@ import os import re import subprocess import sys +import time from dataclasses import dataclass from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config"))) @dataclass(frozen=True) @@ -233,6 +235,12 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False) if process.returncode != 0: return process.returncode + if verify_healed(target): + print(f"healed: {target['area']} / {target['check']}") + else: + print(f"still failing: {target['area']} / {target['check']}") + print("Run ./jeannie status --details for the current evidence.") + return 1 if not yes: print() print("Dry run only. Re-run with: ./jeannie heal apply --yes") @@ -242,6 +250,62 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int: return 0 +def verify_healed(target: dict[str, object]) -> bool: + timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180")) + interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10")) + deadline = time.monotonic() + timeout + while True: + if target_is_healed(target): + return True + if time.monotonic() >= deadline: + return False + remaining = int(deadline - time.monotonic()) + print(f"waiting for heal verification ({max(remaining, 0)}s left)...") + time.sleep(interval) + + +def target_is_healed(target: dict[str, object]) -> bool: + command = str(target.get("command") or "") + worker_match = re.search(r"\bworkers\s+restart\s+(\d+)\b", command) + if worker_match: + return pimox_worker_ready(int(worker_match.group(1))) + + try: + status = load_status(None) + except SystemExit: + return False + area = str(target.get("area") or "") + check = str(target.get("check") or "") + for row in status_rows(status): + if str(row.get("area") or "") == area and str(row.get("check") or "") == check: + return False + return True + + +def pimox_worker_ready(index: int) -> bool: + node_name = f"pimox-worker-{index:02d}" + process = subprocess.run( + [ + "kubectl", + "--kubeconfig", + str(DEFAULT_KUBECONFIG), + "get", + "node", + node_name, + "--no-headers", + ], + cwd=REPO_ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + ) + if process.returncode != 0: + return False + parts = process.stdout.split() + return len(parts) >= 2 and parts[1] == "Ready" + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) subparsers = parser.add_subparsers(dest="command", required=True)