Verify heal target after action

This commit is contained in:
juvdiaz 2026-06-30 14:24:19 -06:00
parent 8e19df1c71
commit f5c55af74f
4 changed files with 72 additions and 3 deletions

View File

@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
one action. Re-run `./jeannie status` after every heal cycle so Jeannie
can reassess before touching the next issue. If a specific Pimox worker is
NotReady, heal targets that worker with `./jeannie workers restart N`
instead of repeating a broad cluster start. Destructive actions remain blocked.
instead of repeating a broad cluster start. After an action runs, Jeannie waits
for the target condition to clear and reports `healed` or `still failing`.
Destructive actions remain blocked.
## Model Behavior Observatory

View File

@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
can reassess before touching the next issue. If a specific Pimox worker is
NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
instead of repeating a broad cluster start. Destructive actions remain blocked.
instead of repeating a broad cluster start. After an action runs, Jeannie waits
for the target condition to clear and reports `healed` or `still failing`.
Destructive actions remain blocked.
## Model Behavior Observatory

View File

@ -75,7 +75,8 @@ read-only and may include cited RAG context when the local index is available.
: Plan or apply guarded self-healing actions. `apply` executes at most one
highest-impact auto-eligible action allowed by
`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
`status` before the next heal cycle.
`status` before the next heal cycle. After an action runs, Jeannie waits for the
target condition to clear and reports `healed` or `still failing`.
`capacity`
: Print a compact read-only placement report covering Debian, Kubernetes,

View File

@ -9,11 +9,13 @@ import os
import re
import subprocess
import sys
import time
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config")))
@dataclass(frozen=True)
@ -233,6 +235,12 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
if process.returncode != 0:
return process.returncode
if verify_healed(target):
print(f"healed: {target['area']} / {target['check']}")
else:
print(f"still failing: {target['area']} / {target['check']}")
print("Run ./jeannie status --details for the current evidence.")
return 1
if not yes:
print()
print("Dry run only. Re-run with: ./jeannie heal apply --yes")
@ -242,6 +250,62 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
return 0
def verify_healed(target: dict[str, object]) -> bool:
timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180"))
interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10"))
deadline = time.monotonic() + timeout
while True:
if target_is_healed(target):
return True
if time.monotonic() >= deadline:
return False
remaining = int(deadline - time.monotonic())
print(f"waiting for heal verification ({max(remaining, 0)}s left)...")
time.sleep(interval)
def target_is_healed(target: dict[str, object]) -> bool:
command = str(target.get("command") or "")
worker_match = re.search(r"\bworkers\s+restart\s+(\d+)\b", command)
if worker_match:
return pimox_worker_ready(int(worker_match.group(1)))
try:
status = load_status(None)
except SystemExit:
return False
area = str(target.get("area") or "")
check = str(target.get("check") or "")
for row in status_rows(status):
if str(row.get("area") or "") == area and str(row.get("check") or "") == check:
return False
return True
def pimox_worker_ready(index: int) -> bool:
node_name = f"pimox-worker-{index:02d}"
process = subprocess.run(
[
"kubectl",
"--kubeconfig",
str(DEFAULT_KUBECONFIG),
"get",
"node",
node_name,
"--no-headers",
],
cwd=REPO_ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
check=False,
)
if process.returncode != 0:
return False
parts = process.stdout.split()
return len(parts) >= 2 and parts[1] == "Ready"
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True)