Verify heal target after action

This commit is contained in:
juvdiaz 2026-06-30 14:24:19 -06:00
parent 8e19df1c71
commit f5c55af74f
4 changed files with 72 additions and 3 deletions

View File

@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
one action. Re-run `./jeannie status` after every heal cycle so Jeannie one action. Re-run `./jeannie status` after every heal cycle so Jeannie
can reassess before touching the next issue. If a specific Pimox worker is can reassess before touching the next issue. If a specific Pimox worker is
NotReady, heal targets that worker with `./jeannie workers restart N` NotReady, heal targets that worker with `./jeannie workers restart N`
instead of repeating a broad cluster start. Destructive actions remain blocked. instead of repeating a broad cluster start. After an action runs, Jeannie waits
for the target condition to clear and reports `healed` or `still failing`.
Destructive actions remain blocked.
## Model Behavior Observatory ## Model Behavior Observatory

View File

@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
can reassess before touching the next issue. If a specific Pimox worker is can reassess before touching the next issue. If a specific Pimox worker is
NotReady, heal targets that worker with `./{{ main_script }} workers restart N` NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
instead of repeating a broad cluster start. Destructive actions remain blocked. instead of repeating a broad cluster start. After an action runs, Jeannie waits
for the target condition to clear and reports `healed` or `still failing`.
Destructive actions remain blocked.
## Model Behavior Observatory ## Model Behavior Observatory

View File

@ -75,7 +75,8 @@ read-only and may include cited RAG context when the local index is available.
: Plan or apply guarded self-healing actions. `apply` executes at most one : Plan or apply guarded self-healing actions. `apply` executes at most one
highest-impact auto-eligible action allowed by highest-impact auto-eligible action allowed by
`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run `infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
`status` before the next heal cycle. `status` before the next heal cycle. After an action runs, Jeannie waits for the
target condition to clear and reports `healed` or `still failing`.
`capacity` `capacity`
: Print a compact read-only placement report covering Debian, Kubernetes, : Print a compact read-only placement report covering Debian, Kubernetes,

View File

@ -9,11 +9,13 @@ import os
import re import re
import subprocess import subprocess
import sys import sys
import time
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1] REPO_ROOT = Path(__file__).resolve().parents[1]
DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config")))
@dataclass(frozen=True) @dataclass(frozen=True)
@ -233,6 +235,12 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False) process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
if process.returncode != 0: if process.returncode != 0:
return process.returncode return process.returncode
if verify_healed(target):
print(f"healed: {target['area']} / {target['check']}")
else:
print(f"still failing: {target['area']} / {target['check']}")
print("Run ./jeannie status --details for the current evidence.")
return 1
if not yes: if not yes:
print() print()
print("Dry run only. Re-run with: ./jeannie heal apply --yes") print("Dry run only. Re-run with: ./jeannie heal apply --yes")
@ -242,6 +250,62 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
return 0 return 0
def verify_healed(target: dict[str, object]) -> bool:
timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180"))
interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10"))
deadline = time.monotonic() + timeout
while True:
if target_is_healed(target):
return True
if time.monotonic() >= deadline:
return False
remaining = int(deadline - time.monotonic())
print(f"waiting for heal verification ({max(remaining, 0)}s left)...")
time.sleep(interval)
def target_is_healed(target: dict[str, object]) -> bool:
command = str(target.get("command") or "")
worker_match = re.search(r"\bworkers\s+restart\s+(\d+)\b", command)
if worker_match:
return pimox_worker_ready(int(worker_match.group(1)))
try:
status = load_status(None)
except SystemExit:
return False
area = str(target.get("area") or "")
check = str(target.get("check") or "")
for row in status_rows(status):
if str(row.get("area") or "") == area and str(row.get("check") or "") == check:
return False
return True
def pimox_worker_ready(index: int) -> bool:
node_name = f"pimox-worker-{index:02d}"
process = subprocess.run(
[
"kubectl",
"--kubeconfig",
str(DEFAULT_KUBECONFIG),
"get",
"node",
node_name,
"--no-headers",
],
cwd=REPO_ROOT,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
check=False,
)
if process.returncode != 0:
return False
parts = process.stdout.split()
return len(parts) >= 2 and parts[1] == "Ready"
def main() -> int: def main() -> int:
parser = argparse.ArgumentParser(description=__doc__) parser = argparse.ArgumentParser(description=__doc__)
subparsers = parser.add_subparsers(dest="command", required=True) subparsers = parser.add_subparsers(dest="command", required=True)