Verify heal target after action
This commit is contained in:
parent
8e19df1c71
commit
f5c55af74f
|
|
@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
|
|||
one action. Re-run `./jeannie status` after every heal cycle so Jeannie
|
||||
can reassess before touching the next issue. If a specific Pimox worker is
|
||||
NotReady, heal targets that worker with `./jeannie workers restart N`
|
||||
instead of repeating a broad cluster start. Destructive actions remain blocked.
|
||||
instead of repeating a broad cluster start. After an action runs, Jeannie waits
|
||||
for the target condition to clear and reports `healed` or `still failing`.
|
||||
Destructive actions remain blocked.
|
||||
|
||||
## Model Behavior Observatory
|
||||
|
||||
|
|
|
|||
|
|
@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
|
|||
one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
|
||||
can reassess before touching the next issue. If a specific Pimox worker is
|
||||
NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
|
||||
instead of repeating a broad cluster start. Destructive actions remain blocked.
|
||||
instead of repeating a broad cluster start. After an action runs, Jeannie waits
|
||||
for the target condition to clear and reports `healed` or `still failing`.
|
||||
Destructive actions remain blocked.
|
||||
|
||||
## Model Behavior Observatory
|
||||
|
||||
|
|
|
|||
|
|
@ -75,7 +75,8 @@ read-only and may include cited RAG context when the local index is available.
|
|||
: Plan or apply guarded self-healing actions. `apply` executes at most one
|
||||
highest-impact auto-eligible action allowed by
|
||||
`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
|
||||
`status` before the next heal cycle.
|
||||
`status` before the next heal cycle. After an action runs, Jeannie waits for the
|
||||
target condition to clear and reports `healed` or `still failing`.
|
||||
|
||||
`capacity`
|
||||
: Print a compact read-only placement report covering Debian, Kubernetes,
|
||||
|
|
|
|||
64
scripts/heal
64
scripts/heal
|
|
@ -9,11 +9,13 @@ import os
|
|||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config")))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -233,6 +235,12 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
|
|||
process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
|
||||
if process.returncode != 0:
|
||||
return process.returncode
|
||||
if verify_healed(target):
|
||||
print(f"healed: {target['area']} / {target['check']}")
|
||||
else:
|
||||
print(f"still failing: {target['area']} / {target['check']}")
|
||||
print("Run ./jeannie status --details for the current evidence.")
|
||||
return 1
|
||||
if not yes:
|
||||
print()
|
||||
print("Dry run only. Re-run with: ./jeannie heal apply --yes")
|
||||
|
|
@ -242,6 +250,62 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
|
|||
return 0
|
||||
|
||||
|
||||
def verify_healed(target: dict[str, object]) -> bool:
|
||||
timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180"))
|
||||
interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10"))
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if target_is_healed(target):
|
||||
return True
|
||||
if time.monotonic() >= deadline:
|
||||
return False
|
||||
remaining = int(deadline - time.monotonic())
|
||||
print(f"waiting for heal verification ({max(remaining, 0)}s left)...")
|
||||
time.sleep(interval)
|
||||
|
||||
|
||||
def target_is_healed(target: dict[str, object]) -> bool:
|
||||
command = str(target.get("command") or "")
|
||||
worker_match = re.search(r"\bworkers\s+restart\s+(\d+)\b", command)
|
||||
if worker_match:
|
||||
return pimox_worker_ready(int(worker_match.group(1)))
|
||||
|
||||
try:
|
||||
status = load_status(None)
|
||||
except SystemExit:
|
||||
return False
|
||||
area = str(target.get("area") or "")
|
||||
check = str(target.get("check") or "")
|
||||
for row in status_rows(status):
|
||||
if str(row.get("area") or "") == area and str(row.get("check") or "") == check:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def pimox_worker_ready(index: int) -> bool:
|
||||
node_name = f"pimox-worker-{index:02d}"
|
||||
process = subprocess.run(
|
||||
[
|
||||
"kubectl",
|
||||
"--kubeconfig",
|
||||
str(DEFAULT_KUBECONFIG),
|
||||
"get",
|
||||
"node",
|
||||
node_name,
|
||||
"--no-headers",
|
||||
],
|
||||
cwd=REPO_ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL,
|
||||
check=False,
|
||||
)
|
||||
if process.returncode != 0:
|
||||
return False
|
||||
parts = process.stdout.split()
|
||||
return len(parts) >= 2 and parts[1] == "Ready"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
|
|
|||
Loading…
Reference in New Issue