Verify heal target after action
This commit is contained in:
parent
8e19df1c71
commit
f5c55af74f
|
|
@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
|
||||||
one action. Re-run `./jeannie status` after every heal cycle so Jeannie
|
one action. Re-run `./jeannie status` after every heal cycle so Jeannie
|
||||||
can reassess before touching the next issue. If a specific Pimox worker is
|
can reassess before touching the next issue. If a specific Pimox worker is
|
||||||
NotReady, heal targets that worker with `./jeannie workers restart N`
|
NotReady, heal targets that worker with `./jeannie workers restart N`
|
||||||
instead of repeating a broad cluster start. Destructive actions remain blocked.
|
instead of repeating a broad cluster start. After an action runs, Jeannie waits
|
||||||
|
for the target condition to clear and reports `healed` or `still failing`.
|
||||||
|
Destructive actions remain blocked.
|
||||||
|
|
||||||
## Model Behavior Observatory
|
## Model Behavior Observatory
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -286,7 +286,9 @@ proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
|
||||||
one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
|
one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
|
||||||
can reassess before touching the next issue. If a specific Pimox worker is
|
can reassess before touching the next issue. If a specific Pimox worker is
|
||||||
NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
|
NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
|
||||||
instead of repeating a broad cluster start. Destructive actions remain blocked.
|
instead of repeating a broad cluster start. After an action runs, Jeannie waits
|
||||||
|
for the target condition to clear and reports `healed` or `still failing`.
|
||||||
|
Destructive actions remain blocked.
|
||||||
|
|
||||||
## Model Behavior Observatory
|
## Model Behavior Observatory
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,8 @@ read-only and may include cited RAG context when the local index is available.
|
||||||
: Plan or apply guarded self-healing actions. `apply` executes at most one
|
: Plan or apply guarded self-healing actions. `apply` executes at most one
|
||||||
highest-impact auto-eligible action allowed by
|
highest-impact auto-eligible action allowed by
|
||||||
`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
|
`infra/agent-sandbox/policy.tsv`; destructive actions remain blocked. Re-run
|
||||||
`status` before the next heal cycle.
|
`status` before the next heal cycle. After an action runs, Jeannie waits for the
|
||||||
|
target condition to clear and reports `healed` or `still failing`.
|
||||||
|
|
||||||
`capacity`
|
`capacity`
|
||||||
: Print a compact read-only placement report covering Debian, Kubernetes,
|
: Print a compact read-only placement report covering Debian, Kubernetes,
|
||||||
|
|
|
||||||
64
scripts/heal
64
scripts/heal
|
|
@ -9,11 +9,13 @@ import os
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
DEFAULT_KUBECONFIG = Path(os.environ.get("KUBECONFIG", os.environ.get("LAB_KUBECONFIG_PATH", "/home/jv/.kube/config")))
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|
@ -233,6 +235,12 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
|
||||||
process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
|
process = subprocess.run(command.split(), cwd=REPO_ROOT, text=True, check=False)
|
||||||
if process.returncode != 0:
|
if process.returncode != 0:
|
||||||
return process.returncode
|
return process.returncode
|
||||||
|
if verify_healed(target):
|
||||||
|
print(f"healed: {target['area']} / {target['check']}")
|
||||||
|
else:
|
||||||
|
print(f"still failing: {target['area']} / {target['check']}")
|
||||||
|
print("Run ./jeannie status --details for the current evidence.")
|
||||||
|
return 1
|
||||||
if not yes:
|
if not yes:
|
||||||
print()
|
print()
|
||||||
print("Dry run only. Re-run with: ./jeannie heal apply --yes")
|
print("Dry run only. Re-run with: ./jeannie heal apply --yes")
|
||||||
|
|
@ -242,6 +250,62 @@ def apply_plan(plan: list[dict[str, object]], yes: bool) -> int:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def verify_healed(target: dict[str, object]) -> bool:
|
||||||
|
timeout = int(os.environ.get("LAB_HEAL_VERIFY_TIMEOUT_SECONDS", "180"))
|
||||||
|
interval = int(os.environ.get("LAB_HEAL_VERIFY_INTERVAL_SECONDS", "10"))
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while True:
|
||||||
|
if target_is_healed(target):
|
||||||
|
return True
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
return False
|
||||||
|
remaining = int(deadline - time.monotonic())
|
||||||
|
print(f"waiting for heal verification ({max(remaining, 0)}s left)...")
|
||||||
|
time.sleep(interval)
|
||||||
|
|
||||||
|
|
||||||
|
def target_is_healed(target: dict[str, object]) -> bool:
|
||||||
|
command = str(target.get("command") or "")
|
||||||
|
worker_match = re.search(r"\bworkers\s+restart\s+(\d+)\b", command)
|
||||||
|
if worker_match:
|
||||||
|
return pimox_worker_ready(int(worker_match.group(1)))
|
||||||
|
|
||||||
|
try:
|
||||||
|
status = load_status(None)
|
||||||
|
except SystemExit:
|
||||||
|
return False
|
||||||
|
area = str(target.get("area") or "")
|
||||||
|
check = str(target.get("check") or "")
|
||||||
|
for row in status_rows(status):
|
||||||
|
if str(row.get("area") or "") == area and str(row.get("check") or "") == check:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def pimox_worker_ready(index: int) -> bool:
|
||||||
|
node_name = f"pimox-worker-{index:02d}"
|
||||||
|
process = subprocess.run(
|
||||||
|
[
|
||||||
|
"kubectl",
|
||||||
|
"--kubeconfig",
|
||||||
|
str(DEFAULT_KUBECONFIG),
|
||||||
|
"get",
|
||||||
|
"node",
|
||||||
|
node_name,
|
||||||
|
"--no-headers",
|
||||||
|
],
|
||||||
|
cwd=REPO_ROOT,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if process.returncode != 0:
|
||||||
|
return False
|
||||||
|
parts = process.stdout.split()
|
||||||
|
return len(parts) >= 2 and parts[1] == "Ready"
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue