diff --git a/README.md b/README.md index 698621f..3761b02 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,9 @@ Normal `status` remains read-only. The heal planner maps known findings to guarded actions, chooses one highest-impact auto-eligible finding, checks the proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that one action. Re-run `./jeannie status` after every heal cycle so Jeannie -can reassess before touching the next issue. Destructive actions remain blocked. +can reassess before touching the next issue. If a specific Pimox worker is +NotReady, heal targets that worker with `./jeannie workers restart N` +instead of repeating a broad cluster start. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/README.md.tmpl b/README.md.tmpl index 4c6270f..abf4f9a 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -284,7 +284,9 @@ Normal `status` remains read-only. The heal planner maps known findings to guarded actions, chooses one highest-impact auto-eligible finding, checks the proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie -can reassess before touching the next issue. Destructive actions remain blocked. +can reassess before touching the next issue. If a specific Pimox worker is +NotReady, heal targets that worker with `./{{ main_script }} workers restart N` +instead of repeating a broad cluster start. Destructive actions remain blocked. ## Model Behavior Observatory diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 3ebb286..3cfc13e 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -212,7 +212,7 @@ visible Uptime Kuma monitor coverage. `workers ` : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`, -`tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and +`tailnet`, `start`, `stop`, `restart`, `drain`, `uncordon`, `recreate-plan`, and `rebalance`. `change-journal {list|path}` diff --git a/infra/agent-sandbox/policy.tsv b/infra/agent-sandbox/policy.tsv index 8e7c6d6..b4a84cc 100644 --- a/infra/agent-sandbox/policy.tsv +++ b/infra/agent-sandbox/policy.tsv @@ -8,6 +8,7 @@ pattern decision reason ./jeannie ai-evals allow Read-only eval harness. ./jeannie prompt-injection-lab allow Read-only security lab harness. ./jeannie start-cluster allow Idempotent recovery path that starts saved Kubernetes services and worker VMs without destroying state. +./jeannie workers restart allow Targeted Pimox worker VM restart for one-at-a-time self-healing. ./jeannie deploy-gitea allow Idempotent Debian-hosted Gitea Compose repair path for this homelab. ./jeannie rpi-services allow Idempotent RPi Pi-hole/Unbound/Uptime Kuma Compose repair path for this homelab. ./jeannie up approval High-blast-radius deployment. diff --git a/jeannie b/jeannie index d20915a..dac204d 100755 --- a/jeannie +++ b/jeannie @@ -6579,7 +6579,7 @@ Cluster Lifecycle rebuild-cluster Recreate the cluster through the guarded path. stop-cluster Stop Kubernetes and worker VMs without destroy. start-cluster Start Kubernetes and desired worker VMs. - workers + workers Manage Pimox/Kubernetes workers. move-prometheus-stack-workers Move monitoring workloads to worker nodes. doctor-versions [--all|--details|--verbose|--json] diff --git a/scripts/heal b/scripts/heal index 59399eb..7811f1d 100755 --- a/scripts/heal +++ b/scripts/heal @@ -6,6 +6,7 @@ from __future__ import annotations import argparse import json import os +import re import subprocess import sys from dataclasses import dataclass @@ -79,6 +80,26 @@ def rule_for(row: dict[str, str]) -> HealRule | None: return None +def pimox_worker_index(row: dict[str, str]) -> int | None: + text = " ".join(str(row.get(key, "")) for key in ("summary", "detail", "check")) + match = re.search(r"\bpimox-worker-(\d{1,2})\b", text) + if not match: + return None + return int(match.group(1)) + + +def command_for(row: dict[str, str], rule: HealRule) -> tuple[str, str, int]: + if rule.check == "Kubernetes nodes Ready": + index = pimox_worker_index(row) + if index is not None: + return ( + f"./jeannie workers restart {index}", + f"Restart Pimox VM for pimox-worker-{index:02d}; a running VM with a NotReady kubelet can be wedged.", + 98, + ) + return rule.command, rule.reason, rule.priority + + def build_plan(status: dict[str, object]) -> list[dict[str, object]]: planned: list[dict[str, object]] = [] seen_commands: set[str] = set() @@ -99,19 +120,20 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]: } ) continue - if rule.command in seen_commands and rule.auto: + command, reason, priority = command_for(row, rule) + if command in seen_commands and rule.auto: continue - seen_commands.add(rule.command) + seen_commands.add(command) planned.append( { "check": row.get("check", ""), "area": row.get("area", ""), "status": row.get("status", ""), - "command": rule.command, + "command": command, "risk": rule.risk, "auto": rule.auto, - "priority": rule.priority, - "reason": rule.reason, + "priority": priority, + "reason": reason, "summary": row.get("summary", ""), } ) diff --git a/scripts/workers b/scripts/workers index fec69af..a6b9e1d 100755 --- a/scripts/workers +++ b/scripts/workers @@ -21,6 +21,7 @@ Commands: tailnet Install/join Tailscale on generated Pimox workers start [index|all] Start Pimox worker VM(s) stop [index|all] Gracefully stop Pimox worker VM(s) + restart [index|all] Gracefully restart Pimox worker VM(s) drain Drain a Kubernetes worker node uncordon Mark a Kubernetes worker node schedulable recreate-plan Print the safe recreate sequence for one Pimox worker @@ -138,6 +139,29 @@ stop_workers() { done } +restart_workers() { + local index + local vmid + local timeout="${LAB_WORKER_SHUTDOWN_TIMEOUT:-90}" + + for index in $(target_indexes "${1:-all}"); do + vmid="$(vmid_for_index "$index")" + echo "Restarting Pimox worker index $index VM $vmid..." + pimox_ssh "set -eu +if ! sudo '$PIMOX_QM_BIN' status '$vmid' >/dev/null 2>&1; then + echo 'Pimox worker VM $vmid does not exist.' >&2 + exit 1 +fi +if sudo '$PIMOX_QM_BIN' status '$vmid' | grep -q 'status: running'; then + if ! sudo '$PIMOX_QM_BIN' shutdown '$vmid' --timeout '$timeout'; then + echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM $vmid.' + sudo '$PIMOX_QM_BIN' stop '$vmid' + fi +fi +sudo '$PIMOX_QM_BIN' start '$vmid'" + done +} + drain_node() { local node="${1:-}" @@ -277,6 +301,9 @@ case "${1:-}" in stop) stop_workers "${2:-all}" ;; + restart) + restart_workers "${2:-all}" + ;; drain) drain_node "${2:-}" ;;