Restart NotReady Pimox worker during heal

README.md

@@ -284,7 +284,9 @@ Normal `status` remains read-only. The heal planner maps known findings to
 guarded actions, chooses one highest-impact auto-eligible finding, checks the
 proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
 one action. Re-run `./jeannie status` after every heal cycle so Jeannie
-can reassess before touching the next issue. Destructive actions remain blocked.
+can reassess before touching the next issue. If a specific Pimox worker is
+NotReady, heal targets that worker with `./jeannie workers restart N`
+instead of repeating a broad cluster start. Destructive actions remain blocked.
 
 ## Model Behavior Observatory
 

README.md.tmpl

@@ -284,7 +284,9 @@ Normal `status` remains read-only. The heal planner maps known findings to
 guarded actions, chooses one highest-impact auto-eligible finding, checks the
 proposed command against `infra/agent-sandbox/policy.tsv`, and runs only that
 one action. Re-run `./{{ main_script }} status` after every heal cycle so Jeannie
-can reassess before touching the next issue. Destructive actions remain blocked.
+can reassess before touching the next issue. If a specific Pimox worker is
+NotReady, heal targets that worker with `./{{ main_script }} workers restart N`
+instead of repeating a broad cluster start. Destructive actions remain blocked.
 
 ## Model Behavior Observatory
 

docs/jeannie.1.md

@@ -212,7 +212,7 @@ visible Uptime Kuma monitor coverage.
 
 `workers <command>`
 : Manage Kubernetes/Pimox worker lifecycle operations. Commands include `list`,
-`tailnet`, `start`, `stop`, `drain`, `uncordon`, `recreate-plan`, and
+`tailnet`, `start`, `stop`, `restart`, `drain`, `uncordon`, `recreate-plan`, and
 `rebalance`.
 
 `change-journal {list|path}`

infra/agent-sandbox/policy.tsv

@@ -8,6 +8,7 @@ pattern	decision	reason
 ./jeannie ai-evals	allow	Read-only eval harness.
 ./jeannie prompt-injection-lab	allow	Read-only security lab harness.
 ./jeannie start-cluster	allow	Idempotent recovery path that starts saved Kubernetes services and worker VMs without destroying state.
+./jeannie workers restart	allow	Targeted Pimox worker VM restart for one-at-a-time self-healing.
 ./jeannie deploy-gitea	allow	Idempotent Debian-hosted Gitea Compose repair path for this homelab.
 ./jeannie rpi-services	allow	Idempotent RPi Pi-hole/Unbound/Uptime Kuma Compose repair path for this homelab.
 ./jeannie up	approval	High-blast-radius deployment.

jeannie

@@ -6579,7 +6579,7 @@ Cluster Lifecycle
   rebuild-cluster                            Recreate the cluster through the guarded path.
   stop-cluster                               Stop Kubernetes and worker VMs without destroy.
   start-cluster                              Start Kubernetes and desired worker VMs.
-  workers <list|tailnet|start|stop|drain|uncordon|recreate-plan|rebalance>
+  workers <list|tailnet|start|stop|restart|drain|uncordon|recreate-plan|rebalance>
                                              Manage Pimox/Kubernetes workers.
   move-prometheus-stack-workers              Move monitoring workloads to worker nodes.
   doctor-versions [--all|--details|--verbose|--json]

scripts/heal

@@ -6,6 +6,7 @@ from __future__ import annotations
 import argparse
 import json
 import os
+import re
 import subprocess
 import sys
 from dataclasses import dataclass
@@ -79,6 +80,26 @@ def rule_for(row: dict[str, str]) -> HealRule | None:
     return None
 
 
+def pimox_worker_index(row: dict[str, str]) -> int | None:
+    text = " ".join(str(row.get(key, "")) for key in ("summary", "detail", "check"))
+    match = re.search(r"\bpimox-worker-(\d{1,2})\b", text)
+    if not match:
+        return None
+    return int(match.group(1))
+
+
+def command_for(row: dict[str, str], rule: HealRule) -> tuple[str, str, int]:
+    if rule.check == "Kubernetes nodes Ready":
+        index = pimox_worker_index(row)
+        if index is not None:
+            return (
+                f"./jeannie workers restart {index}",
+                f"Restart Pimox VM for pimox-worker-{index:02d}; a running VM with a NotReady kubelet can be wedged.",
+                98,
+            )
+    return rule.command, rule.reason, rule.priority
+
+
 def build_plan(status: dict[str, object]) -> list[dict[str, object]]:
     planned: list[dict[str, object]] = []
     seen_commands: set[str] = set()
@@ -99,19 +120,20 @@ def build_plan(status: dict[str, object]) -> list[dict[str, object]]:
                 }
             )
             continue
-        if rule.command in seen_commands and rule.auto:
+        command, reason, priority = command_for(row, rule)
+        if command in seen_commands and rule.auto:
             continue
-        seen_commands.add(rule.command)
+        seen_commands.add(command)
         planned.append(
             {
                 "check": row.get("check", ""),
                 "area": row.get("area", ""),
                 "status": row.get("status", ""),
-                "command": rule.command,
+                "command": command,
                 "risk": rule.risk,
                 "auto": rule.auto,
-                "priority": rule.priority,
-                "reason": rule.reason,
+                "priority": priority,
+                "reason": reason,
                 "summary": row.get("summary", ""),
             }
         )

scripts/workers

@@ -21,6 +21,7 @@ Commands:
   tailnet                    Install/join Tailscale on generated Pimox workers
   start [index|all]          Start Pimox worker VM(s)
   stop [index|all]           Gracefully stop Pimox worker VM(s)
+  restart [index|all]        Gracefully restart Pimox worker VM(s)
   drain <node>               Drain a Kubernetes worker node
   uncordon <node>            Mark a Kubernetes worker node schedulable
   recreate-plan <index>      Print the safe recreate sequence for one Pimox worker
@@ -138,6 +139,29 @@ stop_workers() {
     done
 }
 
+restart_workers() {
+    local index
+    local vmid
+    local timeout="${LAB_WORKER_SHUTDOWN_TIMEOUT:-90}"
+
+    for index in $(target_indexes "${1:-all}"); do
+        vmid="$(vmid_for_index "$index")"
+        echo "Restarting Pimox worker index $index VM $vmid..."
+        pimox_ssh "set -eu
+if ! sudo '$PIMOX_QM_BIN' status '$vmid' >/dev/null 2>&1; then
+  echo 'Pimox worker VM $vmid does not exist.' >&2
+  exit 1
+fi
+if sudo '$PIMOX_QM_BIN' status '$vmid' | grep -q 'status: running'; then
+  if ! sudo '$PIMOX_QM_BIN' shutdown '$vmid' --timeout '$timeout'; then
+    echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM $vmid.'
+    sudo '$PIMOX_QM_BIN' stop '$vmid'
+  fi
+fi
+sudo '$PIMOX_QM_BIN' start '$vmid'"
+    done
+}
+
 drain_node() {
     local node="${1:-}"
 
@@ -277,6 +301,9 @@ case "${1:-}" in
     stop)
         stop_workers "${2:-all}"
         ;;
+    restart)
+        restart_workers "${2:-all}"
+        ;;
     drain)
         drain_node "${2:-}"
         ;;