Add Jeannie output explainer

README.md

@@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
 ./jeannie recover-plan
 ./jeannie recover-power --dry-run
 ./jeannie scorecard
+./jeannie explain scorecard
 ./jeannie gitops-status
 ./jeannie cert-check
 ./jeannie grafana-dashboards list
@@ -333,6 +334,14 @@ edge/public checks. Use `recover-power --dry-run` to preview the sequence.
 health, edge health, capacity pressure, security posture, public certificates,
 inventory, and docs freshness into a compact pass/warn/fail report.
 
+`explain` is the read-only follow-up for warnings, failures, and corrective
+output. It can run safe report commands itself, such as
+`./jeannie explain scorecard`, `status`, `capacity`, or
+`resource-budget`, and it can explain captured output with
+`./jeannie explain output < saved-output.txt`. It lists what each
+finding means, diagnostic commands, whether a Jeannie fix command already
+exists, and risk/auto-fix safety.
+
 Report-oriented commands can share the reusable renderer in
 `scripts/report-render`. It reads status TSV rows and prints compact grouped
 summaries by default, with details or JSON available for future TUI/web output.

README.md.tmpl

@@ -275,6 +275,7 @@ Run a read-only health snapshot from the Debian server with:
 ./{{ main_script }} recover-plan
 ./{{ main_script }} recover-power --dry-run
 ./{{ main_script }} scorecard
+./{{ main_script }} explain scorecard
 ./{{ main_script }} gitops-status
 ./{{ main_script }} cert-check
 ./{{ main_script }} grafana-dashboards list
@@ -333,6 +334,14 @@ edge/public checks. Use `recover-power --dry-run` to preview the sequence.
 health, edge health, capacity pressure, security posture, public certificates,
 inventory, and docs freshness into a compact pass/warn/fail report.
 
+`explain` is the read-only follow-up for warnings, failures, and corrective
+output. It can run safe report commands itself, such as
+`./{{ main_script }} explain scorecard`, `status`, `capacity`, or
+`resource-budget`, and it can explain captured output with
+`./{{ main_script }} explain output < saved-output.txt`. It lists what each
+finding means, diagnostic commands, whether a Jeannie fix command already
+exists, and risk/auto-fix safety.
+
 Report-oriented commands can share the reusable renderer in
 `scripts/report-render`. It reads status TSV rows and prints compact grouped
 summaries by default, with details or JSON available for future TUI/web output.

docs/jeannie.1.md

@@ -124,6 +124,13 @@ Kubernetes pod requests/limits and Debian disk/memory signals. Use
 `control-plane taint` after Pimox workers are stable to keep normal app/data
 plane pods off the Debian host.
 
+`explain <read-only-command> [args...]`
+: Explain warnings, failures, and corrective findings from Jeannie output,
+including the meaning, diagnostic commands, whether a Jeannie fix command
+already exists, risk, and auto-fix safety. This command is read-only and refuses
+mutating commands. Use `explain scorecard [--all]` for scorecard-specific
+category explanations, or pipe captured output into `explain output`.
+
 `artifact-cache {status|up|down|instructions}`
 : Manage the optional Debian-host artifact cache stack for apt packages and
 Docker Hub pull-through image caching.

infra/jeannie-explain/findings.tsv

@@ -0,0 +1,36 @@
+pattern	meaning	diagnose_commands	jeannie_fix_commands	other_fix_commands	risk	auto_fix	notes
+unsupported template variable	Generated docs found a template placeholder that render-docs does not support.	./jeannie validate	none	Edit README.md.tmpl or scripts/render-docs	low	no	Usually a docs/template mismatch rather than an infrastructure issue.
+generated README.md is stale	README.md does not match README.md.tmpl rendering.	./jeannie validate	none	scripts/render-docs	low	candidate	A future docs-render Jeannie command could safely wrap this.
+missing_resource_policy	Workloads are missing resource requests or limits, so capacity planning is unreliable.	./jeannie resource-budget;./jeannie capacity	none	Edit app manifests or Helm values	medium	no	Add requests first, then limits for workloads you own.
+missing requests	One or more containers do not declare CPU or memory requests.	./jeannie resource-budget --details;./jeannie capacity	none	Edit app manifests or Helm values	medium	no	Start with application namespaces before system namespaces.
+missing limits	One or more containers do not declare CPU or memory limits.	./jeannie resource-budget --details;./jeannie capacity	none	Edit app manifests or Helm values	medium	no	Limits are useful but can cause throttling; set them deliberately.
+resource policy	Capacity or resource policy needs attention.	./jeannie resource-budget;./jeannie capacity	none	Edit infra/resource-budgets.yml or app manifests	medium	no	Do not hide pressure by only changing the budget file.
+diskpressure	A Kubernetes node reports disk pressure.	./jeannie capacity;./jeannie doctor-cluster	none	Clean images/logs or expand storage	high	no	DiskPressure can evict pods and should be handled quickly.
+memorypressure	A Kubernetes node reports memory pressure.	./jeannie capacity;./jeannie doctor-cluster	none	Reduce workloads or add worker capacity	high	no	Check whether Debian is carrying app pods.
+notready	A Kubernetes node is not Ready.	./jeannie doctor-cluster;./jeannie workers list	./jeannie start-cluster	none	high	no	Only start/recreate workers after checking node state.
+problem pods	Pods are Pending, CrashLooping, failing image pulls, or otherwise unhealthy.	./jeannie status;./jeannie doctor-cluster	none	kubectl describe/logs for the affected pod	medium	no	Use namespace and pod name from the original output.
+crashloopbackoff	A pod is repeatedly crashing.	./jeannie doctor-cluster	none	kubectl logs/describe for the pod	medium	no	Look for config, secret, dependency, or command errors.
+imagepullbackoff	A pod cannot pull its image.	./jeannie doctor-cluster	none	Check registry, image tag, platform, and pull secrets	medium	no	This often means a tag/platform mismatch or registry reachability issue.
+errimagepull	A pod cannot pull its image.	./jeannie doctor-cluster	none	Check registry, image tag, platform, and pull secrets	medium	no	This often means a tag/platform mismatch or registry reachability issue.
+kubernetes api unavailable	The Kubernetes API is not reachable.	./jeannie doctor-cluster;./jeannie start-cluster	./jeannie start-cluster	none	high	no	Starting the cluster is explicit; do not do it automatically.
+connection refused	A service endpoint is refusing connections.	./jeannie status;./jeannie doctor-cluster;./jeannie doctor-edge	none	Check the named service/container/pod	medium	no	Use the host and port in the original output.
+timed out	A service or DNS query timed out.	./jeannie status;./jeannie doctor-rpi;./jeannie doctor-edge	none	Check routing, firewall, DNS, or service health	medium	no	Timeouts often point to routing/firewall/service-down issues.
+no servers could be reached	DNS queries are failing.	./jeannie doctor-rpi;./jeannie status	./jeannie rpi-services	none	medium	no	RPi service redeploys can restart DNS; run explicitly.
+pihole	Pi-hole DNS or web health needs attention.	./jeannie doctor-rpi;./jeannie status	./jeannie rpi-services	none	medium	no	Check DNS before redeploying to avoid extending an outage.
+unbound	Unbound recursive DNS health needs attention.	./jeannie doctor-rpi	./jeannie rpi-services	none	medium	no	Pi-hole should still have fallback DNS if configured.
+uptime kuma	Uptime Kuma health or monitor coverage needs attention.	./jeannie doctor-rpi;./jeannie route-inventory	./jeannie rpi-services	none	low	no	Monitor seeding can be rerun, but verify credentials/config first.
+gitea	Gitea local, SSH, or public route health needs attention.	./jeannie doctor-gitea;./jeannie status	./jeannie deploy-gitea;./jeannie bootstrap-gitea-repo	none	medium	no	Deploy/bootstrap can change repo service state.
+http 404	A route answered but the expected path was not found.	./jeannie doctor-edge;./jeannie route-inventory	none	Check ingress, reverse proxy path, and app root URL	medium	no	404 can mean path rewrite or app sub-path mismatch.
+http 502	The reverse proxy cannot reach the backend.	./jeannie doctor-edge;./jeannie status	none	Check backend service, Traefik, edge proxy, and routes	high	no	502 is usually backend unreachable or unhealthy.
+certificate	TLS certificate, expiry, or public HTTPS validation needs attention.	./jeannie cert-check;./jeannie doctor-edge	./jeannie plan edge;./jeannie up	none	medium	no	Edge apply can affect public routing, so keep it explicit.
+duckdns-to-oci	Public DNS does not point where the inventory expects.	./jeannie cert-check;./jeannie doctor-edge	none	Update DuckDNS or edge inventory	medium	no	DNS drift can break public routes and cert issuance.
+tailscale	Tailscale IP, route acceptance, or peer reachability needs attention.	./jeannie status;./jeannie doctor-edge;./jeannie doctor-rpi	none	Check tailscale status/routes on the named host	medium	no	Tailscale ACL/route changes should be deliberate.
+docker root	Docker root path is not where expected.	./jeannie status;./jeannie preflight	./jeannie fix-debian-docker-root	none	medium	no	Docker root migration is stateful; verify host before running.
+pimox storage	Pimox storage is missing, wrong, or unhealthy.	./jeannie preflight;./jeannie doctor-cluster	none	Check pvesm status and opi5_ssd mount	high	no	Storage mistakes can break or strand VM disks.
+backup	Backup freshness or restore drill readiness needs attention.	./jeannie backup-status;./jeannie recover-plan	./jeannie backup-gitea;./jeannie state-backup;./jeannie drill-restore	none	medium	no	Backups are safer than restores, but keep drills explicit.
+restore drill	Restore drill reports are missing or stale.	./jeannie backup-status;./jeannie recover-plan	./jeannie drill-restore;./jeannie drill-gitea-restore;./jeannie drill-pihole-restore	none	medium	no	Drills can be noisy and should be run intentionally.
+argocd	Argo CD sync or health needs attention.	./jeannie gitops-status;./jeannie status	./jeannie apps	none	medium	no	App deployment can change workloads.
+outofsync	An Argo CD app differs from Git.	./jeannie gitops-status	./jeannie apps	none	medium	no	Confirm desired branch before syncing.
+degraded	An Argo CD app or workload is degraded.	./jeannie gitops-status;./jeannie status	none	Check app events/logs	medium	no	Fix depends on the degraded app.
+edge	Public edge routing or OCI edge service needs attention.	./jeannie doctor-edge;./jeannie cert-check	./jeannie plan edge;./jeannie up	none	high	no	Edge fixes can affect internet-facing access.
+security	Security policy or defensive scan needs attention.	./jeannie security-scan;./jeannie tailnet-policy-check	none	Review reported policy or finding	medium	no	Do not let AI auto-remediate security policy yet.

infra/jeannie-explain/scorecard.tsv

@@ -0,0 +1,11 @@
+label	meaning	diagnose_commands	jeannie_fix_commands	other_fix_commands	risk	auto_fix	notes
+Inventory	The canonical homelab inventory is missing or invalid, so Jeannie may use stale or unsafe defaults.	./jeannie inventory-check;./jeannie validate	none	Edit homelab.yml	medium	no	Fix the inventory before running mutating pipeline commands.
+Documentation freshness	Generated docs are stale or README.md.tmpl contains an unsupported template variable.	./jeannie validate	none	scripts/render-docs	low	candidate	A docs-render Jeannie command could safely wrap scripts/render-docs later.
+Backup readiness	Gitea, OpenTofu state, or restore drill backups are missing or stale.	./jeannie backup-status;./jeannie recover-plan	./jeannie backup-gitea;./jeannie state-backup;./jeannie drill-restore	none	medium	no	Backups are usually safe to create, but restore drills and retention choices should stay explicit.
+GitOps health	Argo CD applications, repository wiring, or sync health need attention.	./jeannie gitops-status;./jeannie status	./jeannie apps	none	medium	no	App syncs can change workloads, so Jeannie should explain before applying.
+DNS and RPi health	Pi-hole, Unbound, Uptime Kuma, RPi SSH, or RPi Docker state is unhealthy.	./jeannie doctor-rpi;./jeannie status	./jeannie rpi-services	none	medium	no	RPi service redeploys can restart DNS, so require explicit user action.
+Cluster health	Kubernetes API, nodes, pods, or Pimox workers are unhealthy.	./jeannie doctor-cluster;./jeannie status;./jeannie workers list	./jeannie start-cluster;./jeannie control-plane taint	none	medium	no	Start/taint operations are safe when intentional, but not automatic.
+Edge health	OCI edge, Traefik, public routes, or Gitea public routing need attention.	./jeannie doctor-edge;./jeannie cert-check;./jeannie status	./jeannie plan edge;./jeannie up	none	high	no	Edge fixes can alter public routing or apply infrastructure.
+Capacity pressure	Resource requests, limits, node pressure, storage pressure, or placement signals need attention.	./jeannie resource-budget;./jeannie capacity;./jeannie control-plane status	./jeannie control-plane taint;./jeannie workers list	none	medium	no	Most capacity fixes require manifest changes or scheduling decisions.
+Security posture	Tailscale ACL policy or static security checks need attention.	./jeannie tailnet-policy-check;./jeannie security-scan	none	Edit infra/tailscale/tailnet-policy.hujson	medium	no	Policy changes should be reviewed before applying to the tailnet.
+Public certificates	Public DNS, TLS expiry, website/Gitea URLs, or DuckDNS edge drift need attention.	./jeannie cert-check;./jeannie doctor-edge	./jeannie plan edge;./jeannie up	none	medium	no	Certificate and edge fixes can affect public access.

jeannie

@@ -6034,6 +6034,13 @@ route_inventory() {
     "${REPO_ROOT}/scripts/route-inventory"
 }
 
+explain() {
+    if [ "${2:-}" != "output" ]; then
+        require_debian_server "explain"
+    fi
+    "${REPO_ROOT}/scripts/explain" "${@:2}"
+}
+
 recover_power() {
     require_debian_server "recover-power"
     record_change_journal "recover-power" "$@"
@@ -6127,6 +6134,7 @@ Observability And GitOps
   route-inventory                            Report routes and Uptime Kuma coverage.
   control-plane {status|taint|untaint|pods}
                                              Keep normal pods off the Debian control plane.
+  explain <command>|output                   Explain warnings/errors and fix commands.
 
 Recovery And State
   backup-gitea                               Back up the Debian-hosted Gitea data.
@@ -6251,6 +6259,9 @@ case "${1:-}" in
     route-inventory)
         route_inventory
         ;;
+    explain)
+        explain "$@"
+        ;;
     recover-power)
         recover_power "$@"
         ;;

scripts/explain

@@ -0,0 +1,292 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+SCORECARD_MAP="${HOMELAB_SCORECARD_EXPLAIN_MAP:-${REPO_ROOT}/infra/jeannie-explain/scorecard.tsv}"
+FINDINGS_MAP="${HOMELAB_FINDINGS_EXPLAIN_MAP:-${REPO_ROOT}/infra/jeannie-explain/findings.tsv}"
+EXPLAIN_INPUT_FILE=""
+
+cleanup() {
+    if [ -n "$EXPLAIN_INPUT_FILE" ] && [ -f "$EXPLAIN_INPUT_FILE" ]; then
+        rm -f "$EXPLAIN_INPUT_FILE"
+    fi
+}
+trap cleanup EXIT
+
+usage() {
+    cat <<'EOF'
+Usage:
+  ./jeannie explain scorecard [--all]
+  ./jeannie explain <read-only-command> [args...]
+  ./jeannie explain output < file
+
+Explain warnings, failures, and corrective findings from Jeannie output. This is
+read-only and refuses mutating commands.
+
+Examples:
+  ./jeannie explain scorecard
+  ./jeannie explain status
+  ./jeannie explain capacity
+  ./jeannie explain resource-budget --details
+  ./jeannie status | ./jeannie explain output
+EOF
+}
+
+is_read_only_command() {
+    case "$1" in
+        status|capacity|recover-plan|scorecard|gitops-status|cert-check|backup-status|synthetic-checks|resource-budget|route-inventory|golden-ledger|workers|change-journal|map|validate|access-audit|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|secrets-check|tailnet-policy-check|ai-check|security-scan|security-host|security-trivy|security-secrets|security-nuclei|security-web|security-logs|security-attack-path|control-plane)
+            return 0
+            ;;
+        *)
+            return 1
+            ;;
+    esac
+}
+
+is_read_only_subcommand() {
+    local command_name="$1"
+    local subcommand="${2:-}"
+
+    case "$command_name" in
+        workers)
+            [ -z "$subcommand" ] || [ "$subcommand" = "list" ]
+            ;;
+        golden-ledger)
+            [ -z "$subcommand" ] || [ "$subcommand" = "show" ] || [ "$subcommand" = "check" ]
+            ;;
+        change-journal)
+            [ -z "$subcommand" ] || [ "$subcommand" = "list" ] || [ "$subcommand" = "path" ]
+            ;;
+        control-plane)
+            [ -z "$subcommand" ] || [ "$subcommand" = "status" ] || [ "$subcommand" = "pods" ]
+            ;;
+        *)
+            return 0
+            ;;
+    esac
+}
+
+run_target() {
+    local command_name="$1"
+    shift || true
+
+    if ! is_read_only_command "$command_name" || ! is_read_only_subcommand "$command_name" "${1:-}"; then
+        echo "Refusing to explain by running mutating or unknown command: ./jeannie ${command_name} $*" >&2
+        echo "Use './jeannie explain output < saved-output.txt' for arbitrary pasted output." >&2
+        return 2
+    fi
+
+    "${REPO_ROOT}/jeannie" "$command_name" "$@" 2>&1 || true
+}
+
+explain_output() {
+    local topic="$1"
+    local show_all="$2"
+    local input_file="$3"
+
+    if [ ! -s "$FINDINGS_MAP" ]; then
+        echo "Missing findings explain map: $FINDINGS_MAP" >&2
+        return 1
+    fi
+    if [ ! -s "$SCORECARD_MAP" ]; then
+        echo "Missing scorecard explain map: $SCORECARD_MAP" >&2
+        return 1
+    fi
+
+    python3 - "$topic" "$show_all" "$input_file" "$SCORECARD_MAP" "$FINDINGS_MAP" <<'PY'
+import csv
+import re
+import sys
+
+topic, show_all_arg, input_file, scorecard_map, findings_map = sys.argv[1:6]
+show_all = show_all_arg.lower() == "true"
+
+def read_tsv(path, key):
+    with open(path, encoding="utf-8", newline="") as handle:
+        return {row[key].lower(): row for row in csv.DictReader(handle, delimiter="\t")}
+
+scorecard_info = read_tsv(scorecard_map, "label")
+finding_info = read_tsv(findings_map, "pattern")
+
+with open(input_file, encoding="utf-8") as handle:
+    text = handle.read()
+
+lines = text.splitlines()
+
+def split_commands(value):
+    return [item.strip() for item in (value or "").split(";") if item.strip() and item.strip() != "none"]
+
+def print_info(info):
+    diagnose = split_commands(info.get("diagnose_commands"))
+    jeannie_fix = split_commands(info.get("jeannie_fix_commands"))
+    other_fix = split_commands(info.get("other_fix_commands"))
+
+    print(f"  means: {info.get('meaning') or 'No explain entry exists yet.'}")
+    if diagnose:
+        print("  diagnose:")
+        for command in diagnose:
+            print(f"    {command}")
+    if jeannie_fix:
+        print("  Jeannie fix command exists: yes")
+        for command in jeannie_fix:
+            print(f"    {command}")
+    else:
+        print("  Jeannie fix command exists: no")
+    if other_fix:
+        print("  non-Jeannie fix:")
+        for command in other_fix:
+            print(f"    {command}")
+    print(f"  risk: {info.get('risk') or 'unknown'}")
+    print(f"  auto-fix safe: {info.get('auto_fix') or 'no'}")
+    if info.get("notes"):
+        print(f"  note: {info['notes']}")
+
+def parse_scorecard():
+    rows = []
+    pattern = re.compile(r"^(?P<label>.{1,28}) (?P<status>pass|warn|fail)(?: - (?P<summary>.*))?$")
+    for line in lines:
+        match = pattern.match(line)
+        if not match:
+            continue
+        label = match.group("label").strip()
+        status = match.group("status").strip()
+        summary = (match.group("summary") or "").strip()
+        if not show_all and status == "pass":
+            continue
+        rows.append((label, status, summary))
+    return rows
+
+def interesting_lines():
+    keywords = [
+        "fail",
+        "warn",
+        "error",
+        "missing",
+        "unavailable",
+        "unhealthy",
+        "degraded",
+        "outofsync",
+        "notready",
+        "pressure",
+        "stale",
+        "expired",
+        "timeout",
+        "timed out",
+        "refused",
+        "404",
+        "502",
+        "backoff",
+        "crashloop",
+        "imagepull",
+        "no servers could be reached",
+        "needs cleanup",
+    ]
+    found = []
+    for line in lines:
+        lowered = line.lower()
+        if any(keyword in lowered for keyword in keywords):
+            found.append(line.strip())
+    return found
+
+def match_findings(line):
+    lowered = line.lower()
+    matches = []
+    for pattern, info in finding_info.items():
+        if pattern and pattern in lowered:
+            matches.append((pattern, info))
+    return matches
+
+print(f"Jeannie Explain: {topic}")
+print("=" * (17 + len(topic)))
+print("mode: read-only")
+
+if topic == "scorecard":
+    rows = parse_scorecard()
+    if not rows:
+        print("\nNo scorecard warnings or failures found.")
+    for label, status, summary in rows:
+        info = scorecard_info.get(label.lower(), {})
+        print(f"\n{label}")
+        print(f"  status: {status}")
+        if summary:
+            print(f"  observed: {summary}")
+        print_info(info)
+    raise SystemExit(0)
+
+found_lines = interesting_lines()
+seen_patterns = set()
+matched_any = False
+
+for line in found_lines:
+    matches = match_findings(line)
+    if not matches:
+        continue
+    print(f"\nFinding: {line}")
+    for pattern, info in matches:
+        if pattern in seen_patterns:
+            continue
+        seen_patterns.add(pattern)
+        matched_any = True
+        print(f"  matched: {pattern}")
+        print_info(info)
+
+if not matched_any:
+    if found_lines:
+        print("\nPotential issues were found, but no specific explain rule matched yet.")
+        print("Lines:")
+        for line in found_lines[:20]:
+            print(f"  {line}")
+        if len(found_lines) > 20:
+            print(f"  ... {len(found_lines) - 20} more")
+    else:
+        print("\nNo warning/error/corrective lines found.")
+PY
+}
+
+main() {
+    local topic="${1:-}"
+    local show_all=false
+
+    case "$topic" in
+        ""|-h|--help|help)
+            usage
+            return 0
+            ;;
+    esac
+    shift || true
+
+    EXPLAIN_INPUT_FILE="$(mktemp "${TMPDIR:-/tmp}/jeannie-explain.XXXXXX")"
+
+    if [ "$topic" = "output" ]; then
+        cat >"$EXPLAIN_INPUT_FILE"
+        explain_output "output" "$show_all" "$EXPLAIN_INPUT_FILE"
+        return
+    fi
+
+    if [ "$topic" = "scorecard" ]; then
+        while [ "$#" -gt 0 ]; do
+            case "$1" in
+                --all)
+                    show_all=true
+                    ;;
+                -h|--help|help)
+                    usage
+                    return 0
+                    ;;
+                *)
+                    usage >&2
+                    return 2
+                    ;;
+            esac
+            shift
+        done
+        run_target scorecard >"$EXPLAIN_INPUT_FILE"
+        explain_output scorecard "$show_all" "$EXPLAIN_INPUT_FILE"
+        return
+    fi
+
+    run_target "$topic" "$@" >"$EXPLAIN_INPUT_FILE"
+    explain_output "$topic" "$show_all" "$EXPLAIN_INPUT_FILE"
+}
+
+main "$@"