refactor: modularize jeannie script into lib/jeannie library

lib/jeannie/cluster.sh

@@ -0,0 +1,499 @@
+#!/usr/bin/env bash
+
+pimox_ssh() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+
+    shift 3
+    ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@"
+}
+
+pimox_guest_ipv4() {
+    local guest_json
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local vmid="$4"
+    local ip_prefix="$5"
+    local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
+
+    guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)"
+    if [[ -z "${guest_json}" ]]; then
+        return 1
+    fi
+
+    GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY'
+import json
+import os
+import sys
+
+prefix = sys.argv[1]
+try:
+    interfaces = json.loads(os.environ.get("GUEST_JSON", ""))
+except Exception:
+    sys.exit(1)
+
+for iface in interfaces or []:
+    for address in iface.get("ip-addresses") or []:
+        if address.get("ip-address-type") != "ipv4":
+            continue
+        ip = address.get("ip-address", "")
+        if not ip or ip.startswith(("127.", "169.254.")):
+            continue
+        if prefix and not ip.startswith(prefix):
+            continue
+        print(ip)
+        sys.exit(0)
+sys.exit(1)
+PY
+}
+
+pimox_worker_vm_debug() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local vmid="$4"
+    local qm_bin="$5"
+    local script
+    local encoded_script
+
+    script="$(cat <<'EOF'
+set +e
+echo "-- addresses --"
+ip -br addr
+echo "-- routes --"
+ip route
+echo "-- units --"
+systemctl is-active ssh sshd 2>/dev/null || true
+echo "-- listeners --"
+ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true
+echo "-- sshd config test --"
+sudo sshd -t 2>&1 || true
+echo "-- recent ssh logs --"
+journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
+EOF
+)"
+    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
+    pimox_ssh "${host}" "${user}" "${key_path}" "set +e
+echo 'Pimox VM ${vmid} status:'
+sudo '${qm_bin}' status '${vmid}'
+echo 'Pimox VM ${vmid} config summary:'
+sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
+echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
+sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true
+echo 'Pimox VM ${vmid} guest SSH diagnostics:'
+sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\" || true" >&2 || true
+}
+
+pimox_worker_guest_agent_recovery_hint() {
+    local host="$1"
+    local user="$2"
+    local vmid="$3"
+    local qm_bin="$4"
+
+    cat >&2 <<EOF
+
+QEMU guest agent did not become available for Pimox VM ${vmid}.
+Fast checks:
+  ssh ${user}@${host} 'sudo ${qm_bin} status ${vmid}; sudo ${qm_bin} config ${vmid}; sudo ${qm_bin} guest cmd ${vmid} network-get-interfaces'
+
+Recovery options:
+  1. If the clone is still booting slowly, rerun the Pimox stage with a longer agent wait:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=600 ./jeannie up
+  2. If the same VM still reports "QEMU guest agent is not running", recreate the worker clone:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
+  3. If a recreated worker still has no guest agent, rebuild the template and worker:
+     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
+EOF
+}
+
+pimox_guest_exec_exitcode() {
+    local guest_exec_json="$1"
+
+    GUEST_EXEC_JSON="${guest_exec_json}" python3 - <<'PY'
+import json
+import os
+import sys
+
+try:
+    document = json.loads(os.environ.get("GUEST_EXEC_JSON", ""))
+except Exception:
+    sys.exit(2)
+
+try:
+    print(int(document.get("exitcode", 1)))
+except Exception:
+    sys.exit(2)
+PY
+}
+
+pimox_worker_guest_ssh_repair() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local vmid="$4"
+    local qm_bin="$5"
+    local script
+    local encoded_script
+    local repair_output
+    local repair_exitcode
+
+    script="$(cat <<'EOF'
+set -eu
+ssh_unit=""
+if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
+  ssh_unit=ssh.service
+elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then
+  ssh_unit=sshd.service
+else
+  echo "Neither ssh.service nor sshd.service exists in the guest" >&2
+  exit 1
+fi
+
+sudo ssh-keygen -A
+sudo install -d -m 0755 /run/sshd
+sudo mkdir -p /etc/ssh/sshd_config.d
+sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
+Port 22
+ListenAddress 0.0.0.0
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitRootLogin no
+UsePAM yes
+SSHD_CONFIG
+
+sudo sshd -t
+sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
+sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
+sudo systemctl enable "$ssh_unit" >/dev/null
+sudo systemctl restart "$ssh_unit"
+sleep 2
+sudo systemctl is-active "$ssh_unit"
+if command -v ss >/dev/null 2>&1; then
+  ss -ltnp 2>/dev/null || ss -ltn
+  ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
+fi
+EOF
+)"
+    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
+    if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
+        echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2
+        printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
+        return 1
+    fi
+
+    printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
+    if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then
+        return 0
+    fi
+    echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2
+    if [[ -n "${repair_exitcode:-}" ]]; then
+        echo "Guest exit code: ${repair_exitcode}" >&2
+    fi
+    return 1
+}
+
+pimox_generated_mac() {
+    local vmid="$1"
+
+    printf '02:68:10:%02x:%02x:%02x\n' \
+        $(((vmid >> 16) & 255)) \
+        $(((vmid >> 8) & 255)) \
+        $((vmid & 255))
+}
+
+pimox_worker_static_ip() {
+    local index="$1"
+    local static_ips="$2"
+    local current_index=1
+    local ip
+
+    static_ips="${static_ips//,/ }"
+    for ip in ${static_ips}; do
+        if ((current_index == index)); then
+            printf '%s\n' "${ip}"
+            return 0
+        fi
+        current_index=$((current_index + 1))
+    done
+
+    return 1
+}
+
+pimox_worker_net0_config() {
+    local vmid="$1"
+    local bridge="$2"
+    local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-deterministic}"
+    local mac
+
+    case "${mac_mode}" in
+        auto)
+            printf 'virtio,bridge=%s\n' "${bridge}"
+            ;;
+        deterministic)
+            mac="$(pimox_generated_mac "${vmid}")"
+            printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}"
+            ;;
+        *)
+            echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2
+            return 1
+            ;;
+    esac
+}
+
+pimox_worker_cpu_affinity() {
+    local index="$1"
+    local affinities="$2"
+    local worker_cores="$3"
+    local affinity
+    local affinity_index=1
+    local cpu_count
+
+    for affinity in ${affinities}; do
+        if ((affinity_index == index)); then
+            if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then
+                echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2
+                exit 1
+            fi
+            if ((cpu_count != worker_cores)); then
+                echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2
+                exit 1
+            fi
+            printf '%s\n' "${affinity}"
+            return 0
+        fi
+        affinity_index=$((affinity_index + 1))
+    done
+
+    echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2
+    exit 1
+}
+
+pimox_start_vm_with_cpuset() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local qm_bin="$4"
+    local vmid="$5"
+    local cpuset="$6"
+
+    if [[ -z "${cpuset}" ]]; then
+        pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'"
+        return
+    fi
+    if ! cpuset_cpu_count "${cpuset}" >/dev/null; then
+        echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2
+        exit 1
+    fi
+
+    # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated
+    # command under taskset so PSCI brings secondary guest CPUs online reliably.
+    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
+vmid='${vmid}'
+qm_bin='${qm_bin}'
+cpuset='${cpuset}'
+
+if ! command -v taskset >/dev/null 2>&1; then
+  echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2
+  exit 1
+fi
+if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then
+  echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2
+  exit 1
+fi
+
+start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\")
+cleanup() {
+  rm -f \"\$start_script\"
+}
+trap cleanup 0
+
+sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\"
+if ! awk -v vmid=\"\$vmid\" '
+  NR == 1 && ($1 == "/usr/bin/kvm" || $1 ~ /qemu-system-aarch64$/) { kvm = 1 }
+  $1 == "-id" && $2 == vmid { id = 1 }
+  END { exit (kvm && id) ? 0 : 1 }
+' \"\$start_script\"; then
+  echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2
+  exit 1
+fi
+smp_count=\$(awk '$1 == "-smp" { count++ } END { print count + 0 }' \"\$start_script\")
+if [ \"\$smp_count\" -ne 1 ]; then
+  echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2
+  exit 1
+fi
+
+sudo taskset -c \"\$cpuset\" bash \"\$start_script\""
+}
+
+pimox_shutdown_vm_gracefully() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local qm_bin="$4"
+    local vmid="$5"
+
+    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
+sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
+elapsed=0
+while [ \"\$elapsed\" -lt 300 ]; do
+  if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
+    exit 0
+  fi
+  sleep 5
+  elapsed=\$((elapsed + 5))
+done
+echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2
+exit 1"
+}
+
+pimox_guest_cpu_count() {
+    local guest_ip="$1"
+    local guest_user="$2"
+    local guest_key_path="$3"
+    local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
+
+    ssh -i "${guest_key_path}" \
+        -o BatchMode=yes \
+        -o ConnectTimeout=10 \
+        -o IdentitiesOnly=yes \
+        -o StrictHostKeyChecking=accept-new \
+        -o UserKnownHostsFile="${known_hosts_file}" \
+        "${guest_user}@${guest_ip}" \
+        'nproc --all'
+}
+
+cluster_worker_var_file_has_workers() {
+    local var_file="$1"
+
+    [[ -s "${var_file}" ]] || return 1
+    python3 - "${var_file}" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as handle:
+    document = json.load(handle)
+sys.exit(0 if document.get("worker_nodes") else 1)
+PY
+}
+
+cluster_worker_targets() {
+    local worker_ssh_targets="${WORKER_SSH_TARGETS-}"
+    local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
+
+    CLUSTER_WORKER_TARGETS=()
+    read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}"
+
+    if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then
+        while IFS= read -r target; do
+            [[ -n "${target}" ]] || continue
+            if [[ ! " ${CLUSTER_WORKER_TARGETS[*]-} " =~ [[:space:]]${target}[[:space:]] ]]; then
+                CLUSTER_WORKER_TARGETS+=("${target}")
+            fi
+        done < <(python3 - "${var_file}" <<'PY'
+import json
+import sys
+
+var_file = sys.argv[1]
+with open(var_file, encoding="utf-8") as handle:
+    document = json.load(handle)
+
+for key, node in sorted((document.get("worker_nodes") or {}).items()):
+    host = node.get("host")
+    user = node.get("user")
+    if host and user:
+        print(f"{user}@{host}")
+PY
+)
+    fi
+}
+
+pimox_worker_count_default() {
+    local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
+    local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
+    local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
+    local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
+    local max_count=0
+    local count
+    local state_count
+
+    if [[ -s "${spec_file}" ]]; then
+        count="$(awk -F'\t' -v key_prefix="${worker_key_prefix}" -v node_prefix="${worker_node_prefix}-" '
+            $1 ~ "^" key_prefix "[0-9]+$" {
+                worker_index = substr($1, length(key_prefix) + 1) + 0
+                if (worker_index > max) max = worker_index
+            }
+            $4 ~ "^" node_prefix "[0-9]+$" {
+                worker_index = substr($4, length(node_prefix) + 1) + 0
+                if (worker_index > max) max = worker_index
+            }
+            END { print max + 0 }
+        ' "${spec_file}")"
+        if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
+            max_count="${count}"
+        fi
+    fi
+
+    if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then
+        count="$(python3 - "${var_file}" "${worker_key_prefix}" "${worker_node_prefix}" <<'PY'
+import json
+import re
+import sys
+
+var_file, worker_key_prefix, worker_node_prefix = sys.argv[1:4]
+with open(var_file, encoding="utf-8") as handle:
+    document = json.load(handle)
+nodes = document.get("worker_nodes") or {}
+highest = 0
+key_pattern = re.compile(rf"^{re.escape(worker_key_prefix)}(\d+)$")
+node_pattern = re.compile(rf"^{re.escape(worker_node_prefix)}-(\d+)$")
+for key, node in nodes.items():
+    for candidate in (key, str(node.get("node_name", ""))):
+        match = key_pattern.match(candidate) or node_pattern.match(candidate)
+        if match:
+            highest = max(highest, int(match.group(1)))
+print(highest)
+PY
+)"
+        if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
+            max_count="${count}"
+        fi
+    fi
+
+    state_count="$(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" state show null_resource.worker_nodes_required 2>/dev/null |
+        awk -F'"' '/"worker_count"[[:space:]]*=/ { print $4; found = 1 } END { exit found ? 0 : 1 }' || true)"
+    if [[ "${state_count}" =~ ^[0-9]+$ && "${state_count}" -gt "${max_count}" ]]; then
+        max_count="${state_count}"
+    fi
+
+    count="${LAB_PIMOX_DEFAULT_WORKER_COUNT:-1}"
+    if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
+        max_count="${count}"
+    fi
+
+    printf '%s\n' "${max_count}"
+}
+
+pimox_worker_count_effective() {
+    local configured="${LAB_PIMOX_WORKER_COUNT:-}"
+    local detected
+
+    detected="$(pimox_worker_count_default)"
+    if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then
+        printf '%s\n' "${configured}"
+        return 0
+    fi
+    printf '%s\n' "${detected}"
+}
+
+cluster_control_plane_tracked() {
+    tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane"
+}
+
+cluster_admin_kubeconfig_present() {
+    [[ -s "${KUBECONFIG_PATH_PATH}" ]] ||
+        sudo test -s /etc/kubernetes/admin.conf 2>/dev/null ||
+        [[ -n "${KUBECONFIG_PATH:-}" && -s "${KUBECONFIG_PATH}" ]]
+}

lib/jeannie/preflight.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+homelab_preflight() {
+    local phase="${1:-full}"
+    local failures=0
+
+    require_debian_server "preflight"
+
+    if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then
+        echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}."
+        return 0
+    fi
+
+    case "${phase}" in
+        early | full)
+            ;;
+        *)
+            echo "Unsupported preflight phase '${phase}'." >&2
+            exit 1
+            ;;
+    esac
+
+    echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..."
+
+    preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1))
+    preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1))
+    preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1))
+    if [[ "${phase}" == "full" ]]; then
+        preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1))
+        preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1))
+    fi
+    preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip
+    preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
+    preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" check_edge_ssh || failures=$((failures + 1))
+
+    if ((failures > 0)); then
+        echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2
+        exit 1
+    fi
+
+    echo "${phase^} preflight checks passed."
+}
+
+doctor_preapply() {
+    local failures=0
+
+    require_debian_server "doctor-preapply"
+
+    echo "Running pre-apply readiness checks..."
+    preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1))
+    preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1))
+    preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1))
+    preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1))
+    preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1))
+    preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
+    preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1))
+    preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1))
+
+    if ((failures > 0)); then
+        echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2
+        exit 1
+    fi
+
+    echo "Pre-apply doctor checks passed."
+}
+
+inventory_check() {
+    "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
+}

lib/jeannie/security.sh

@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+
+secrets_init() {
+    local key_file
+    local key_dir
+    local recipient
+    local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}"
+    local example_file="${REPO_ROOT}/.sops.yaml.example"
+
+    require_debian_server "secrets-init"
+    ensure_sops_age_tools
+
+    key_file="$(sops_age_key_file)"
+    key_dir="$(dirname "${key_file}")"
+    mkdir -p "${key_dir}"
+    chmod 700 "${key_dir}"
+
+    if [[ ! -s "${key_file}" ]]; then
+        echo "Generating age identity at ${key_file}..."
+        age-keygen -o "${key_file}"
+        chmod 600 "${key_file}"
+    else
+        echo "Using existing age identity at ${key_file}."
+        chmod 600 "${key_file}"
+    fi
+
+    recipient="$(sops_age_recipient "${key_file}")"
+    if [[ -z "${recipient}" ]]; then
+        echo "Could not read the public recipient from ${key_file}." >&2
+        exit 1
+    fi
+
+    if [[ -e "${config_file}" ]]; then
+        if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then
+            echo "${config_file} still contains the placeholder age recipient." >&2
+            exit 1
+        fi
+        echo "SOPS config already exists: ${config_file}"
+    else
+        if [[ ! -s "${example_file}" ]]; then
+            echo "Missing ${example_file}" >&2
+            exit 1
+        fi
+        sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}"
+        echo "Wrote ${config_file} with Debian host age recipient ${recipient}."
+    fi
+
+    echo "Private age key: ${key_file}"
+    echo "Public age recipient: ${recipient}"
+    echo "Review and commit ${config_file}; never commit ${key_file}."
+}
+
+secrets_check_file() {
+    local file="$1"
+    local key_file="$2"
+
+    case "${file}" in
+        *.json)
+            grep -q '"sops"' "${file}" || {
+                echo "${file} does not look SOPS-encrypted." >&2
+                return 1
+            }
+            ;;
+        *)
+            grep -q '^sops:' "${file}" || {
+                echo "${file} does not look SOPS-encrypted." >&2
+                return 1
+            }
+            ;;
+    esac
+
+    if [[ -s "${key_file}" ]]; then
+        SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null
+    fi
+}
+
+secrets_check() {
+    local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}"
+    local key_file
+    local recipient=""
+    local failures=0
+    local found_files=0
+    local file
+
+    key_file="$(sops_age_key_file)"
+
+    if [[ ! -s "${config_file}" ]]; then
+        echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2
+        failures=$((failures + 1))
+    elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then
+        echo "${config_file} still contains the placeholder age recipient." >&2
+        failures=$((failures + 1))
+    fi
+
+    if [[ -s "${key_file}" ]]; then
+        recipient="$(sops_age_recipient "${key_file}")"
+        if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then
+            echo "${config_file} does not include this host's age recipient ${recipient}." >&2
+            failures=$((failures + 1))
+        fi
+    else
+        echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting."
+    fi
+
+    if command -v git >/dev/null 2>&1; then
+        while IFS= read -r file; do
+            [[ -n "${file}" ]] || continue
+            found_files=$((found_files + 1))
+            secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1))
+        done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json')
+    fi
+
+    if ((failures > 0)); then
+        echo "Secret checks failed with ${failures} issue(s)." >&2
+        exit 1
+    fi
+
+    if ((found_files == 0)); then
+        echo "SOPS config checks passed. No encrypted secret files are committed yet."
+    else
+        echo "SOPS config checks passed for ${found_files} encrypted secret file(s)."
+    fi
+}
+
+security_scan() {
+    require_debian_server "security-scan"
+    "${REPO_ROOT}/scripts/security-scan" all
+}
+
+security_prepare() {
+    require_debian_server "security-prepare"
+    "${REPO_ROOT}/scripts/security-scan" prepare
+}
+
+security_zap() {
+    require_debian_server "security-zap"
+    "${REPO_ROOT}/scripts/security-scan" zap
+}
+
+security_host() {
+    require_debian_server "security-host"
+    "${REPO_ROOT}/scripts/security-scan" host
+}
+
+security_trivy() {
+    require_debian_server "security-trivy"
+    "${REPO_ROOT}/scripts/security-scan" trivy
+}
+
+security_secrets() {
+    require_debian_server "security-secrets"
+    "${REPO_ROOT}/scripts/security-scan" secrets
+}
+
+security_nuclei() {
+    require_debian_server "security-nuclei"
+    "${REPO_ROOT}/scripts/security-scan" nuclei
+}
+
+security_web() {
+    require_debian_server "security-web"
+    "${REPO_ROOT}/scripts/security-scan" web
+}
+
+security_logs() {
+    require_debian_server "security-logs"
+    "${REPO_ROOT}/scripts/security-logs"
+}
+
+security_runtime() {
+    require_debian_server "security-runtime"
+    if ! command -v kubectl >/dev/null 2>&1; then
+        echo "kubectl is required for security-runtime." >&2
+        exit 1
+    fi
+
+    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/tetragon --timeout=30s
+    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -l app.kubernetes.io/name=tetragon -o wide
+    cat <<'EOF'
+
+Watch recent Tetragon events:
+  kubectl -n kube-system logs -l app.kubernetes.io/name=tetragon -c export-stdout --tail=100 -f
+
+Useful practice trigger:
+  kubectl -n security-lab exec deploy/juice-shop -- sh -c 'id; uname -a'
+EOF
+}
+
+security_attack_path() {
+    require_debian_server "security-attack-path"
+    "${REPO_ROOT}/scripts/security-attack-path" "${@:2}"
+}