#!/usr/bin/env bash
# shellcheck disable=SC2034,SC2043,SC2140

load_homelab_inventory_defaults() {
    local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
    local rendered_defaults

    if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then
        return 0
    fi

    rendered_defaults="$(python3 - "${inventory_file}" <<'PY'
import re
import json
import shlex
import sys

inventory_file = sys.argv[1]
mapping = {
    "domain.base": "LAB_DOMAIN",
    "domain.public_url": "LAB_PUBLIC_URL",
    "domain.gitea_url": "LAB_GITEA_ROOT_URL",
    "network.lan_cidr": "LAB_LAN_CIDR",
    "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX",
    "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP",
    "hosts.debian.user": "LAB_DEBIAN_USER",
    "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP",
    "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP",
    "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT",
    "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH",
    "hosts.rpi4.user": "LAB_RPI_USER",
    "hosts.rpi4.lan_ip": "LAB_RPI_HOST",
    "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP",
    "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT",
    "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT",
    "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT",
    "hosts.opi5_pimox.user": "LAB_PIMOX_USER",
    "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST",
    "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE",
    "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE",
    "hosts.oci_edge.user": "LAB_EDGE_USER",
    "hosts.oci_edge.public_ip": "LAB_EDGE_HOST",
    "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR",
    "services.gitea.http_port": "LAB_GITEA_HTTP_PORT",
    "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT",
    "services.gitea.root_url": "LAB_GITEA_ROOT_URL",
    "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL",
    "services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR",
    "services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT",
    "services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL",
    "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT",
    "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS",
    "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR",
    "services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE",
    "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT",
    "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT",
    "ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER",
    "ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED",
    "ai_gateway.url": "LAB_AI_GATEWAY_URL",
    "ai_gateway.model": "LAB_AI_GATEWAY_MODEL",
    "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS",
    "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR",
    "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID",
    "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT",
}

def flatten(document, prefix=""):
    values = {}
    if isinstance(document, dict):
        for key, value in document.items():
            path = f"{prefix}.{key}" if prefix else str(key)
            if isinstance(value, dict):
                values.update(flatten(value, path))
            elif isinstance(value, list):
                values[path] = ",".join(str(item) for item in value)
            elif value is not None:
                values[path] = str(value)
    return values


def parse_simple_inventory(path):
    stack = []
    values = {}
    pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$")

    with open(path, encoding="utf-8") as handle:
        for raw_line in handle:
            if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "):
                continue
            match = pattern.match(raw_line.rstrip("\n"))
            if not match:
                continue
            indent = len(match.group(1))
            key = match.group(2)
            value = (match.group(3) or "").strip()
            while stack and stack[-1][0] >= indent:
                stack.pop()
            current_path = ".".join([item[1] for item in stack] + [key])
            if value == "":
                stack.append((indent, key))
                continue
            if " #" in value:
                value = value.split(" #", 1)[0].strip()
            values[current_path] = value.strip("\"'")
    return values


try:
    import yaml
except ImportError:
    values = parse_simple_inventory(inventory_file)
else:
    with open(inventory_file, encoding="utf-8") as handle:
        values = flatten(yaml.safe_load(handle) or {})

for path, env_name in mapping.items():
    value = values.get(path)
    if value:
        print(f": ${{{env_name}:={shlex.quote(value)}}}")

debian_ip = values.get("hosts.debian.lan_ip")
gitea_http = values.get("services.gitea.http_port")
if debian_ip:
    print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}")
if debian_ip and gitea_http:
    print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}")
gitea_ts = values.get("hosts.debian.tailscale_ip")
if gitea_ts:
    print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}")
subdomains = values.get("domain.subdomains")
if subdomains:
    if isinstance(subdomains, str):
        subdomain_values = [item.strip() for item in subdomains.split(",") if item.strip()]
    else:
        subdomain_values = list(subdomains)
    if subdomain_values:
        print(f": ${{LAB_ADDITIONAL_SERVER_NAMES_JSON:={shlex.quote(json.dumps(subdomain_values))}}}")
PY
)"

    if [[ -n "${rendered_defaults}" ]]; then
        eval "${rendered_defaults}"
    fi
}

export_if_unset() {
    local name="$1"
    local value="$2"

    if [[ -z "${value}" ]]; then
        return 0
    fi
    if [[ -z "${!name:-}" ]]; then
        printf -v "${name}" '%s' "${value}"
    fi
    # shellcheck disable=SC2163
    export "${name?}"
}

export_homelab_inventory_tf_vars() {
    export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}"
    export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}"
    export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}"
    export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}"
    export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}"
    export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}"
    export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}"
    export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}"
    export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}"
    export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}"
    export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}"
    export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}"
    export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}"
    export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}"
    export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}"
    export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}"
    export_if_unset TF_VAR_additional_server_names "${LAB_ADDITIONAL_SERVER_NAMES_JSON:-}"
    export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}"
    export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}"
    export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}"
    export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}"
    export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}"
    export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}"
    export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}"
    export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}"
    export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}"
}

require_debian_server() {
    local command_name="$1"
    local os_id=""

    if [[ "$(uname -s)" != "Linux" ]]; then
        echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2
        exit 1
    fi

    if [[ -r /etc/os-release ]]; then
        os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)"
    fi

    if [[ "${os_id}" != "debian" ]]; then
        echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2
        exit 1
    fi
}

tofu_state_has_resource() {
    local stack="$1"
    local resource_address="$2"

    tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1
}

helm_release_secret_exists() {
    local namespace="$1"
    local release_name="$2"
    local secret_name

    secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \
        -l "owner=helm,name=${release_name}" \
        -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"

    [[ -n "${secret_name}" ]]
}

kubernetes_resource_exists() {
    local namespace="$1"
    local resource_kind="$2"
    local resource_name="$3"

    if [[ -n "${namespace}" ]]; then
        kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
        return $?
    fi

    kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
}

adopt_tofu_helm_release() {
    local stack="$1"
    local resource_address="$2"
    local namespace="$3"
    local release_name="$4"
    local has_state=false
    local has_release_secret=false

    if tofu_state_has_resource "${stack}" "${resource_address}"; then
        has_state=true
    fi
    if helm_release_secret_exists "${namespace}" "${release_name}"; then
        has_release_secret=true
    fi

    if [[ "${has_state}" == "true" && "${has_release_secret}" == "false" ]]; then
        echo "Removing stale Helm release state for ${namespace}/${release_name} from ${stack} (${resource_address}) because no Helm release secret exists..."
        tofu -chdir="${REPO_ROOT}/${stack}" state rm "${resource_address}"
        return 0
    fi

    if [[ "${has_state}" == "true" ]]; then
        return 0
    fi
    if [[ "${has_release_secret}" == "false" ]]; then
        return 0
    fi

    echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..."
    tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}"
}

adopt_tofu_kubernetes_resource() {
    local stack="$1"
    local resource_address="$2"
    local namespace="$3"
    local resource_kind="$4"
    local resource_name="$5"
    local import_id="$6"

    if tofu_state_has_resource "${stack}" "${resource_address}"; then
        return 0
    fi
    if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then
        return 0
    fi

    echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..."
    tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
}

adopt_tofu_kubernetes_manifest() {
    local stack="$1"
    local resource_address="$2"
    local namespace="$3"
    local kubectl_kind="$4"
    local api_version="$5"
    local manifest_kind="$6"
    local resource_name="$7"
    local import_id

    if tofu_state_has_resource "${stack}" "${resource_address}"; then
        return 0
    fi
    if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then
        return 0
    fi

    import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}"
    echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..."
    tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
}

adopt_platform_existing_resources() {
    local stack="bootstrap/platform"

    adopt_tofu_helm_release "${stack}" "helm_release.calico_crds" "tigera-operator" "calico-crds"
    adopt_tofu_helm_release "${stack}" "helm_release.calico" "tigera-operator" "calico"
    adopt_tofu_helm_release "${stack}" "helm_release.openebs" "openebs" "openebs"
    adopt_tofu_helm_release "${stack}" "helm_release.metallb[\"enabled\"]" "metallb-system" "metallb"
    adopt_tofu_helm_release "${stack}" "helm_release.traefik[\"enabled\"]" "traefik" "traefik"
    adopt_tofu_helm_release "${stack}" "helm_release.argocd" "argocd" "argocd"
    adopt_tofu_helm_release "${stack}" "helm_release.kyverno" "kyverno" "kyverno"
    adopt_tofu_helm_release "${stack}" "helm_release.kyverno_policies" "kyverno" "kyverno-policies"
    adopt_tofu_helm_release "${stack}" "helm_release.loki" "monitoring" "loki"
    adopt_tofu_helm_release "${stack}" "helm_release.promtail" "monitoring" "promtail"
    adopt_tofu_helm_release "${stack}" "helm_release.prometheus_stack" "monitoring" "prometheus-stack"
    adopt_tofu_kubernetes_resource \
        "${stack}" \
        "kubernetes_storage_class_v1.openebs_hostpath_retain" \
        "" \
        "storageclass" \
        "openebs-hostpath-retain" \
        "openebs-hostpath-retain"
    adopt_tofu_kubernetes_resource \
        "${stack}" \
        "kubernetes_namespace_v1.monitoring" \
        "" \
        "namespace" \
        "monitoring" \
        "monitoring"
}

adopt_apps_existing_resources() {
    local stack="bootstrap/apps"
    local namespace="${TF_VAR_argocd_namespace:-argocd}"

    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["container-registry"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "container-registry"
    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["website-production"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "website-production"
    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["demos-static"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "demos-static"
    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["heimdall"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "heimdall"
    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["n8n"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "n8n"
    adopt_tofu_kubernetes_manifest \
        "${stack}" \
        'kubernetes_manifest.argocd_application["supply-chain-policy"]' \
        "${namespace}" \
        "applications.argoproj.io" \
        "argoproj.io/v1alpha1" \
        "Application" \
        "supply-chain-policy"
}

ensure_homelab_node_labels() {
    local control_plane_node="${LAB_CONTROL_PLANE_NODE_NAME:-debian}"
    local raspberry_node="${LAB_RASPBERRY_NODE_NAME:-raspberry}"
    local prometheus_selector="homelab.dev/node-role=app,homelab.dev/storage=ssd"
    local node
    local target_nodes

    echo "Applying homelab labels to existing Kubernetes nodes..."
    while IFS= read -r node; do
        [[ -n "${node}" ]] || continue

        if [[ "${node}" == "${control_plane_node}" ]]; then
            kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
                homelab.dev/node-role=control-plane \
                homelab.dev/storage=local \
                homelab.dev/workload-class=control-plane \
                --overwrite
            continue
        fi

        kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
            node-role.kubernetes.io/worker=worker \
            --overwrite

        if [[ "${node}" == pimox-worker-* ]]; then
            kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
                homelab.dev/node-role=app \
                homelab.dev/storage=ssd \
                homelab.dev/workload-class=platform \
                --overwrite
        elif [[ "${node}" == "${raspberry_node}" ]]; then
            kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
                homelab.dev/node-role=edge-app \
                homelab.dev/storage=local \
                homelab.dev/workload-class=edge \
                --overwrite
        fi
    done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')

    target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)"
    if [[ -z "${target_nodes}" ]]; then
        echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2
        exit 1
    fi
}

delete_prometheus_stack_storage() {
    local namespace="${1:-monitoring}"
    local pattern='(^|-)prometheus-stack-(prometheus|alertmanager|grafana)(-|$)|^prometheus-prometheus-stack|^alertmanager-prometheus-stack|^storage-prometheus-stack-grafana'
    local pvc_names
    local pv_names

    pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null |
        awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')"
    pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \
        -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null |
        awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')"

    if [[ -n "${pvc_names}" ]]; then
        echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..."
        printf '%s\n' "${pvc_names}" |
            xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s
    fi

    if [[ -n "${pv_names}" ]]; then
        echo "Deleting old prometheus-stack retained PV objects..."
        printf '%s\n' "${pv_names}" |
            xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false
    fi
}

run_tofu_stack() {
    local stack="$1"
    local auto_approve="${LAB_AUTO_APPROVE:-true}"
    local -a apply_args=()

    if [[ "${stack}" == "bootstrap/cluster" ]]; then
        ensure_cluster_worker_var_file
    fi
    if [[ "${stack}" == "bootstrap/edge" ]]; then
        ensure_edge_tailscale_routes
        ensure_edge_haproxy_stats_password
    fi

    if truthy "${auto_approve}"; then
        apply_args+=("-auto-approve")
    elif ! disabled_value "${auto_approve}"; then
        echo "LAB_AUTO_APPROVE must be true or false." >&2
        exit 1
    fi

    if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
        apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
    fi

    tofu -chdir="${REPO_ROOT}/${stack}" init
    ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
    if [[ "${stack}" == "bootstrap/platform" ]]; then
        adopt_platform_existing_resources
    fi
    if [[ "${stack}" == "bootstrap/apps" ]]; then
        adopt_apps_existing_resources
    fi
    ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
    tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}"
}

run_tofu_plan_stack() {
    local stack="$1"
    local -a plan_args=()

    if [[ "${stack}" == "bootstrap/cluster" ]]; then
        ensure_cluster_worker_var_file
    fi

    if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
        plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
    fi

    tofu -chdir="${REPO_ROOT}/${stack}" init
    tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}"
}

backup_tofu_state() {
    local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}"
    local timestamp
    local archive
    local -a paths=()
    local path

    timestamp="$(date +%Y%m%d-%H%M%S)"
    archive="${backup_dir}/tofu-state-${timestamp}.tgz"
    mkdir -p "${backup_dir}"

    for path in \
        bootstrap/provisioning/terraform.tfstate \
        bootstrap/provisioning/terraform.tfstate.backup \
        bootstrap/cluster/terraform.tfstate \
        bootstrap/cluster/terraform.tfstate.backup \
        bootstrap/platform/terraform.tfstate \
        bootstrap/platform/terraform.tfstate.backup \
        bootstrap/apps/terraform.tfstate \
        bootstrap/apps/terraform.tfstate.backup \
        bootstrap/edge/terraform.tfstate \
        bootstrap/edge/terraform.tfstate.backup \
        .lab/cluster-workers.auto.tfvars.json \
        .lab/pimox-workers.tsv \
        .lab/manual-workers.tsv; do
        if [[ -e "${REPO_ROOT}/${path}" ]]; then
            paths+=("${path}")
        fi
    done

    if ((${#paths[@]} == 0)); then
        echo "No local OpenTofu state files found to back up."
        return 0
    fi

    tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}"
    chmod 0600 "${archive}"
    echo "Backed up local OpenTofu state to ${archive}."
}

move_prometheus_stack_workers() {
    local stack="bootstrap/platform"
    local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}"
    local auto_approve="${LAB_AUTO_APPROVE:-true}"
    local -a approve_args=()

    require_debian_server "move-prometheus-stack-workers"

    if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then
        cat >&2 <<'EOF'
move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data.
Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended.
EOF
        exit 1
    fi

    if truthy "${auto_approve}"; then
        approve_args+=("-auto-approve")
    elif ! disabled_value "${auto_approve}"; then
        echo "LAB_AUTO_APPROVE must be true or false." >&2
        exit 1
    fi

    export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
    export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"

    echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted."
    ensure_homelab_node_labels
    tofu -chdir="${REPO_ROOT}/${stack}" init
    adopt_platform_existing_resources
    tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}"
    delete_prometheus_stack_storage "${namespace}"
    tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}"
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
}

doctor_versions_report() {
    local api_version
    local api_minor
    local containerd_major_count
    local kubelet_minor_count
    local node_rows
    local status=0

    require_debian_server "doctor-versions"
    ensure_python3

    api_version="$(
        kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
            python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
    )"
    api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
    node_rows="$(
        kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
            -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
    )"

    if [[ -z "${node_rows}" ]]; then
        echo "No Kubernetes nodes found." >&2
        return 1
    fi

    printf 'API server: %s\n\n' "${api_version}"
    printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME"
    printf '%s\n' "${node_rows}" |
        awk -F '\t' '{
            version = $2
            gsub(/^v/, "", version)
            split(version, parts, ".")
            printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3
        }'

    kubelet_minor_count="$(
        printf '%s\n' "${node_rows}" |
            awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
            sort -u |
            wc -l |
            tr -d ' '
    )"
    containerd_major_count="$(
        printf '%s\n' "${node_rows}" |
            awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
            sort -u |
            wc -l |
            tr -d ' '
    )"

    if [[ "${kubelet_minor_count}" != "1" ]]; then
        echo
        echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2
        status=1
    fi

    if printf '%s\n' "${node_rows}" |
        awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then
        echo
        echo "At least one kubelet does not match the API server minor (${api_minor})." >&2
        status=1
    fi

    if [[ "${containerd_major_count}" -gt 1 ]]; then
        echo
        echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2
        status=1
    fi

    if [[ "${status}" -eq 0 ]]; then
        echo
        echo "Node Kubernetes and containerd versions are aligned."
    fi

    return "${status}"
}

doctor_versions_compact() {
    local api_version
    local api_minor
    local node_rows
    local kubelet_minor_count
    local containerd_major_count
    local mismatch_count
    local failures=0

    ensure_python3
    report_ui_begin

    if ! status_kubernetes_api_ok; then
        report_ui_emit fail "Kubernetes" "API" "not reachable" "kubectl readyz failed" "./jeannie start-cluster"
        report_ui_render "Doctor Versions"
        report_ui_cleanup
        return 1
    fi

    api_version="$(
        kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
            python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
    )"
    api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
    node_rows="$(
        kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
            -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
    )"

    if [[ -z "${node_rows}" ]]; then
        report_ui_emit fail "Kubernetes" "Nodes" "none found" "kubectl returned no node rows" "./jeannie doctor-cluster"
        report_ui_render "Doctor Versions"
        report_ui_cleanup
        return 1
    fi

    kubelet_minor_count="$(
        printf '%s\n' "${node_rows}" |
            awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
            sort -u |
            wc -l |
            tr -d ' '
    )"
    mismatch_count="$(
        printf '%s\n' "${node_rows}" |
            awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) count++ } END { print count + 0 }'
    )"
    containerd_major_count="$(
        printf '%s\n' "${node_rows}" |
            awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
            sort -u |
            wc -l |
            tr -d ' '
    )"

    report_ui_emit ok "Kubernetes" "API version" "${api_version}"
    if [[ "${kubelet_minor_count}" != "1" || "${mismatch_count}" != "0" ]]; then
        report_ui_emit fail "Kubernetes" "Kubelet versions" "${mismatch_count} node(s) differ from ${api_minor}" "unique kubelet minors=${kubelet_minor_count}" "./jeannie doctor-versions --verbose"
        failures=$((failures + 1))
    else
        report_ui_emit ok "Kubernetes" "Kubelet versions" "aligned to ${api_minor}"
    fi
    if [[ "${containerd_major_count}" -gt 1 ]]; then
        report_ui_emit warn "Runtime" "Containerd versions" "mixed major versions" "unique majors=${containerd_major_count}" "./jeannie doctor-versions --verbose"
    else
        report_ui_emit ok "Runtime" "Containerd versions" "single major version"
    fi

    report_ui_render "Doctor Versions"
    report_ui_cleanup
    return "${failures}"
}

doctor_versions() {
    require_debian_server "doctor-versions"
    if ! report_ui_parse_args "$@"; then
        return 1
    fi
    if [[ "${REPORT_UI_VERBOSE}" == "true" ]]; then
        doctor_versions_report
        exit "$?"
    fi
    doctor_versions_compact
    exit "$?"
}

worker_index_is_skipped() {
    local index="$1"
    local skip_indexes="$2"
    local skip_index

    skip_indexes="${skip_indexes//,/ }"
    for skip_index in ${skip_indexes}; do
        [[ -z "${skip_index}" ]] && continue
        if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then
            echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2
            exit 1
        fi
        if ((skip_index == index)); then
            return 0
        fi
    done

    return 1
}

ensure_python3() {
    if command -v python3 >/dev/null 2>&1; then
        return 0
    fi

    sudo apt-get update
    sudo apt-get install -y --no-install-recommends python3
}

ensure_go_toolchain() {
    require_debian_server "go toolchain"

    if command -v go >/dev/null 2>&1; then
        return 0
    fi

    echo "Installing Go toolchain for Jeannie core helpers..."
    sudo apt-get update
    sudo apt-get install -y --no-install-recommends golang-go

    if ! command -v go >/dev/null 2>&1; then
        echo "go is still unavailable after installing golang-go." >&2
        exit 1
    fi
}

detect_route_interface() {
    local target="$1"

    ip route get "${target}" 2>/dev/null | awk '
        {
            for (i = 1; i <= NF; i++) {
                if ($i == "dev") {
                    print $(i + 1)
                    exit
                }
            }
        }
    '
}

pimox_ssh() {
    local host="$1"
    local user="$2"
    local key_path="$3"

    shift 3
    ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@"
}

pimox_guest_ipv4() {
    local guest_json
    local host="$1"
    local user="$2"
    local key_path="$3"
    local vmid="$4"
    local ip_prefix="$5"
    local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"

    guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)"
    if [[ -z "${guest_json}" ]]; then
        return 1
    fi

    GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY'
import json
import os
import sys

prefix = sys.argv[1]
try:
    interfaces = json.loads(os.environ.get("GUEST_JSON", ""))
except Exception:
    sys.exit(1)

for iface in interfaces or []:
    for address in iface.get("ip-addresses") or []:
        if address.get("ip-address-type") != "ipv4":
            continue
        ip = address.get("ip-address", "")
        if not ip or ip.startswith(("127.", "169.254.")):
            continue
        if prefix and not ip.startswith(prefix):
            continue
        print(ip)
        sys.exit(0)
sys.exit(1)
PY
}

pimox_worker_vm_debug() {
    local host="$1"
    local user="$2"
    local key_path="$3"
    local vmid="$4"
    local qm_bin="$5"
    local script
    local encoded_script

    script="$(cat <<'EOF'
set +e
echo "-- addresses --"
ip -br addr
echo "-- routes --"
ip route
echo "-- units --"
systemctl is-active ssh sshd 2>/dev/null || true
echo "-- listeners --"
ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true
echo "-- sshd config test --"
sudo sshd -t 2>&1 || true
echo "-- recent ssh logs --"
journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
EOF
)"
    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
    pimox_ssh "${host}" "${user}" "${key_path}" "set +e
echo 'Pimox VM ${vmid} status:'
sudo '${qm_bin}' status '${vmid}'
echo 'Pimox VM ${vmid} config summary:'
sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true
echo 'Pimox VM ${vmid} guest SSH diagnostics:'
sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\" || true" >&2 || true
}

pimox_worker_guest_agent_recovery_hint() {
    local host="$1"
    local user="$2"
    local vmid="$3"
    local qm_bin="$4"

    cat >&2 <<EOF

QEMU guest agent did not become available for Pimox VM ${vmid}.
Fast checks:
  ssh ${user}@${host} 'sudo ${qm_bin} status ${vmid}; sudo ${qm_bin} config ${vmid}; sudo ${qm_bin} guest cmd ${vmid} network-get-interfaces'

Recovery options:
  1. If the clone is still booting slowly, rerun the Pimox stage with a longer agent wait:
     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=600 ./jeannie up
  2. If the same VM still reports "QEMU guest agent is not running", recreate the worker clone:
     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
  3. If a recreated worker still has no guest agent, rebuild the template and worker:
     LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
EOF
}

pimox_guest_exec_exitcode() {
    local guest_exec_json="$1"

    GUEST_EXEC_JSON="${guest_exec_json}" python3 - <<'PY'
import json
import os
import sys

try:
    document = json.loads(os.environ.get("GUEST_EXEC_JSON", ""))
except Exception:
    sys.exit(2)

try:
    print(int(document.get("exitcode", 1)))
except Exception:
    sys.exit(2)
PY
}

pimox_worker_guest_ssh_repair() {
    local host="$1"
    local user="$2"
    local key_path="$3"
    local vmid="$4"
    local qm_bin="$5"
    local script
    local encoded_script
    local repair_output
    local repair_exitcode

    script="$(cat <<'EOF'
set -eu
ssh_unit=""
if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
  ssh_unit=ssh.service
elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then
  ssh_unit=sshd.service
else
  echo "Neither ssh.service nor sshd.service exists in the guest" >&2
  exit 1
fi

sudo ssh-keygen -A
sudo install -d -m 0755 /run/sshd
sudo mkdir -p /etc/ssh/sshd_config.d
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
Port 22
ListenAddress 0.0.0.0
PubkeyAuthentication yes
PasswordAuthentication no
KbdInteractiveAuthentication no
PermitRootLogin no
UsePAM yes
SSHD_CONFIG

sudo sshd -t
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
sudo systemctl enable "$ssh_unit" >/dev/null
sudo systemctl restart "$ssh_unit"
sleep 2
sudo systemctl is-active "$ssh_unit"
if command -v ss >/dev/null 2>&1; then
  ss -ltnp 2>/dev/null || ss -ltn
  ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
fi
EOF
)"
    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
    if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
        echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2
        printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
        return 1
    fi

    printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
    if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then
        return 0
    fi
    echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2
    if [[ -n "${repair_exitcode:-}" ]]; then
        echo "Guest exit code: ${repair_exitcode}" >&2
    fi
    return 1
}

wait_for_pimox_guest_ssh() {
    local host="$1"
    local user="$2"
    local key_path="$3"
    local vmid="$4"
    local guest_user="$5"
    local guest_key_path="$6"
    local ip_prefix="$7"
    local timeout_seconds="$8"
    local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
    local deadline
    local elapsed
    local guest_ip
    local ip_filter_description
    local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
    local last_guest_ip=""
    local last_known_hosts_ip=""
    local last_ssh_output=""
    local last_host_probe_output=""
    local host_probe_failures=0
    local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-false}"
    local next_log
    local ssh_deadline=0
    local ssh_output
    local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}"
    local ssh_refused_since=0
    local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}"
    local ssh_repair_attempted=false

    ip_filter_description="matching prefix ${ip_prefix}"
    if [[ -z "${ip_prefix}" ]]; then
        ip_filter_description="that is not loopback or link-local"
    fi
    if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then
        echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
        return 1
    fi
    if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then
        echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2
        return 1
    fi
    if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
        echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
        return 1
    fi
    mkdir -p "$(dirname "${known_hosts_file}")"
    touch "${known_hosts_file}"
    chmod 0600 "${known_hosts_file}"

    deadline=$((SECONDS + timeout_seconds))
    next_log="${SECONDS}"
    while ((SECONDS < deadline)); do
        guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)"
        if [[ -n "${guest_ip}" ]]; then
            if ((ssh_deadline == 0)); then
                ssh_deadline=$((SECONDS + ssh_timeout_seconds))
            elif ((SECONDS >= ssh_deadline)); then
                break
            fi
            last_guest_ip="${guest_ip}"
            if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then
                ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true
                last_known_hosts_ip="${guest_ip}"
            fi
            if truthy "${require_host_probe}"; then
                if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then
                    host_probe_failures=0
                else
                    host_probe_failures=$((host_probe_failures + 1))
                    last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}"
                    if ((host_probe_failures >= 3)); then
                        echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2
                        echo "Latest host probe: ${last_host_probe_output}" >&2
                        echo "Fast checks:" >&2
                        echo "  ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2
                        echo "  arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2
                        echo "  Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2
                        pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
                        return 1
                    fi
                    sleep 10
                    continue
                fi
            fi
            if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then
                printf '%s\n' "${guest_ip}"
                return 0
            fi
            last_ssh_output="${ssh_output}"
            if [[ "${ssh_output}" == *"Connection refused"* ]]; then
                if ((ssh_refused_since == 0)); then
                    ssh_refused_since="${SECONDS}"
                fi
                if [[ "${ssh_repair_attempted}" == "false" ]]; then
                    echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2
                    pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true
                    ssh_repair_attempted=true
                elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then
                    echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2
                    echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2
                    pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
                    return 1
                fi
            else
                ssh_refused_since=0
            fi
        fi

        if ((SECONDS >= next_log)); then
            elapsed=$((timeout_seconds - (deadline - SECONDS)))
            if [[ -n "${last_guest_ip}" ]]; then
                echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2
                if [[ -n "${last_ssh_output}" ]]; then
                    echo "Last SSH failure: ${last_ssh_output}" >&2
                fi
            else
                echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2
            fi
            next_log=$((SECONDS + 60))
        fi
        sleep 10
    done

    if [[ -n "${last_guest_ip}" ]]; then
        echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2
        if [[ -n "${last_ssh_output}" ]]; then
            echo "Last SSH failure: ${last_ssh_output}" >&2
        fi
        if [[ -n "${last_host_probe_output}" ]]; then
            echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2
        fi
    else
        echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2
    fi
    pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"

    return 1
}

pimox_generated_mac() {
    local vmid="$1"

    printf '02:68:10:%02x:%02x:%02x\n' \
        $(((vmid >> 16) & 255)) \
        $(((vmid >> 8) & 255)) \
        $((vmid & 255))
}

pimox_worker_static_ip() {
    local index="$1"
    local static_ips="$2"
    local current_index=1
    local ip

    static_ips="${static_ips//,/ }"
    for ip in ${static_ips}; do
        if ((current_index == index)); then
            printf '%s\n' "${ip}"
            return 0
        fi
        current_index=$((current_index + 1))
    done

    return 1
}

pimox_worker_net0_config() {
    local vmid="$1"
    local bridge="$2"
    local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-deterministic}"
    local mac

    case "${mac_mode}" in
        auto)
            printf 'virtio,bridge=%s\n' "${bridge}"
            ;;
        deterministic)
            mac="$(pimox_generated_mac "${vmid}")"
            printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}"
            ;;
        *)
            echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2
            return 1
            ;;
    esac
}

validate_ipv4_cidr_or_host() {
    local value="$1"

    [[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]]
}

configure_pimox_worker_guest_network() {
    local pimox_host="$1"
    local pimox_user="$2"
    local pimox_key="$3"
    local vmid="$4"
    local qm_bin="$5"
    local worker_ip="$6"
    local gateway="$7"
    local dns_servers="$8"
    local interface_name="$9"
    local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}"
    local deadline
    local script
    local encoded_script
    local network_output
    local network_exitcode
    local worker_address
    local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}"
    local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-false}"
    local lan_probe_block=""

    if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
        echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2
        return 1
    fi
    if [[ "${worker_ip}" == */* && "${worker_ip}" != */24 ]]; then
        echo "LAB_PIMOX_WORKER_STATIC_IPS currently supports host IPv4 values or /24 CIDRs, got '${worker_ip}'." >&2
        return 1
    fi
    if [[ "${worker_ip}" != */* ]]; then
        worker_ip="${worker_ip}/24"
    fi
    worker_address="${worker_ip%%/*}"
    if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
        echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2
        return 1
    fi
    if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then
        echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2
        return 1
    fi
    if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
        echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2
        return 1
    fi
    if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
        echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2
        return 1
    fi
    if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then
        echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2
        return 1
    fi

    if truthy "${require_lan_probe}"; then
        lan_probe_block="$(cat <<EOF
echo "guest-check: ping gateway ${gateway}"
ping -c 2 -W 2 '${gateway}'
EOF
)"
        if [[ -n "${lan_probe_host}" ]]; then
            lan_probe_block+="
echo \"guest-check: ping Debian/probe host ${lan_probe_host}\"
ping -c 2 -W 2 '${lan_probe_host}'"
        fi
    else
        lan_probe_block="echo 'guest-check: LAN probe disabled by LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false'"
    fi

    script="$(cat <<EOF
set -eu
cat >/tmp/homelab-interfaces <<'INTERFACES'
auto lo
iface lo inet loopback

auto ${interface_name}
iface ${interface_name} inet static
  address ${worker_address}
  netmask 255.255.255.0
  gateway ${gateway}
  dns-nameservers ${dns_servers}
INTERFACES
sudo cp /tmp/homelab-interfaces /etc/network/interfaces
sudo mkdir -p /etc/cloud/cloud.cfg.d
printf '%s\n' 'network: {config: disabled}' | sudo tee /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg >/dev/null
for unit in systemd-networkd systemd-networkd.socket NetworkManager dhcpcd; do
  sudo systemctl disable --now "\$unit" >/dev/null 2>&1 || true
done
sudo pkill -x dhclient >/dev/null 2>&1 || true
sudo pkill -x dhcpcd >/dev/null 2>&1 || true
sudo systemctl enable networking >/dev/null 2>&1 || true
sudo ifdown --force ${interface_name} 2>/dev/null || true
sudo ip addr flush dev ${interface_name} scope global || true
sudo ip route flush dev ${interface_name} || true
sudo ip link set ${interface_name} up
sudo ip addr add ${worker_ip} dev ${interface_name}
sudo ip route replace default via ${gateway} dev ${interface_name} src ${worker_address}
printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null
sudo ifup --force ${interface_name} 2>/dev/null || true
sudo ssh-keygen -A
sudo install -d -m 0755 /run/sshd
sudo mkdir -p /etc/ssh/sshd_config.d
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
Port 22
ListenAddress 0.0.0.0
PubkeyAuthentication yes
PasswordAuthentication no
KbdInteractiveAuthentication no
PermitRootLogin no
UsePAM yes
SSHD_CONFIG
sudo sshd -t
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
ssh_unit=ssh.service
if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
  ssh_unit=sshd.service
fi
sudo systemctl enable "\$ssh_unit" >/dev/null
sudo systemctl restart "\$ssh_unit"
ip -br addr show ${interface_name}
ip route
ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
${lan_probe_block}
echo 'guest-check: ssh service'
sudo systemctl is-active "\$ssh_unit"
if command -v ss >/dev/null 2>&1; then
  ss -ltnp 2>/dev/null || ss -ltn
  ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
fi
EOF
)"
    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"

    deadline=$((SECONDS + timeout_seconds))
    while ((SECONDS < deadline)); do
        if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
            printf '%s\n' "${network_output}" | sed 's/^/    guest-network: /'
            if network_exitcode="$(pimox_guest_exec_exitcode "${network_output}")" && [[ "${network_exitcode}" == "0" ]]; then
                return 0
            fi
            echo "Static guest network validation failed inside Pimox VM ${vmid}." >&2
            if [[ -n "${network_exitcode:-}" ]]; then
                echo "Guest exit code: ${network_exitcode}" >&2
            fi
            return 1
        fi
        sleep 5
    done

    echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2
    if [[ -n "${network_output:-}" ]]; then
        printf '%s\n' "${network_output}" | sed 's/^/    /' >&2
    fi
    pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}"
    return 1
}

configure_pimox_worker_forwarding() {
    local pimox_host="$1"
    local pimox_user="$2"
    local pimox_key="$3"
    local worker_ip="$4"
    local enabled="${LAB_PIMOX_WORKER_CONFIGURE_FORWARDING:-true}"
    local backup_dir="${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}"
    local worker_address

    if disabled_value "${enabled}"; then
        return 0
    fi
    if ! truthy "${enabled}"; then
        echo "LAB_PIMOX_WORKER_CONFIGURE_FORWARDING must be true or false." >&2
        return 1
    fi
    if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
        echo "Invalid Pimox worker forwarding IP '${worker_ip}'." >&2
        return 1
    fi
    worker_address="${worker_ip%%/*}"

    pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\"
if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then
  iptables_bin=/usr/sbin/iptables
fi
if [ -z \"\$iptables_bin\" ]; then
  if command -v apt-get >/dev/null 2>&1; then
    echo 'iptables not found on Pimox host; installing iptables for worker forwarding rules.'
    sudo apt-get update
    sudo apt-get install -y --no-install-recommends iptables
  fi
fi
iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\"
if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then
  iptables_bin=/usr/sbin/iptables
fi
if [ -z \"\$iptables_bin\" ]; then
  echo 'iptables not found on Pimox host and could not be installed; cannot configure worker forwarding rules.' >&2
  exit 1
fi
iptables_wait_seconds='${LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS:-60}'
case \"\$iptables_wait_seconds\" in
  ''|*[!0-9]*)
    echo \"LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS must be a non-negative integer, got '\$iptables_wait_seconds'.\" >&2
    exit 1
    ;;
esac
iptables_wait_args=
if \"\$iptables_bin\" -h 2>&1 | grep -q -- ' -w'; then
  iptables_wait_args=\"-w \$iptables_wait_seconds\"
else
  echo 'Pimox host iptables does not support -w; proceeding without xtables lock wait.'
fi
iptables_cmd() {
  sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\"
}
iptables_save_bin=\"\$(command -v iptables-save 2>/dev/null || true)\"
if [ -z \"\$iptables_save_bin\" ] && [ -x /usr/sbin/iptables-save ]; then
  iptables_save_bin=/usr/sbin/iptables-save
fi
backup_dir='${backup_dir}'
case \"\$backup_dir\" in
  '~'|'~/') backup_dir=\"\$HOME\" ;;
  '~/'*) backup_dir=\"\$HOME/\${backup_dir#~/}\" ;;
esac
mkdir -p \"\$backup_dir\"
backup_file=\"\$backup_dir/iptables-before-jeannie-workers-\$(date +%Y%m%d-%H%M%S).rules\"
if [ -n \"\$iptables_save_bin\" ]; then
  sudo \"\$iptables_save_bin\" >\"\$backup_file\"
  echo \"Saved Pimox iptables backup: \$backup_file\"
fi
sudo mkdir -p /etc/sysctl.d
printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-pimox-worker-forwarding.conf >/dev/null
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
chain=FORWARD
if iptables_cmd -S DOCKER-USER >/dev/null 2>&1; then
  chain=DOCKER-USER
fi
if ! iptables_cmd -C \"\$chain\" -s '${worker_address}/32' -j ACCEPT 2>/dev/null; then
  iptables_cmd -I \"\$chain\" 1 -s '${worker_address}/32' -j ACCEPT
fi
if ! iptables_cmd -C \"\$chain\" -d '${worker_address}/32' -j ACCEPT 2>/dev/null; then
  iptables_cmd -I \"\$chain\" 1 -d '${worker_address}/32' -j ACCEPT
fi
echo \"Pimox worker forwarding rules active in \$chain for ${worker_address}\""
}

iptables_cmd() {
  sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\"
}

cpuset_cpu_count() {
    local cpuset="$1"
    local count=0
    local part
    local start
    local end
    local -a parts

    IFS=',' read -r -a parts <<<"${cpuset}"
    for part in "${parts[@]}"; do
        if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
            start="${BASH_REMATCH[1]}"
            end="${BASH_REMATCH[2]}"
            if ((end < start)); then
                return 1
            fi
            count=$((count + end - start + 1))
        elif [[ "${part}" =~ ^[0-9]+$ ]]; then
            count=$((count + 1))
        else
            return 1
        fi
    done

    printf '%s\n' "${count}"
}

pimox_worker_cpu_affinity() {
    local index="$1"
    local affinities="$2"
    local worker_cores="$3"
    local affinity
    local affinity_index=1
    local cpu_count

    for affinity in ${affinities}; do
        if ((affinity_index == index)); then
            if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then
                echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2
                exit 1
            fi
            if ((cpu_count != worker_cores)); then
                echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2
                exit 1
            fi
            printf '%s\n' "${affinity}"
            return 0
        fi
        affinity_index=$((affinity_index + 1))
    done

    echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2
    exit 1
}

pimox_start_vm_with_cpuset() {
    local host="$1"
    local user="$2"
    local key_path="$3"
    local qm_bin="$4"
    local vmid="$5"
    local cpuset="$6"

    if [[ -z "${cpuset}" ]]; then
        pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'"
        return
    fi
    if ! cpuset_cpu_count "${cpuset}" >/dev/null; then
        echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2
        exit 1
    fi

    # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated
    # command under taskset so PSCI brings secondary guest CPUs online reliably.
    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
vmid='${vmid}'
qm_bin='${qm_bin}'
cpuset='${cpuset}'

if ! command -v taskset >/dev/null 2>&1; then
  echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2
  exit 1
fi
if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then
  echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2
  exit 1
fi

start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\")
cleanup() {
  rm -f \"\$start_script\"
}
trap cleanup 0

sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\"
if ! { grep -Fq -- '/usr/bin/kvm' \"\$start_script\" || grep -Fq -- 'qemu-system-aarch64' \"\$start_script\"; } ||
  ! grep -Fq -- \"-id \$vmid\" \"\$start_script\"; then
  echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2
  sed 's/^/    showcmd: /' \"\$start_script\" >&2
  exit 1
fi
smp_count=\$(grep -c '^[[:space:]]*-smp[[:space:]]' \"\$start_script\" || true)
if [ \"\$smp_count\" -ne 1 ]; then
  echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2
  exit 1
fi

sudo taskset -c \"\$cpuset\" bash \"\$start_script\""
}

pimox_shutdown_vm_gracefully() {
    local host="$1"
    local user="$2"
    local key_path="$3"
    local qm_bin="$4"
    local vmid="$5"

    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
elapsed=0
while [ \"\$elapsed\" -lt 300 ]; do
  if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
    exit 0
  fi
  sleep 5
  elapsed=\$((elapsed + 5))
done
echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2
exit 1"
}

pimox_guest_cpu_count() {
    local guest_ip="$1"
    local guest_user="$2"
    local guest_key_path="$3"
    local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"

    ssh -i "${guest_key_path}" \
        -o BatchMode=yes \
        -o ConnectTimeout=10 \
        -o IdentitiesOnly=yes \
        -o StrictHostKeyChecking=accept-new \
        -o UserKnownHostsFile="${known_hosts_file}" \
        "${guest_user}@${guest_ip}" \
        'nproc --all'
}

ensure_pimox_worker_node() {
    local index="$1"
    local spec_file="$2"
    local pimox_host="$3"
    local pimox_user="$4"
    local pimox_key="$5"
    local template_vmid="$6"
    local bridge="$7"
    local worker_base_vmid="$8"
    local worker_name_prefix="$9"
    local worker_node_prefix="${10}"
    local worker_key_prefix="${11}"
    local worker_cores="${12}"
    local worker_memory="${13}"
    local worker_user="${14}"
    local worker_key_path="${15}"
    local ip_prefix="${16}"
    local timeout_seconds="${17}"
    local qm_bin="${18}"
    local worker_storage="${19}"
    local worker_replace_existing="${20}"
    local worker_cpu_affinity="${21}"
    local worker_kvm_cpuset="${22}"
    local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
    local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
    local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}"
    local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}"
    local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}"
    local padded
    local vmid
    local worker_key
    local worker_name
    local node_name
    local net0_config
    local guest_ip
    local guest_cpu_count
    local static_ip

    printf -v padded '%02d' "${index}"
    vmid=$((worker_base_vmid + index - 1))
    worker_key="${worker_key_prefix}${padded}"
    worker_name="${worker_name_prefix}-${padded}"
    node_name="${worker_node_prefix}-${padded}"
    if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
        exit 1
    fi
    if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
        if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
            echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2
            exit 1
        fi
        if ! truthy "${worker_replace_existing}" &&
            ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then
            echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}."
            worker_replace_existing=true
        fi
        if truthy "${worker_replace_existing}"; then
            echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..."
            pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true
elapsed=0
while [ \"\$elapsed\" -lt 300 ]; do
  if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
    break
  fi
  sleep 5
  elapsed=\$((elapsed + 5))
done
sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
        else
            pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
was_running=false
had_legacy_args=false
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then
  was_running=true
fi
if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
  had_legacy_args=true
fi
sudo '${qm_bin}' set '${vmid}' \
  --agent enabled=1 \
  --bios ovmf \
  --boot 'order=scsi0;net0' \
  --cores '${worker_cores}' \
  --memory '${worker_memory}' \
  --net0 '${net0_config}' \
  --numa 0 \
  --ostype l26 \
  --scsihw virtio-scsi-pci \
  --sockets 1 \
  --vga virtio
if [ \"\$had_legacy_args\" = true ]; then
  sudo '${qm_bin}' set '${vmid}' --delete args
fi
if [ -n '${worker_cpu_affinity}' ]; then
  affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
    case \"\$affinity_output\" in
      *'Unknown option: affinity'*)
        echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
        ;;
      *)
        printf '%s\n' \"\$affinity_output\" >&2
        exit 1
        ;;
    esac
  }
fi
if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then
  echo 'Restarting VM ${vmid} to apply removal of legacy qm args.'
  sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
  elapsed=0
  while [ \"\$elapsed\" -lt 300 ]; do
    if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
      break
    fi
    sleep 5
    elapsed=\$((elapsed + 5))
  done
  if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
    echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2
    exit 1
  fi
fi"
        fi
    fi

    if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
        pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! ip link show '${bridge}' >/dev/null 2>&1; then
  echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2
  exit 1
fi
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
  pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
  echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2
  exit 1
fi
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
  echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2
  exit 1
fi
sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}'
sudo '${qm_bin}' set '${vmid}' \
  --agent enabled=1 \
  --bios ovmf \
  --boot 'order=scsi0;net0' \
  --cores '${worker_cores}' \
  --memory '${worker_memory}' \
  --net0 '${net0_config}' \
  --numa 0 \
  --ostype l26 \
  --scsihw virtio-scsi-pci \
  --sockets 1 \
  --vga virtio
if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
  sudo '${qm_bin}' set '${vmid}' --delete args
fi
if [ -n '${worker_cpu_affinity}' ]; then
  affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
    case \"\$affinity_output\" in
      *'Unknown option: affinity'*)
        echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
        ;;
      *)
        printf '%s\n' \"\$affinity_output\" >&2
        exit 1
        ;;
    esac
}
fi
sudo '${qm_bin}' set '${vmid}' --onboot 1"
    fi

    if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then
        pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
    elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then
        echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2
        exit 1
    fi

    case "${worker_network_mode}" in
        static)
            if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then
                echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2
                exit 1
            fi
            echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..."
            configure_pimox_worker_forwarding \
                "${pimox_host}" \
                "${pimox_user}" \
                "${pimox_key}" \
                "${static_ip}"
            if ! configure_pimox_worker_guest_network \
                "${pimox_host}" \
                "${pimox_user}" \
                "${pimox_key}" \
                "${vmid}" \
                "${qm_bin}" \
                "${static_ip}" \
                "${worker_gateway}" \
                "${worker_dns_servers}" \
                "${worker_interface}"; then
                pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}"
                exit 1
            fi
            ;;
        dhcp)
            ;;
        *)
            echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2
            exit 1
            ;;
    esac

    if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
        echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2
        exit 1
    fi

    guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
    if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
        echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2
        pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}"
        pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
        if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
            echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2
            exit 1
        fi
        guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
    fi
    if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
        echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2
        echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2
        exit 1
    fi

    printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
}

write_cluster_worker_var_file() {
    local var_file="$1"
    local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}'
    local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}"
    local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}"
    shift

    LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \
    LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \
    LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \
    LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \
    LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \
    LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \
    LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \
    LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \
        python3 - "${var_file}" "$@" <<'PY'
import json
import os
from pathlib import Path
import sys

var_file = sys.argv[1]
spec_files = sys.argv[2:]
nodes = {}
node_labels = {}

try:
    raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"])
    manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"])
    pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"])
except json.JSONDecodeError as exc:
    raise SystemExit(f"Invalid node label JSON: {exc}") from exc

if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}:
    nodes["raspberrypi"] = {
        "host": os.environ["LAB_RASPBERRY_HOST"],
        "user": os.environ["LAB_RASPBERRY_USER"],
        "node_name": os.environ["LAB_RASPBERRY_NODE_NAME"],
        "ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"],
    }
    node_labels["raspberrypi"] = raspberry_labels

for spec_file in spec_files:
    path = Path(spec_file)
    if not path.exists() or path.stat().st_size == 0:
        continue
    labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels
    with path.open(encoding="utf-8") as handle:
        for line_number, line in enumerate(handle, start=1):
            line = line.rstrip("\n")
            if not line or line.startswith("#"):
                continue
            try:
                key, host, user, node_name, ssh_key_path = line.split("\t")
            except ValueError as exc:
                raise SystemExit(
                    f"{spec_file}:{line_number}: expected tab-separated "
                    "key, host, user, node_name, ssh_key_path"
                ) from exc
            nodes[key] = {
                "host": host,
                "user": user,
                "node_name": node_name,
                "ssh_key_path": ssh_key_path,
            }
            node_labels[key] = labels

with open(var_file, "w", encoding="utf-8") as handle:
    json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2)
    handle.write("\n")
PY
}

prepare_cluster_worker_var_file() {
    local include_raspberry_default="$1"
    local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv"
    local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
    local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
    local include_manual="${LAB_INCLUDE_MANUAL_WORKERS:-false}"
    local -a spec_files=()

    export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}"

    mkdir -p "${REPO_ROOT}/.lab"
    ensure_static_pimox_worker_spec_file "${pimox_spec_file}"
    if truthy "${include_manual}"; then
        spec_files+=("${manual_spec_file}")
    elif ! disabled_value "${include_manual}"; then
        echo "LAB_INCLUDE_MANUAL_WORKERS must be true or false." >&2
        return 1
    fi
    spec_files+=("${pimox_spec_file}")
    write_cluster_worker_var_file "${var_file}" "${spec_files[@]}"
    export LAB_CLUSTER_VAR_FILE="${var_file}"
}

ensure_static_pimox_worker_spec_file() {
    local spec_file="$1"
    local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
    local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
    local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
    local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
    local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
    local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
    local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
    local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
    local index
    local padded
    local worker_key
    local node_name
    local static_ip

    if [[ -s "${spec_file}" || "${worker_network_mode}" != "static" ]]; then
        return 0
    fi
    if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
        echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2
        return 1
    fi
    if ((worker_count == 0)); then
        return 0
    fi

    mkdir -p "$(dirname "${spec_file}")"
    : >"${spec_file}"
    for ((index = 1; index <= worker_count; index++)); do
        if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
            continue
        fi
        if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then
            echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2
            return 1
        fi
        printf -v padded '%02d' "${index}"
        worker_key="${worker_key_prefix}${padded}"
        node_name="${worker_node_prefix}-${padded}"
        printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${static_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
    done
}

cluster_worker_var_file_has_workers() {
    local var_file="$1"

    [[ -s "${var_file}" ]] || return 1
    python3 - "${var_file}" <<'PY'
import json
import sys

with open(sys.argv[1], encoding="utf-8") as handle:
    document = json.load(handle)
sys.exit(0 if document.get("worker_nodes") else 1)
PY
}

preflight_check() {
    local description="$1"
    local output
    shift

    printf '%s %s... ' '-->' "${description}"
    if output="$("$@" 2>&1)"; then
        printf 'ok\n'
        return 0
    fi

    printf 'failed\n'
    if [[ -n "${output}" ]]; then
        printf '%s\n' "${output}" | sed 's/^/    /' >&2
    fi
    return 1
}

preflight_warn() {
    local description="$1"
    local output
    shift

    printf '%s %s... ' '-->' "${description}"
    if output="$("$@" 2>&1)"; then
        printf 'ok\n'
        return 0
    fi

    printf 'warning\n'
    if [[ -n "${output}" ]]; then
        printf '%s\n' "${output}" | sed 's/^/    /' >&2
    fi
    return 0
}

check_gitea_reachable() {
    local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}"
    local output

    if output="$(curl -fsS --max-time 10 "${gitea_url}" 2>&1 >/dev/null)"; then
        return 0
    fi

    printf '%s\n' "${output}" >&2
    cat >&2 <<'EOF'
Gitea local HTTP failed. Run ./jeannie doctor-gitea --verbose.
If the local Gitea container is down or unhealthy, run ./jeannie deploy-gitea.
EOF
    return 1
}

check_debian_docker_root() {
    local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
    local actual_root

    if ! command -v docker >/dev/null 2>&1; then
        echo "docker command was not found on Debian host" >&2
        return 1
    fi
    actual_root="$(sudo docker info --format '{{.DockerRootDir}}')"
    if [[ "${actual_root}" != "${expected_root}" ]]; then
        echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2
        return 1
    fi
}

fix_debian_docker_root() {
    local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
    local daemon_file="/etc/docker/daemon.json"
    local current_root=""

    require_debian_server "fix-debian-docker-root"

    if ! command -v docker >/dev/null 2>&1; then
        echo "docker command was not found on Debian host" >&2
        exit 1
    fi

    current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
    if [[ "${current_root}" == "${expected_root}" ]]; then
        echo "Debian Docker root is already ${expected_root}."
        return 0
    fi

    echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..."
    sudo mkdir -p "${expected_root}" /etc/docker

    echo "Stopping Docker services..."
    sudo systemctl stop docker 2>/dev/null || true
    sudo systemctl stop containerd 2>/dev/null || true

    if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then
        echo "Copying existing Docker data to ${expected_root}..."
        sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/"
    fi

    sudo python3 - "${daemon_file}" "${expected_root}" <<'PY'
import json
import os
import sys

path, expected_root = sys.argv[1:3]
document = {}
if os.path.exists(path) and os.path.getsize(path) > 0:
    with open(path, encoding="utf-8") as handle:
        document = json.load(handle)
document["data-root"] = expected_root
with open(path, "w", encoding="utf-8") as handle:
    json.dump(document, handle, indent=2, sort_keys=True)
    handle.write("\n")
PY

    echo "Starting Docker services..."
    sudo systemctl start containerd
    sudo systemctl start docker

    check_debian_docker_root
    echo "Debian Docker root is now ${expected_root}."
}

check_debian_tailscale_ip() {
    local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}"

    if [[ -z "${expected_ip}" ]]; then
        return 0
    fi
    if ! command -v tailscale >/dev/null 2>&1; then
        echo "tailscale command was not found on Debian host" >&2
        return 1
    fi
    if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then
        echo "Debian Tailscale IP does not include ${expected_ip}" >&2
        tailscale ip -4 >&2 || true
        return 1
    fi
}

check_edge_ssh() {
    local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
    local edge_user="${LAB_EDGE_USER:-ubuntu}"

    ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true"
}

check_rpi_ssh() {
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"

    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true"
}

check_rpi_docker_root_state() {
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
    local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"

    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
if [ \"\$actual_root\" = '${nvme_root}' ]; then
  mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; }
  exit 0
fi
if [ \"\$actual_root\" = '${fallback_root}' ]; then
  echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2
  exit 0
fi
echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2
exit 1"
}

check_rpi_tailscale_ip() {
    local expected_ip="${LAB_RPI_TAILSCALE_IP:-}"
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"

    if [[ -z "${expected_ip}" ]]; then
        return 0
    fi
    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'"
}

check_pimox_storage() {
    local mode="${LAB_PIMOX_PIPELINE:-true}"
    local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
    local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
    local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
    local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"

    if disabled_value "${mode}"; then
        return 0
    fi

    pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
  pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
  echo 'pvesm was not found' >&2
  exit 1
fi
sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'"
}

check_local_disk_free_gib() {
    local path="$1"
    local min_gib="$2"
    local available_kib
    local min_kib

    available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')"
    min_kib=$((min_gib * 1024 * 1024))
    if ((available_kib < min_kib)); then
        echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2
        return 1
    fi
}

check_systemd_active() {
    local unit="$1"

    systemctl is-active --quiet "${unit}"
}

check_rpi_docker_writable() {
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"

    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
sudo mkdir -p \"\$root/.homelab-check\"
sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\"
sudo rm -f \"\$root/.homelab-check/write-test\""
}

check_edge_disk_free() {
    local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
    local edge_user="${LAB_EDGE_USER:-ubuntu}"
    local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}"

    ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu
available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\"
min_kib=$((min_gib * 1024 * 1024))
if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then
  echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2
  exit 1
fi"
}

check_pihole_dns_query() {
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}"
    local output

    if command -v dig >/dev/null 2>&1; then
        output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || {
            printf '%s\n' "${output}" >&2
            echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2
            return 1
        }
        if [[ -z "${output}" ]]; then
            echo "Pi-hole returned no A records for ${query_name}" >&2
            return 1
        fi
        return 0
    fi
    if command -v nslookup >/dev/null 2>&1; then
        nslookup "${query_name}" "${rpi_host}" >/dev/null
        return $?
    fi

    echo "dig or nslookup is required to test Pi-hole DNS directly." >&2
    return 1
}

homelab_preflight() {
    local phase="${1:-full}"
    local failures=0

    require_debian_server "preflight"

    if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then
        echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}."
        return 0
    fi

    case "${phase}" in
        early | full)
            ;;
        *)
            echo "Unsupported preflight phase '${phase}'." >&2
            exit 1
            ;;
    esac

    echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..."

    preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1))
    preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1))
    preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1))
    if [[ "${phase}" == "full" ]]; then
        preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1))
        preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1))
    fi
    preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip
    preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
    preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" check_edge_ssh || failures=$((failures + 1))

    if ((failures > 0)); then
        echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2
        exit 1
    fi

    echo "${phase^} preflight checks passed."
}

doctor_preapply() {
    local failures=0

    require_debian_server "doctor-preapply"

    echo "Running pre-apply readiness checks..."
    preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1))
    preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1))
    preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1))
    preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1))
    preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1))
    preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
    preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1))
    preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1))

    if ((failures > 0)); then
        echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2
        exit 1
    fi

    echo "Pre-apply doctor checks passed."
}

inventory_check() {
    "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
}

run_pimox_pipeline() {
    local mode="${LAB_PIMOX_PIPELINE:-true}"
    local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
    local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
    local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
    local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
    local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}"
    local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}"
    local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}"
    local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}"
    local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}"
    local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}"
    local provisioning_interface
    local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
    local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
    local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}"
    local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
    local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
    local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
    local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}"
    local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}"
    local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}"
    local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}"
    local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}"
    local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}"
    local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}"
    local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
    local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
    local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
    local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}"
    local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}"
    local worker_wait_timeout_seconds="${LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS:-600}"
    local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
    local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
    local index
    local readiness_output
    local readiness_status
    local worker_cpu_affinity
    local cpuset_cpu_total
    local cpuset_name
    local cpuset_value
    local required_cores

    if disabled_value "${mode}"; then
        return 0
    fi

    if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then
        mode="true"
    fi

    if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
        echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2
        exit 1
    fi
    if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then
        echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2
        exit 1
    fi
    if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then
        echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2
        exit 1
    fi
    if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then
        echo "Pimox template and worker CPU and memory values must be greater than zero." >&2
        exit 1
    fi
    for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do
        cpuset_value="${!cpuset_name}"
        if [[ -z "${cpuset_value}" ]]; then
            continue
        fi
        if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then
            echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2
            exit 1
        fi
        required_cores="${template_cores}"
        if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then
            required_cores="${worker_cores}"
        fi
        if ((cpuset_cpu_total < required_cores)); then
            echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2
            exit 1
        fi
    done
    if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then
        echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2
        exit 1
    fi
    if ! [[ "${worker_wait_timeout_seconds}" =~ ^[0-9]+$ && "${worker_wait_timeout_seconds}" -gt 0 ]]; then
        echo "LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS must be a positive integer." >&2
        exit 1
    fi
    if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then
        echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2
        exit 1
    fi
    if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
        echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2
        exit 1
    fi
    if [[ "${worker_storage}" == "local" ]]; then
        echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2
        exit 1
    fi

    set +e
    readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then
  echo 'qm was not found in PATH and ${qm_bin} is not executable'
  exit 1
fi
if ! ip link show '${bridge}' >/dev/null 2>&1; then
  echo 'bridge ${bridge} was not found'
  exit 1
fi
if ! sudo -n true >/dev/null 2>&1; then
  echo 'passwordless sudo is not available for ${pimox_user}'
  exit 1
fi
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
  pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
  echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}'
  exit 1
fi
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then
  echo 'Pimox worker storage ${worker_storage} was not found or is not active'
  exit 1
fi" 2>&1)"
    readiness_status=$?
    set -e
    if ((readiness_status != 0)); then
        if [[ "${mode}" == "auto" ]]; then
            echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready."
            return 0
        fi
        echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2
        exit 1
    fi

    ensure_python3
    provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}"
    if [[ -z "${provisioning_interface}" ]]; then
        echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2
        exit 1
    fi

    export TF_VAR_provisioning_interface="${provisioning_interface}"
    export TF_VAR_pimox_host="${pimox_host}"
    export TF_VAR_pimox_user="${pimox_user}"
    export TF_VAR_pimox_ssh_key_path="${pimox_key}"
    export TF_VAR_pimox_qm_bin="${qm_bin}"
    export TF_VAR_pimox_template_bridge="${bridge}"
    export TF_VAR_pimox_template_vmid="${template_vmid}"
    export TF_VAR_pimox_template_name="${template_name}"
    export TF_VAR_pimox_template_cores="${template_cores}"
    export TF_VAR_pimox_template_memory="${template_memory}"
    export TF_VAR_pimox_template_replace_existing="${template_replace_existing}"
    export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}"
    export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}"
    export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}"
    export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}"
    export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}"
    export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}"

    echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..."
    run_tofu_stack "bootstrap/provisioning"

    if ((worker_count == 0)); then
        return 0
    fi

    if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then
        echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2
        exit 1
    fi
    pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1"
    echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage."

    mkdir -p "${REPO_ROOT}/.lab"
    : >"${spec_file}"
    for ((index = 1; index <= worker_count; index++)); do
        if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
            echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
            continue
        fi

        worker_cpu_affinity=""
        if [[ -n "${worker_cpu_affinities}" ]]; then
            worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")"
        fi
        ensure_pimox_worker_node \
            "${index}" \
            "${spec_file}" \
            "${pimox_host}" \
            "${pimox_user}" \
            "${pimox_key}" \
            "${template_vmid}" \
            "${bridge}" \
            "${worker_base_vmid}" \
            "${worker_name_prefix}" \
            "${worker_node_prefix}" \
            "${worker_key_prefix}" \
            "${worker_cores}" \
            "${worker_memory}" \
            "${worker_user}" \
            "${worker_key_path}" \
            "${ip_prefix}" \
            "${worker_wait_timeout_seconds}" \
            "${qm_bin}" \
            "${worker_storage}" \
            "${worker_replace_existing}" \
            "${worker_cpu_affinity}" \
            "${worker_kvm_cpuset}"
    done

    write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}"
    export LAB_CLUSTER_VAR_FILE="${var_file}"
}

run_openwrt_pipeline() {
    local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}"
    local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
    local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
    local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
    local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
    local vmid="${LAB_OPENWRT_VMID:-9100}"
    local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}"
    local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}"
    local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}"
    local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}"
    local cores="${LAB_OPENWRT_CORES:-2}"
    local memory="${LAB_OPENWRT_MEMORY:-512}"
    local version="${LAB_OPENWRT_VERSION:-24.10.6}"
    local image_url="${LAB_OPENWRT_IMAGE_URL:-}"
    local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}"
    local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}"
    local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}"
    local start_vm="${LAB_OPENWRT_START:-false}"
    local onboot="${LAB_OPENWRT_ONBOOT:-false}"
    local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}"
    local root_key_b64=""
    local lan_dhcp_ignore="1"
    local start_vm_flag="false"
    local onboot_flag="0"

    if disabled_value "${mode}"; then
        return 0
    fi
    if ! truthy "${mode}"; then
        echo "LAB_OPENWRT_VM must be true or false." >&2
        exit 1
    fi

    if [[ -z "${image_url}" ]]; then
        image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz"
    fi

    if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then
        echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2
        exit 1
    fi
    for value_name in storage wan_bridge lan_bridge vm_name; do
        local value="${!value_name}"
        if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
            echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2
            exit 1
        fi
    done
    if [[ "${storage}" == "local" ]]; then
        echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2
        exit 1
    fi
    if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then
        echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2
        exit 1
    fi
    if truthy "${lan_dhcp_enabled}"; then
        lan_dhcp_ignore="0"
    fi
    if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then
        echo "LAB_OPENWRT_START must be true or false." >&2
        exit 1
    fi
    if truthy "${start_vm}"; then
        start_vm_flag="true"
    fi
    if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then
        echo "LAB_OPENWRT_ONBOOT must be true or false." >&2
        exit 1
    fi
    if truthy "${onboot}"; then
        onboot_flag="1"
    fi
    if [[ -r "${root_key_path}" ]]; then
        root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')"
    fi

    echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..."
    pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <<EOF
set -euo pipefail

vmid="${vmid}"
vm_name="${vm_name}"
storage="${storage}"
wan_bridge="${wan_bridge}"
lan_bridge="${lan_bridge}"
cores="${cores}"
memory="${memory}"
image_url="${image_url}"
lan_ip="${lan_ip}"
lan_netmask="${lan_netmask}"
lan_dhcp_ignore="${lan_dhcp_ignore}"
start_vm="${start_vm_flag}"
onboot="${onboot_flag}"
root_key_b64="${root_key_b64}"
qm_cmd="${qm_bin}"

if [ ! -x "\$qm_cmd" ]; then
  qm_cmd="\$(command -v qm 2>/dev/null || true)"
fi
if [ -z "\$qm_cmd" ]; then
  echo "qm is not installed on this Pimox host" >&2
  exit 1
fi

pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)"
if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then
  pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z "\$pvesm_cmd" ]; then
  echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2
  exit 1
fi

if ! sudo -n true >/dev/null 2>&1; then
  echo "passwordless sudo is required for OpenWrt VM automation" >&2
  exit 1
fi
if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then
  echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2
  exit 1
fi
if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then
  echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2
  exit 1
fi
if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
  echo "Pimox storage \$storage was not found." >&2
  exit 1
fi

if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then
  if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then
    echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2
    exit 1
  fi
  sudo "\$qm_cmd" set "\$vmid" \\
    --net0 "virtio,bridge=\$wan_bridge" \\
    --net1 "virtio,bridge=\$lan_bridge" \\
    --cores "\$cores" \\
    --memory "\$memory" \\
    --onboot "\$onboot"
  if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then
    sudo "\$qm_cmd" start "\$vmid"
  fi
  exit 0
fi

for required_cmd in curl gzip losetup mount umount awk sed; do
  if ! command -v "\$required_cmd" >/dev/null 2>&1; then
    echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2
    exit 1
  fi
done

tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)"
mnt_dir="\$tmp_dir/root"
loopdev=""
cleanup() {
  if mountpoint -q "\$mnt_dir" 2>/dev/null; then
    sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true
  fi
  if [ -n "\$loopdev" ]; then
    sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true
  fi
  rm -rf "\$tmp_dir"
}
trap cleanup EXIT

mkdir -p "\$mnt_dir"
curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz"
gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img"

loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")"
root_part="\${loopdev}p2"
if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then
  root_part="\${loopdev}p2"
fi
if [ ! -b "\$root_part" ]; then
  echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2
  exit 1
fi

sudo mount "\$root_part" "\$mnt_dir"
sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh"

cat >"\$tmp_dir/network" <<NETWORK
config interface 'loopback'
        option device 'lo'
        option proto 'static'
        option ipaddr '127.0.0.1'
        option netmask '255.0.0.0'

config globals 'globals'
        option ula_prefix 'fd00:68:50::/48'

config interface 'wan'
        option device 'eth0'
        option proto 'dhcp'

config interface 'lan'
        option device 'eth1'
        option proto 'static'
        option ipaddr '\$lan_ip'
        option netmask '\$lan_netmask'
        option ip6assign '60'
NETWORK

cat >"\$tmp_dir/dhcp" <<DHCP
config dnsmasq
        option domainneeded '1'
        option boguspriv '1'
        option filterwin2k '0'
        option localise_queries '1'
        option rebind_protection '1'
        option rebind_localhost '1'
        option local '/lan/'
        option domain 'lan'
        option expandhosts '1'
        option cachesize '1000'
        option authoritative '1'
        option readethers '1'
        option leasefile '/tmp/dhcp.leases'
        option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'

config dhcp 'lan'
        option interface 'lan'
        option start '100'
        option limit '150'
        option leasetime '12h'
        option ignore '\$lan_dhcp_ignore'

config dhcp 'wan'
        option interface 'wan'
        option ignore '1'
DHCP

cat >"\$tmp_dir/firewall" <<'FIREWALL'
config defaults
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option synflood_protect '1'

config zone
        option name 'lan'
        list network 'lan'
        option input 'ACCEPT'
        option output 'ACCEPT'
        option forward 'ACCEPT'

config zone
        option name 'wan'
        list network 'wan'
        option input 'REJECT'
        option output 'ACCEPT'
        option forward 'REJECT'
        option masq '1'
        option mtu_fix '1'

config forwarding
        option src 'lan'
        option dest 'wan'

config rule
        option name 'Allow-DHCP-Renew'
        option src 'wan'
        option proto 'udp'
        option dest_port '68'
        option target 'ACCEPT'
        option family 'ipv4'

config rule
        option name 'Allow-Ping'
        option src 'wan'
        option proto 'icmp'
        option icmp_type 'echo-request'
        option family 'ipv4'
        option target 'ACCEPT'
FIREWALL

cat >"\$tmp_dir/system" <<SYSTEM
config system
        option hostname '\$vm_name'
        option timezone 'UTC'
        option ttylogin '0'
        option log_size '64'
        option urandom_seed '0'
SYSTEM

sudo cp "\$tmp_dir/network" "\$mnt_dir/etc/config/network"
sudo cp "\$tmp_dir/dhcp" "\$mnt_dir/etc/config/dhcp"
sudo cp "\$tmp_dir/firewall" "\$mnt_dir/etc/config/firewall"
sudo cp "\$tmp_dir/system" "\$mnt_dir/etc/config/system"
if [ -n "\$root_key_b64" ]; then
  printf '%s' "\$root_key_b64" | base64 -d >"\$tmp_dir/authorized_keys"
  sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys"
  sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
  sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
fi
sync
sudo umount "\$mnt_dir"
sudo losetup -d "\$loopdev"
loopdev=""

sudo "\$qm_cmd" create "\$vmid" \\
  --name "\$vm_name" \\
  --bios ovmf \\
  --cores "\$cores" \\
  --memory "\$memory" \\
  --net0 "virtio,bridge=\$wan_bridge" \\
  --net1 "virtio,bridge=\$lan_bridge" \\
  --numa 0 \\
  --ostype l26 \\
  --scsihw virtio-scsi-pci \\
  --sockets 1 \\
  --vga virtio \\
  --onboot "\$onboot"

sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0"
sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null
disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')"
if [ -z "\$disk_volume" ]; then
  echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2
  exit 1
fi
sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume"
sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0"

if [ "\$start_vm" = "true" ]; then
  sudo "\$qm_cmd" start "\$vmid"
fi
EOF
}

openwrt() {
    require_debian_server "openwrt"

    LAB_OPENWRT_VM=true run_openwrt_pipeline
}

cleanup_calico_links() {
    ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true
    sudo ip link delete vxlan.calico 2>/dev/null || true
    sudo ip link delete tunl0 2>/dev/null || true
    sudo ip link delete cni0 2>/dev/null || true
    sudo ip link delete kube-ipvs0 2>/dev/null || true
    ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true
}

cleanup_iptables() {
    sudo iptables -F || true
    sudo iptables -X || true
    sudo iptables -t nat -F || true
    sudo iptables -t nat -X || true
    sudo iptables -t mangle -F || true
    sudo iptables -t mangle -X || true
    sudo iptables -t raw -F || true
    sudo iptables -t raw -X || true
    if command -v ipvsadm >/dev/null 2>&1; then
        sudo ipvsadm --clear || true
    fi
}

cleanup_calico_runtime_files() {
    local path

    for path in /run/calico /var/run/calico; do
        if sudo test -e "${path}"; then
            sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true
            sudo rmdir "${path}" 2>/dev/null || true
        fi
    done
}

restore_node_dns() {
    sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf
    if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then
        sudo rm -f /etc/resolv.conf
        sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf
    fi
    sudo systemctl restart systemd-resolved 2>/dev/null || true
}

cleanup_mounts() {
    if command -v findmnt >/dev/null 2>&1; then
        local mount_root
        while IFS= read -r mountpoint; do
            sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
        done < <(
            for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do
                findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true
            done | sort -ru
        )
    fi
    while IFS= read -r mountpoint; do
        sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
    done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true)
    sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true
}

cleanup_node() {
    sudo kubeadm reset --force || true
    sudo systemctl stop kubelet 2>/dev/null || true
    sudo systemctl stop containerd 2>/dev/null || true
    sudo killall containerd-shim-runc-v2 2>/dev/null || true

    cleanup_mounts

    sudo rm -rf \
        /etc/kubernetes/ \
        /var/lib/etcd/ \
        /var/lib/kubelet/ \
        /var/lib/cni/ \
        /etc/cni/net.d \
        /run/flannel \
        /var/lib/calico \
        /var/log/calico \
        /var/lib/containerd/* \
        /run/containerd/* \
        /etc/containerd/certs.d \
        /etc/containerd/config.toml
    cleanup_calico_runtime_files
    sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam

    cleanup_iptables
    cleanup_calico_links
    restore_node_dns

    sudo mkdir -p /etc/containerd/certs.d
    sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
    sudo systemctl start containerd 2>/dev/null || true
}

image_ref_tag() {
    local image_ref="$1"
    local tag

    tag="${image_ref##*:}"
    if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then
        echo "Image reference ${image_ref} must include an immutable tag." >&2
        exit 1
    fi

    printf '%s\n' "${tag}"
}

website_image_tag() {
    local source_hash="$1"

    printf 'src-%s\n' "${source_hash:0:12}"
}

apps_registry_endpoint() {
    if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then
        printf '%s\n' "${TF_VAR_registry_endpoint}"
        return 0
    fi

    demos_registry_endpoint
}

demos_registry_endpoint() {
    local image

    image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")"
    if [[ -z "${image}" || "${image}" != */* ]]; then
        echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2
        exit 1
    fi

    printf '%s\n' "${image%%/*}"
}

website_source_hash() {
    (
        cd "${REPO_ROOT}"
        find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
    )
}

demos_source_hash() {
    (
        cd "${REPO_ROOT}"
        find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
    )
}

registry_image_exists() {
    local registry_endpoint="$1"
    local repository="$2"
    local tag="$3"
    local accept_header

    if ! command -v curl >/dev/null 2>&1; then
        return 1
    fi

    accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json"
    curl -fsS \
        -H "Accept: ${accept_header}" \
        "http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null
}

image_state_value() {
    local state_file="$1"
    local key="$2"

    awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true
}

website_image_is_current() {
    local state_file="$1"
    local source_hash="$2"
    local platforms="$3"
    local image_ref="$4"
    local registry_endpoint="$5"
    local image_tag
    local saved_hash
    local saved_platforms
    local saved_image

    [[ -f "${state_file}" ]] || return 1

    saved_hash="$(image_state_value "${state_file}" source_hash)"
    saved_platforms="$(image_state_value "${state_file}" platforms)"
    saved_image="$(image_state_value "${state_file}" image)"

    [[ "${saved_hash}" == "${source_hash}" ]] || return 1
    [[ "${saved_platforms}" == "${platforms}" ]] || return 1
    [[ "${saved_image}" == "${image_ref}" ]] || return 1

    image_tag="$(image_ref_tag "${image_ref}")"
    registry_image_exists "${registry_endpoint}" php-website "${image_tag}"
}

ensure_website_image_tag_not_reused() {
    local state_file="$1"
    local source_hash="$2"
    local image_ref="$3"
    local saved_hash
    local saved_image

    [[ -f "${state_file}" ]] || return 0

    saved_hash="$(image_state_value "${state_file}" source_hash)"
    saved_image="$(image_state_value "${state_file}" image)"

    if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then
        echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2
        echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2
        exit 1
    fi
}

demos_image_is_current() {
    local state_file="$1"
    local source_hash="$2"
    local platforms="$3"
    local image_ref="$4"
    local registry_endpoint="$5"
    local saved_hash
    local saved_platforms
    local saved_image

    [[ -f "${state_file}" ]] || return 1

    saved_hash="$(image_state_value "${state_file}" source_hash)"
    saved_platforms="$(image_state_value "${state_file}" platforms)"
    saved_image="$(image_state_value "${state_file}" image)"

    [[ "${saved_hash}" == "${source_hash}" ]] || return 1
    [[ "${saved_platforms}" == "${platforms}" ]] || return 1
    [[ "${saved_image}" == "${image_ref}" ]] || return 1

    registry_image_exists "${registry_endpoint}" demos-static latest
}

write_website_image_state() {
    local state_file="$1"
    local source_hash="$2"
    local platforms="$3"
    local image_ref="$4"

    mkdir -p "$(dirname "${state_file}")"
    {
        printf 'source_hash=%s\n' "${source_hash}"
        printf 'platforms=%s\n' "${platforms}"
        printf 'image=%s\n' "${image_ref}"
    } > "${state_file}"
}

write_demos_image_state() {
    local state_file="$1"
    local source_hash="$2"
    local platforms="$3"
    local image_ref="$4"

    mkdir -p "$(dirname "${state_file}")"
    {
        printf 'source_hash=%s\n' "${source_hash}"
        printf 'platforms=%s\n' "${platforms}"
        printf 'image=%s\n' "${image_ref}"
    } > "${state_file}"
}

path_available_mb() {
    local path="$1"

    while [[ ! -e "${path}" && "${path}" != "/" ]]; do
        path="$(dirname "${path}")"
    done

    df -Pm "${path}" | awk 'NR == 2 {print $4}'
}

docker_root_dir() {
    docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n'
}

prune_unused_docker_build_data() {
    docker buildx rm lab-builder 2>/dev/null || true
    docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true
    docker builder prune -af 2>/dev/null || true
    docker system prune -af 2>/dev/null || true
}

ensure_docker_build_space() {
    local docker_root
    local free_mb
    local min_free_mb

    min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}"
    docker_root="$(docker_root_dir)"
    free_mb="$(path_available_mb "${docker_root}")"

    if (( free_mb >= min_free_mb )); then
        return 0
    fi

    echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..."
    prune_unused_docker_build_data
    free_mb="$(path_available_mb "${docker_root}")"

    if (( free_mb < min_free_mb )); then
        echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2
        echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2
        echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2
        exit 1
    fi
}

prepare_buildx_builder() {
    local registry_endpoint="$1"

    docker run --rm --privileged multiarch/qemu-user-static --reset -p yes

    cat <<EOF > "${BUILDX_CONFIG}"
[registry."${registry_endpoint}"]
  http = true
  insecure = true
[registry."127.0.0.1:30500"]
  http = true
  insecure = true
[registry."localhost:30500"]
  http = true
  insecure = true
EOF

    docker buildx rm lab-builder 2>/dev/null || true
    docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use
    docker buildx inspect --bootstrap
}

ensure_cosign_available() {
    local arch
    local cosign_url
    local target_dir
    local target_path
    local tmp_path

    if [[ -n "${COSIGN_BIN}" ]]; then
        if [[ -x "${COSIGN_BIN}" ]]; then
            return 0
        fi
        echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2
        exit 1
    fi

    if command -v cosign >/dev/null 2>&1; then
        COSIGN_BIN="$(command -v cosign)"
        return 0
    fi

    case "$(uname -m)" in
        x86_64 | amd64)
            arch="amd64"
            ;;
        aarch64 | arm64)
            arch="arm64"
            ;;
        *)
            echo "Unsupported Cosign install architecture: $(uname -m)" >&2
            exit 1
            ;;
    esac

    target_dir="${HOMELAB_STATE_DIR}/bin"
    target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}"
    if [[ -x "${target_path}" ]]; then
        COSIGN_BIN="${target_path}"
        return 0
    fi

    if ! command -v curl >/dev/null 2>&1; then
        echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2
        exit 1
    fi

    mkdir -p "${target_dir}"
    tmp_path="${target_path}.tmp"
    cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"
    echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..."
    curl -fsSL "${cosign_url}" -o "${tmp_path}"
    chmod 0755 "${tmp_path}"
    mv "${tmp_path}" "${target_path}"
    COSIGN_BIN="${target_path}"
}

ensure_cosign_password() {
    if [[ -n "${COSIGN_PASSWORD:-}" ]]; then
        return 0
    fi

    if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then
        COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
        export COSIGN_PASSWORD
        return 0
    fi

    mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")"
    if command -v openssl >/dev/null 2>&1; then
        openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}"
    elif command -v python3 >/dev/null 2>&1; then
        python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}"
import secrets

print(secrets.token_hex(32))
PY
    else
        echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2
        exit 1
    fi
    chmod 0600 "${COSIGN_PASSWORD_FILE}"
    COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
    export COSIGN_PASSWORD
}

run_cosign() {
    ensure_cosign_available
    ensure_cosign_password

    COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@"
}

ensure_cosign_keypair() {
    ensure_cosign_available
    ensure_cosign_password

    if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
        return 0
    fi

    if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
        echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2
        exit 1
    fi

    mkdir -p "$(dirname "${COSIGN_KEY_PATH}")"
    echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..."
    COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}"
    chmod 0600 "${COSIGN_KEY_PATH}"
    chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}"
}

publish_cosign_public_key_config_map() {
    kubectl --kubeconfig "${KUBECONFIG_PATH}" create namespace kyverno --dry-run=client -o yaml |
        kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f -

    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \
        --from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \
        --dry-run=client -o yaml |
        kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f -
}

cosign_registry_flags() {
    local registry_endpoint="$1"
    local image_ref="$2"

    if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then
        printf '%s\n' "--allow-http-registry"
    fi
}

image_supply_chain_metadata_exists() {
    local image_ref="$1"
    local registry_endpoint="$2"
    local digest_ref
    local -a registry_flags=()

    if ! digest_ref="$(image_digest_ref "${image_ref}")"; then
        return 1
    fi

    mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")

    run_cosign verify \
        --key "${COSIGN_PUBLIC_KEY_PATH}" \
        --insecure-ignore-tlog=true \
        "${registry_flags[@]}" \
        "${digest_ref}" >/dev/null 2>&1 &&
        run_cosign verify-attestation \
            --key "${COSIGN_PUBLIC_KEY_PATH}" \
            --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
            --insecure-ignore-tlog=true \
            "${registry_flags[@]}" \
            "${digest_ref}" >/dev/null 2>&1
}

write_image_sbom_predicate() {
    local image_ref="$1"
    local output_file="$2"
    local source_hash="$3"

    docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}"
    python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY'
from datetime import datetime, timezone
import json
import re
import sys

path = sys.argv[1]
image_ref = sys.argv[2]
source_hash = sys.argv[3]

try:
    with open(path, encoding="utf-8") as handle:
        document = json.load(handle)
except json.JSONDecodeError:
    document = None

if not isinstance(document, dict):
    safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-")
    document = {
        "spdxVersion": "SPDX-2.3",
        "dataLicense": "CC0-1.0",
        "SPDXID": "SPDXRef-DOCUMENT",
        "name": f"homelab image SBOM for {image_ref}",
        "documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}",
        "creationInfo": {
            "created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
            "creators": ["Tool: jeannie"],
        },
        "packages": [
            {
                "name": image_ref,
                "SPDXID": "SPDXRef-Image",
                "downloadLocation": "NOASSERTION",
                "filesAnalyzed": False,
                "licenseConcluded": "NOASSERTION",
                "licenseDeclared": "NOASSERTION",
                "copyrightText": "NOASSERTION",
            }
        ],
        "relationships": [
            {
                "spdxElementId": "SPDXRef-DOCUMENT",
                "relationshipType": "DESCRIBES",
                "relatedSpdxElement": "SPDXRef-Image",
            }
        ],
    }
    with open(path, "w", encoding="utf-8") as handle:
        json.dump(document, handle, sort_keys=True)
        handle.write("\n")

if not isinstance(document, dict):
    raise SystemExit("image SBOM predicate is not a JSON object")
if document.get("SPDXID") != "SPDXRef-DOCUMENT":
    raise SystemExit("image SBOM predicate is not an SPDX document")
if not document.get("packages"):
    raise SystemExit("image SBOM predicate does not include packages")
PY
}

image_digest_ref() {
    local image_ref="$1"
    local digest
    local repository

    if [[ "${image_ref}" == *@sha256:* ]]; then
        printf '%s\n' "${image_ref}"
        return 0
    fi

    digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')"
    if [[ -z "${digest}" || "${digest}" == "<no value>" ]]; then
        echo "Unable to resolve image digest for ${image_ref}." >&2
        return 1
    fi

    if [[ "${image_ref##*/}" == *:* ]]; then
        repository="${image_ref%:*}"
    else
        repository="${image_ref}"
    fi
    printf '%s@%s\n' "${repository}" "${digest}"
}

publish_image_supply_chain_metadata() {
    local image_ref="$1"
    local registry_endpoint="$2"
    local source_hash="$3"
    local digest_ref
    local sbom_file
    local -a registry_flags=()

    if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
        echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM."
        return 0
    fi

    digest_ref="$(image_digest_ref "${image_ref}")"
    mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")
    sbom_file="$(mktemp)"
    write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}"

    echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..."
    run_cosign sign \
        --yes \
        --key "${COSIGN_KEY_PATH}" \
        --tlog-upload=false \
        -a "homelab.dev/source-hash=${source_hash}" \
        "${registry_flags[@]}" \
        "${digest_ref}"
    run_cosign attest \
        --yes \
        --key "${COSIGN_KEY_PATH}" \
        --predicate "${sbom_file}" \
        --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
        --tlog-upload=false \
        "${registry_flags[@]}" \
        "${digest_ref}"
    rm -f "${sbom_file}"

    if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
        echo "Cosign verification failed after signing ${digest_ref}." >&2
        exit 1
    fi
}

dump_argocd_debug() {
    local app="$1"

    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" -o yaml || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd describe application "${app}" || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get pods -o wide || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs deployment/argocd-repo-server --tail=120 || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true
}

dump_namespace_debug() {
    local namespace="$1"

    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get all -o wide || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o wide || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe pods || true
    kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
}

wait_for_namespace() {
    local namespace="$1"
    local app="$2"
    local timeout_seconds="$3"
    local elapsed=0

    until kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; do
        if ((elapsed >= timeout_seconds)); then
            echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2
            dump_argocd_debug "${app}"
            exit 1
        fi
        sleep 5
        elapsed=$((elapsed + 5))
    done
}

wait_for_namespaced_resource() {
    local namespace="$1"
    local kind="$2"
    local name="$3"
    local app="$4"
    local timeout_seconds="$5"
    local elapsed=0

    until kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do
        if ((elapsed >= timeout_seconds)); then
            echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2
            dump_argocd_debug "${app}"
            kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
            exit 1
        fi
        sleep 5
        elapsed=$((elapsed + 5))
    done
}

wait_for_cluster_resource() {
    local kind="$1"
    local name="$2"
    local app="$3"
    local timeout_seconds="$4"
    local elapsed=0
    local api_resource

    api_resource="${kind%%/*}"
    if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get crd "${api_resource}" >/dev/null 2>&1; then
        echo "Required CRD ${api_resource} is not installed; cannot wait for ${kind}/${name} from Argo CD app ${app}." >&2
        kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" \
            -o jsonpath='sync={.status.sync.status} health={.status.health.status} message={.status.operationState.message}{"\n"}' 2>/dev/null || true
        dump_argocd_debug "${app}"
        exit 1
    fi

    until kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${kind}/${name}" >/dev/null 2>&1; do
        if ((elapsed >= timeout_seconds)); then
            echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2
            dump_argocd_debug "${app}"
            exit 1
        fi
        sleep 5
        elapsed=$((elapsed + 5))
    done
}

wait_for_deployment_ready() {
    local namespace="$1"
    local deployment="$2"
    local app="$3"
    local timeout_seconds="$4"
    local desired_replicas
    local ready_replicas
    local elapsed=0

    desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
    desired_replicas="${desired_replicas:-1}"

    until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \
        (( ${ready_replicas:-0} >= desired_replicas )); do
        if ((elapsed >= timeout_seconds)); then
            echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2
            dump_argocd_debug "${app}"
            dump_namespace_debug "${namespace}"
            exit 1
        fi
        sleep 5
        elapsed=$((elapsed + 5))
    done
}

deploy_gitea() {
    local mode="${LAB_GITEA_DEPLOY:-true}"
    local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}"
    local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}"
    local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}"
    local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}"
    local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
    local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
    local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}}"
    local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}"
    local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
    local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml"
    local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}"

    require_debian_server "deploy-gitea"

    if disabled_value "${mode}"; then
        install_gitea_backup_timer
        return 0
    fi

    if [[ ! -s "${compose_file}" ]]; then
        echo "Missing ${compose_file}" >&2
        exit 1
    fi

    echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..."

    ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea"
    scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml"

    ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu
install_dir='${install_dir}'
install_docker='${install_docker}'

install_missing_packages() {
  missing_packages=''
  for package in \"\$@\"; do
    if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then
      missing_packages=\"\$missing_packages \$package\"
    fi
  done
  if [ -n \"\$missing_packages\" ]; then
    sudo apt-get update
    sudo apt-get install -y --no-install-recommends \$missing_packages
  fi
}

install_missing_packages ca-certificates curl iptables

if ! command -v docker >/dev/null 2>&1; then
  if [ \"\$install_docker\" = 'true' ]; then
    curl -fsSL https://get.docker.com | sudo sh
  else
    echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2
    exit 1
  fi
fi

if ! sudo docker compose version >/dev/null 2>&1; then
  install_missing_packages docker-compose-plugin
fi

repair_docker_iptables() {
  if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
    return 0
  fi

  echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...'
  sudo systemctl restart docker
  sleep 3

  if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
    return 0
  fi

  echo 'Docker NAT chain is still missing after restarting Docker.' >&2
  sudo iptables -t nat -S >&2 || true
  sudo systemctl status docker --no-pager -l >&2 || true
  exit 1
}

repair_docker_iptables

sudo mkdir -p \"\$install_dir/data\"
sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\"
sudo chown -R 1000:1000 \"\$install_dir/data\"
sudo tee \"\$install_dir/.env\" >/dev/null <<ENV_EOT
GITEA_IMAGE=${image}
GITEA_CONTAINER_NAME=${container_name}
GITEA_HTTP_PORT=${http_port}
GITEA_SSH_PORT=${ssh_port}
GITEA_DOMAIN=${domain}
GITEA_ROOT_URL=${root_url}
GITEA_UID=1000
GITEA_GID=1000
ENV_EOT

cd \"\$install_dir\"
sudo docker compose pull
sudo docker compose up -d --remove-orphans
sudo docker compose ps
"

    install_gitea_backup_timer
}

install_missing_debian_packages() {
    local missing_packages=()
    local package

    for package in "$@"; do
        if ! dpkg-query -W -f='${Status}' "${package}" 2>/dev/null | grep -q "install ok installed"; then
            missing_packages+=("${package}")
        fi
    done

    if ((${#missing_packages[@]})); then
        sudo apt-get update
        sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
    fi
}

ensure_local_docker_compose() {
    local command_name="$1"
    local install_docker="$2"

    install_missing_debian_packages ca-certificates curl iptables

    if ! command -v docker >/dev/null 2>&1; then
        if [[ "${install_docker}" == "true" ]]; then
            curl -fsSL https://get.docker.com | sudo sh
        else
            echo "Docker is not installed on the Debian host. Install Docker through host bootstrap first, or rerun with LAB_${command_name}_INSTALL_DOCKER=true to allow get.docker.com installation." >&2
            exit 1
        fi
    fi

    if ! sudo docker compose version >/dev/null 2>&1; then
        install_missing_debian_packages docker-compose-plugin
    fi
}

repair_local_docker_iptables() {
    if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
        return 0
    fi

    echo "Docker NAT chain is missing on the Debian host; restarting Docker once to restore iptables state..."
    sudo systemctl restart docker
    sleep 3

    if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
        return 0
    fi

    echo "Docker NAT chain is still missing after restarting Docker." >&2
    sudo iptables -t nat -S >&2 || true
    sudo systemctl status docker --no-pager -l >&2 || true
    exit 1
}

deploy_heimdall() {
    local mode="${LAB_HEIMDALL_DEPLOY:-true}"
    local install_dir="${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}"
    local data_dir="${LAB_HEIMDALL_DATA_DIR:-${install_dir}/data}"
    local config_dir="${LAB_HEIMDALL_CONFIG_DIR:-${data_dir}/config}"
    local http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}"
    local bind_ip="${LAB_HEIMDALL_BIND_IP:-0.0.0.0}"
    local public_url="${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:-lab2025.duckdns.org}/}"
    local image="${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347}"
    local container_name="${HEIMDALL_CONTAINER_NAME:-homelab-heimdall}"
    local seeder_container_name="${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder}"
    local network="${HEIMDALL_NETWORK:-homelab-heimdall}"
    local install_docker="${LAB_HEIMDALL_INSTALL_DOCKER:-false}"
    local source_dir="${REPO_ROOT}/infra/heimdall"
    local elapsed=0
    local value_name
    local value

    require_debian_server "deploy-heimdall"

    if disabled_value "${mode}"; then
        return 0
    fi

    for value_name in install_dir data_dir config_dir http_port bind_ip public_url image container_name seeder_container_name network install_docker; do
        value="${!value_name}"
        if [[ "${value}" == *$'\n'* ]]; then
            echo "${value_name} cannot contain a newline." >&2
            exit 1
        fi
    done

    if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/links.json" || ! -s "${source_dir}/seed-heimdall.py" ]]; then
        echo "Missing Heimdall source files in ${source_dir}" >&2
        exit 1
    fi

    echo "Deploying external Heimdall on the Debian host at ${install_dir}..."

    ensure_local_docker_compose "HEIMDALL" "${install_docker}"
    repair_local_docker_iptables

    sudo mkdir -p "${install_dir}" "${config_dir}"
    sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
    sudo cp "${source_dir}/links.json" "${install_dir}/links.json"
    sudo cp "${source_dir}/seed-heimdall.py" "${install_dir}/seed-heimdall.py"
    sudo chown -R 1000:1000 "${data_dir}"
    sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
HEIMDALL_IMAGE=${image}
HEIMDALL_CONTAINER_NAME=${container_name}
HEIMDALL_SEEDER_CONTAINER_NAME=${seeder_container_name}
HEIMDALL_HTTP_PORT=${http_port}
HEIMDALL_BIND_IP=${bind_ip}
HEIMDALL_PUBLIC_URL=${public_url%/}
HEIMDALL_CONFIG_DIR=${config_dir}
HEIMDALL_UID=1000
HEIMDALL_GID=1000
HEIMDALL_NETWORK=${network}
TZ=America/Mexico_City
ENV_EOT

    (
        cd "${install_dir}"
        sudo docker compose pull
        sudo docker compose up -d --remove-orphans
        sudo docker compose ps
    )

    until curl -fsS --connect-timeout 5 "http://127.0.0.1:${http_port}/" >/dev/null; do
        if ((elapsed >= 180)); then
            echo "Timed out waiting for Heimdall on http://127.0.0.1:${http_port}/" >&2
            (cd "${install_dir}" && sudo docker compose ps && sudo docker compose logs --tail=120) >&2 || true
            exit 1
        fi
        sleep 5
        elapsed=$((elapsed + 5))
    done

    echo "Heimdall is reachable at http://127.0.0.1:${http_port}/ and ${public_url%/}/."
}

deploy_matrix() {
    local mode="${LAB_MATRIX_DEPLOY:-true}"
    local install_dir="${LAB_MATRIX_INSTALL_DIR:-/home/jv/matrix-server}"
    local image=""
    local container_name=""
    local http_port=""
    local ssh_port=""
    local domain=""
    local install_docker="false"
    local source_dir="${REPO_ROOT}/infra/matrix-server"
    local value_name
    local value

    require_debian_server "deploy-matrix"

    if disabled_value "${mode}"; then
        return 0
    fi

    for value_name in install_dir; do
        value="${!value_name}"
        if [[ "${value}" == *$'\n'* ]]; then
            echo "${value_name} cannot contain a newline." >&2
            exit 1
        fi
    done

    if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/homeserver.yaml.tpl" || ! -s "${source_dir}/bootstrap.sh" ]]; then
        echo "Missing Matrix source files in ${source_dir}" >&2
        exit 1
    fi

    echo "Deploying Matrix stack on the Debian host at ${install_dir}..."

    ensure_local_docker_compose "MATRIX" "${install_docker}"
    repair_local_docker_iptables

    sudo mkdir -p "${install_dir}"
    sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
    sudo cp "${source_dir}/homeserver.yaml.tpl" "${install_dir}/homeserver.yaml.tpl"
    sudo cp "${source_dir}/log_config.yaml" "${install_dir}/log_config.yaml"
    sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" 2>/dev/null || {
        sudo mkdir -p "${install_dir}/element-config"
        sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json"
    }
    sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh"
    sudo chmod +x "${install_dir}/bootstrap.sh"

    # Secrets come from the SOPS-encrypted repo file, decrypted at deploy time.
    # (Env vars override the file, for manual/ops overrides.)
    if ! sops_load_secrets "${REPO_ROOT}/infra/matrix-server/matrix.secret.yaml"; then
        exit 1
    fi

    if [[ -z "${MATRIX_POSTGRES_PASSWORD:-}" || -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]]; then
        echo "MATRIX_POSTGRES_PASSWORD and MATRIX_REGISTRATION_SHARED_SECRET missing after decrypt." >&2
        exit 1
    fi

    local matrix_server_name="${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}"
    local matrix_synapse_port="${LAB_MATRIX_SYNAPSE_PORT:-8008}"
    local matrix_element_port="${LAB_MATRIX_ELEMENT_PORT:-8081}"
    local postgres_db="${LAB_MATRIX_POSTGRES_DB:-synapse}"
    local postgres_user="${LAB_MATRIX_POSTGRES_USER:-synapse}"

    sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
MATRIX_SERVER_NAME=${matrix_server_name}
MATRIX_SYNAPSE_PORT=${matrix_synapse_port}
MATRIX_ELEMENT_PORT=${matrix_element_port}
MATRIX_SYNAPSE_CONTAINER_NAME=${LAB_MATRIX_SYNAPSE_CONTAINER_NAME:-matrix-synapse}
MATRIX_POSTGRES_CONTAINER_NAME=${LAB_MATRIX_POSTGRES_CONTAINER_NAME:-matrix-postgres}
MATRIX_ELEMENT_CONTAINER_NAME=${LAB_MATRIX_ELEMENT_CONTAINER_NAME:-matrix-element}
MATRIX_POSTGRES_DB=${postgres_db}
MATRIX_POSTGRES_USER=${postgres_user}
MATRIX_POSTGRES_PASSWORD=${MATRIX_POSTGRES_PASSWORD}
MATRIX_REGISTRATION_SHARED_SECRET=${MATRIX_REGISTRATION_SHARED_SECRET}
MATRIX_MACAROON_SECRET_KEY=${MATRIX_MACAROON_SECRET_KEY:-${MATRIX_REGISTRATION_SHARED_SECRET}}
MATRIX_FORM_SECRET=${MATRIX_FORM_SECRET:-${MATRIX_REGISTRATION_SHARED_SECRET}}
ENV_EOT

    # Pull + render + chown + up. bootstrap.sh is the single source of truth.
    # sudo the docker compose commands; data dirs are chown'd inside bootstrap.
    (
        cd "${install_dir}"
        sudo docker compose --env-file "${install_dir}/.env" -f "${install_dir}/docker-compose.yml" pull
        sudo MATRIX_ENV_FILE="${install_dir}/.env" bash "${install_dir}/bootstrap.sh"
    )

    echo "Matrix stack is reachable at http://127.0.0.1:${matrix_synapse_port}/ and Element at http://127.0.0.1:${matrix_element_port}/."
}

deploy_omniroute() {
    local mode="${LAB_OMNIROUTE_DEPLOY:-true}"
    local install_dir="${LAB_OMNIROUTE_INSTALL_DIR:-/home/jv/homelab-omniroute}"
    local install_docker="false"
    local source_dir="${REPO_ROOT}/infra/omniroute"

    require_debian_server "deploy-omniroute"

    if disabled_value "${mode}"; then
        return 0
    fi

    if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/bootstrap.sh" ]]; then
        echo "Missing OmniRoute source files in ${source_dir}" >&2
        exit 1
    fi

    echo "Deploying OmniRoute LLM gateway on the Debian host at ${install_dir}..."

    ensure_local_docker_compose "OMNIROUTE" "${install_docker}"
    repair_local_docker_iptables

    sudo mkdir -p "${install_dir}"
    sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
    sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh"
    sudo chmod +x "${install_dir}/bootstrap.sh"

    sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
OMNIROUTE_IMAGE=${LAB_OMNIROUTE_IMAGE:-diegosouzapw/omniroute:latest}
OMNIROUTE_CONTAINER_NAME=${LAB_OMNIROUTE_CONTAINER_NAME:-omniroute}
OMNIROUTE_PORT=${LAB_OMNIROUTE_PORT:-20128}
ENV_EOT

    (
        cd "${install_dir}"
        sudo docker compose pull
        sudo bash "${install_dir}/bootstrap.sh"
    )

    echo "OmniRoute is reachable at http://127.0.0.1:${LAB_OMNIROUTE_PORT:-20128}."
}

deploy_hermes() {
    local mode="${LAB_HERMES_DEPLOY:-true}"
    local hermes_home="${LAB_HERMES_HOME:-/home/jv/.hermes}"
    local install_dir="${LAB_HERMES_INSTALL_DIR:-/home/jv/.hermes/hermes-agent}"
    local uv_bin="${LAB_HERMES_UV_BIN:-/home/jv/.hermes/bin/uv}"
    local install_docker="${LAB_HERMES_INSTALL_DOCKER:-false}"
    local value_name
    local value

    require_debian_server "deploy-hermes"

    if disabled_value "${mode}"; then
        return 0
    fi

    for value_name in hermes_home install_dir uv_bin; do
        value="${!value_name}"
        if [[ "${value}" == *$'\n'* ]]; then
            echo "${value_name} cannot contain a newline." >&2
            exit 1
        fi
    done

    echo "Deploying Hermes agent on the Debian host at ${install_dir}..."

    ensure_local_docker_compose "HERMES" "${install_docker}"

    # Install uv if missing.
    if [[ ! -x "${uv_bin}" ]]; then
        mkdir -p "$(dirname "${uv_bin}")"
        echo "Installing uv..."
        curl -LsSf https://astral.sh/uv/install.sh | bash
        # uv installs to ~/.local/bin; copy/alias into HERMES bin if that path differs.
        if command -v uv >/dev/null 2>&1 && [[ ! -x "${uv_bin}" ]]; then
            cp "$(command -v uv)" "${uv_bin}"
        fi
    fi

    # Install Hermes via its official installer (git method, managed venv layout).
    if [[ ! -x "${hermes_home}/hermes-agent/venv/bin/python" ]]; then
        echo "Installing Hermes agent..."
        curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash
    fi

    # Ensure the Matrix platform dependencies (E2EE) are present in the venv.
    if [[ -x "${uv_bin}" && -f "${install_dir}/pyproject.toml" ]]; then
        cd "${install_dir}"
        "${uv_bin}" pip install 'mautrix[encryption]' -p venv/bin/python
    fi

    # Secret env for the gateway (.env under HERMES_HOME, mode 600, never committed).
    # Pull the hermes secrets from the SOPS-encrypted repo file; env vars override.
    if ! sops_load_secrets "${REPO_ROOT}/infra/hermes.secret.yaml"; then
        echo "WARNING: could not load hermes secrets; MATRIX_PASSWORD may be blank." >&2
    fi

    local secrets_file="${hermes_home}/.env"
    sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "${hermes_home}"
    if [[ -f "${secrets_file}" ]]; then
        sudo cp "${secrets_file}" "${secrets_file}.bak-pre-jeannie"
    fi
    sudo tee "${secrets_file}" >/dev/null <<ENV_EOT
OPENAI_BASE_URL=${HERMES_OPENAI_BASE_URL:-http://localhost:${LAB_OMNIROUTE_PORT:-20128}/v1}
HERMES_CUSTOM_LOCALHOST_${LAB_OMNIROUTE_PORT:-20128}_API_KEY=${HERMES_OMNIROUTE_API_KEY:-}
MATRIX_HOMESERVER=${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}
MATRIX_USER_ID=${HERMES_MATRIX_USER_ID:-@hermes:matrix.lab2025.duckdns.org}
MATRIX_PASSWORD=${MATRIX_BOT_PASSWORD:-}
MATRIX_ALLOWED_USERS=${HERMES_MATRIX_ALLOWED_USERS:-@jv:matrix.lab2025.duckdns.org}
MATRIX_E2EE_MODE=required
ENV_EOT
    sudo chmod 600 "${secrets_file}"

    # Create the Matrix bot account (idempotent) if the homeserver is up.
    if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^matrix-synapse$'; then
        if [[ -n "${MATRIX_BOT_PASSWORD:-}" ]]; then
            if ! docker exec matrix-synapse register_new_matrix_user -c /data/homeserver.yaml \
                -u hermes -p "${MATRIX_BOT_PASSWORD}" --no-admin >/dev/null 2>&1; then
                echo "Hermes bot account already exists (expected on re-run) or auto-registration skipped." >&2
            else
                echo "Hermes Matrix bot account ready."
            fi
        else
            echo "MATRIX_BOT_PASSWORD not set; skipping Hermes bot account registration (needed only on fresh install)." >&2
        fi
    fi

    # Install/start the gateway systemd user service (Hermes manages the unit).
    "${hermes_home}/hermes-agent/venv/bin/python" "${hermes_home}/hermes-agent/hermes" gateway install
    systemctl --user daemon-reload
    systemctl --user enable --now hermes-gateway 2>/dev/null || true

    echo "Hermes agent deployed. Run 'hermes' to configure the model, or check gateway logs at ${hermes_home}/logs/gateway.log."
}

deploy_rpi_services() {
    local mode="${LAB_RPI_SERVICES_DEPLOY:-true}"
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
    local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}"
    local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
    local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
    local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}"
    local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}"
    local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}"
    local source_dir="${REPO_ROOT}/infra/rpi-services"
    local value_name
    local value

    require_debian_server "rpi-services"

    if disabled_value "${mode}"; then
        return 0
    fi

    if [[ ! -d "${source_dir}" ]]; then
        echo "Missing ${source_dir}" >&2
        exit 1
    fi

    for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do
        value="${!value_name}"
        if [[ "${value}" == *"'"* ]]; then
            echo "${value_name} cannot contain a single quote." >&2
            exit 1
        fi
    done

    echo "Deploying RPi services on ${rpi_user}@${rpi_host}..."

    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services"
    scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
        "${source_dir}/docker-compose.yml" \
        "${source_dir}/adlists.txt" \
        "${source_dir}/local-dns-records.txt" \
        "${source_dir}/cname-records.txt" \
        "${source_dir}/static-dhcp-hosts.txt" \
        "${source_dir}/uptime-kuma-monitors.json" \
        "${source_dir}/seed-uptime-kuma.js" \
        "${source_dir}/bootstrap.sh" \
        "${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/"

    ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \
LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \
LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \
LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \
LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \
LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \
LAB_RPI_INSTALL_DOCKER='${install_docker}' \
PIHOLE_WEBPASSWORD='${pihole_webpassword}' \
bash /tmp/homelab-rpi-services/bootstrap.sh"
}

gitea_bootstrap_password() {
    if command -v openssl >/dev/null 2>&1; then
        openssl rand -hex 32
        return 0
    fi

    python3 - <<'PY'
import secrets

print(secrets.token_hex(32))
PY
}

gitea_api_base_url() {
    local gitea_host="$1"
    local http_port="$2"
    local candidate
    local api_base_override="${LAB_GITEA_API_BASE_URL:-}"

    if [[ -n "${api_base_override}" ]]; then
        printf '%s\n' "${api_base_override%/}"
        return 0
    fi

    for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do
        if curl -fsS "${candidate}/version" >/dev/null 2>&1; then
            printf '%s\n' "${candidate}"
            return 0
        fi
    done

    echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2
    exit 1
}

gitea_repo_exists() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local owner="$4"
    local repo_name="$5"
    local status

    status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")"
    case "${status}" in
        200)
            return 0
            ;;
        404)
            return 1
            ;;
        401 | 403)
            echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2
            exit 1
            ;;
        *)
            echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2
            exit 1
            ;;
    esac
}

gitea_branch_exists() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local owner="$4"
    local repo_name="$5"
    local branch="$6"
    local status

    status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")"
    case "${status}" in
        200)
            return 0
            ;;
        404)
            return 1
            ;;
        401 | 403)
            echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2
            exit 1
            ;;
        *)
            echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2
            exit 1
            ;;
    esac
}

create_gitea_repo() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local repo_name="$4"
    local default_branch="$5"
    local payload

    payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY'
import json
import sys

repo_name, default_branch = sys.argv[1:3]
print(json.dumps({
    "name": repo_name,
    "private": False,
    "auto_init": False,
    "default_branch": default_branch,
    "description": "Homelab infrastructure configuration",
}))
PY
)"

    curl -fsS \
        -u "${auth_user}:${auth_password}" \
        -H "Content-Type: application/json" \
        -X POST \
        -d "${payload}" \
        "${api_base}/user/repos" >/dev/null
}

gitea_public_key_registered() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local owner="$4"
    local repo_name="$5"
    local public_key_path="$6"
    local repo_keys
    local user_keys

    user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")"
    repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")"

    GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
    GITEA_USER_KEYS="${user_keys}" \
    GITEA_REPO_KEYS="${repo_keys}" \
        python3 - <<'PY'
import json
import os
import sys

public_key = os.environ["GITEA_PUBLIC_KEY"].strip()
for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"):
    for key in json.loads(os.environ[env_name]) or []:
        if key.get("key", "").strip() == public_key:
            sys.exit(0)
sys.exit(1)
PY
}

create_gitea_repo_deploy_key() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local owner="$4"
    local repo_name="$5"
    local title="$6"
    local public_key_path="$7"
    local read_only="$8"
    local payload

    payload="$(
        GITEA_DEPLOY_KEY_TITLE="${title}" \
        GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
        GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \
            python3 - <<'PY'
import json
import os

print(json.dumps({
    "title": os.environ["GITEA_DEPLOY_KEY_TITLE"],
    "key": os.environ["GITEA_PUBLIC_KEY"].strip(),
    "read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true",
}))
PY
    )"

    curl -fsS \
        -u "${auth_user}:${auth_password}" \
        -H "Content-Type: application/json" \
        -X POST \
        -d "${payload}" \
        "${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null
}

ensure_gitea_repo_ssh_access() {
    local api_base="$1"
    local auth_user="$2"
    local auth_password="$3"
    local owner="$4"
    local repo_name="$5"
    local ssh_host="$6"
    local ssh_port="$7"
    local key_path="$8"
    local key_title="$9"
    local key_read_only="${10}"
    local key_dir
    local known_hosts
    local public_key_path
    local read_only_json="false"
    local ssh_repo_url

    if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then
        echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2
        exit 1
    fi

    key_dir="$(dirname "${key_path}")"
    public_key_path="${key_path}.pub"
    mkdir -p "${key_dir}"
    chmod 0700 "${key_dir}"

    if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then
        ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null
    elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then
        ssh-keygen -y -f "${key_path}" >"${public_key_path}"
    elif [[ ! -s "${key_path}" ]]; then
        echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2
        exit 1
    fi

    chmod 0600 "${key_path}"
    chmod 0644 "${public_key_path}"

    if truthy "${key_read_only}"; then
        read_only_json="true"
    fi

    if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then
        echo "Gitea already has Debian host SSH key ${public_key_path}."
    else
        create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}"
        echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}."
    fi

    known_hosts="${HOME}/.ssh/known_hosts"
    touch "${known_hosts}"
    chmod 0644 "${known_hosts}"
    if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then
        ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null
    fi

    ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git"
    git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null ||
        git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}"
    git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"
    git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null
    echo "Gitea SSH remote: ${ssh_repo_url}"
}

bootstrap_gitea_repo() {
    local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}"
    local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
    local gitea_user="${LAB_GITEA_USER:-jv}"
    local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
    local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
    local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
    local root_url="${LAB_GITEA_ROOT_URL:?LAB_GITEA_ROOT_URL is required from homelab.yml}"
    local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}"
    local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}"
    local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}"
    local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}"
    local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}"
    local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}"
    local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}"
    local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}"
    local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}"
    local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
    local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}"
    local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}"
    local api_base
    local public_repo_url
    local direct_repo_url
    local push_url
    local askpass
    local credentials_dir
    local remote_status
    local worktree_status

    require_debian_server "bootstrap-gitea-repo"

    if disabled_value "${mode}"; then
        return 0
    fi

    ensure_python3
    for value_name in repo_owner repo_name default_branch bootstrap_user; do
        local value="${!value_name}"
        if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
            echo "${value_name} contains unsupported characters." >&2
            exit 1
        fi
    done
    if [[ "${bootstrap_email}" == *"'"* ]]; then
        echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2
        exit 1
    fi
    if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then
        echo "LAB_GITEA_SSH_PORT must be numeric." >&2
        exit 1
    fi

    if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then
        # shellcheck disable=SC1090
        source "${credentials_file}"
        bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}"
        bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}"
        bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}"
    fi

    if [[ -z "${bootstrap_password}" ]]; then
        bootstrap_password="$(gitea_bootstrap_password)"
        credentials_dir="$(dirname "${credentials_file}")"
        mkdir -p "${credentials_dir}"
        chmod 0700 "${credentials_dir}"
        {
            printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}"
            printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}"
            printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}"
        } > "${credentials_file}"
        chmod 0600 "${credentials_file}"
        echo "Generated Gitea bootstrap credentials at ${credentials_file}."
    fi

    for value_name in repo_owner repo_name default_branch bootstrap_user; do
        local value="${!value_name}"
        if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
            echo "${value_name} contains unsupported characters." >&2
            exit 1
        fi
    done
    for value_name in bootstrap_email bootstrap_password; do
        local value="${!value_name}"
        if [[ "${value}" == *"'"* ]]; then
            echo "${value_name} cannot contain a single quote." >&2
            exit 1
        fi
    done

    echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..."

    # shellcheck disable=SC2087
    ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" <<EOF
set -euo pipefail

container_name='${container_name}'
bootstrap_user='${bootstrap_user}'
bootstrap_email='${bootstrap_email}'
bootstrap_password='${bootstrap_password}'

if ! sudo docker inspect "\${container_name}" >/dev/null 2>&1; then
  echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2
  exit 1
fi

for attempt in \$(seq 1 60); do
  if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 ||
     curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then
    break
  fi
  if [ "\${attempt}" = "60" ]; then
    echo "Timed out waiting for Gitea API inside \${container_name}." >&2
    exit 1
  fi
  sleep 2
done

if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \
  --username "\${bootstrap_user}" \
  --password "\${bootstrap_password}" \
  --email "\${bootstrap_email}" \
  --admin \
  --must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then
  if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then
    sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \
      --username "\${bootstrap_user}" \
      --password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || {
        cat /tmp/homelab-gitea-user-password.log >&2
        exit 1
      }
  else
    cat /tmp/homelab-gitea-user-create.log >&2
    exit 1
  fi
fi
EOF

    api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")"

    if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then
        echo "Gitea repository ${repo_owner}/${repo_name} already exists."
    else
        if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then
            echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2
            exit 1
        fi
        create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}"
        echo "Created Gitea repository ${repo_owner}/${repo_name}."
    fi

    public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git"
    if [[ "${api_base}" == */git/api/v1 ]]; then
        direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git"
    else
        direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git"
    fi
    push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}"

    git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null
    git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null ||
        git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}"

    if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then
        echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged."
    else
        worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)"
        if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then
            echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2
            echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2
            exit 1
        fi

        askpass="$(mktemp)"
        trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT
        cat > "${askpass}" <<ASKPASS_EOT
#!/usr/bin/env bash
case "\$1" in
  *Username*) printf '%s\n' '${bootstrap_user}' ;;
  *Password*) printf '%s\n' '${bootstrap_password}' ;;
  *) printf '\n' ;;
esac
ASKPASS_EOT
        chmod 0700 "${askpass}"

        GIT_ASKPASS="${askpass}" GIT_TERMINAL_PROMPT=0 \
            git -C "${REPO_ROOT}" push "${push_url}" "HEAD:refs/heads/${default_branch}"
        rm -f "${askpass}"
        trap 'rm -f "${BUILDX_CONFIG}"' EXIT
        echo "Pushed current HEAD to Gitea branch ${default_branch}."
    fi

    remote_status="$(git -C "${REPO_ROOT}" remote get-url gitea)"
    echo "Gitea remote: ${remote_status}"
    if ! disabled_value "${ssh_bootstrap}"; then
        ensure_gitea_repo_ssh_access \
            "${api_base}" \
            "${bootstrap_user}" \
            "${bootstrap_password}" \
            "${repo_owner}" \
            "${repo_name}" \
            "${gitea_host}" \
            "${ssh_port}" \
            "${ssh_key_path}" \
            "${ssh_key_title}" \
            "${ssh_key_read_only}"
    fi
}

install_gitea_backup_timer() {
    local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
    local gitea_user="${LAB_GITEA_USER:-jv}"
    local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local gitea_container="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
    local backup_dir="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
    local backup_script="/usr/local/sbin/homelab-gitea-backup.sh"
    local restore_drill_script="/usr/local/sbin/homelab-gitea-restore-drill.sh"

    sudo tee "${backup_script}" >/dev/null <<BACKUP_SCRIPT_EOT
#!/usr/bin/env bash
set -euo pipefail

GITEA_HOST="\${GITEA_HOST:-${gitea_host}}"
GITEA_USER="\${GITEA_USER:-${gitea_user}}"
GITEA_SSH_KEY_PATH="\${GITEA_SSH_KEY_PATH:-${gitea_key}}"
GITEA_CONTAINER="\${GITEA_CONTAINER:-${gitea_container}}"
GITEA_BACKUP_DIR="\${GITEA_BACKUP_DIR:-${backup_dir}}"
GITEA_BACKUP_RETENTION_DAYS="\${GITEA_BACKUP_RETENTION_DAYS:-30}"
REMOTE_ARCHIVE="/tmp/homelab-gitea-dump.zip"
timestamp="\$(date -u +%Y%m%dT%H%M%SZ)"
tmp_archive="\$(mktemp "/tmp/gitea-\${timestamp}.XXXXXX.zip")"
backup_archive="\${GITEA_BACKUP_DIR}/gitea-\${timestamp}.zip"
remote_host_archive="/tmp/gitea-\${timestamp}.zip"

ssh_gitea() {
    ssh -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "\${GITEA_USER}@\${GITEA_HOST}" "\$@"
}

cleanup() {
    rm -f "\${tmp_archive}"
    ssh_gitea "rm -f '\${remote_host_archive}'; sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
}
trap cleanup EXIT

ssh_gitea "set -eu
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true
sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories'
sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}'
sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}'
sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}'
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true"

scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
    "\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}"

sudo mkdir -p "\${GITEA_BACKUP_DIR}"
sudo chown jv:jv "\${GITEA_BACKUP_DIR}"
sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}"
sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete

echo "Created \${backup_archive}"
BACKUP_SCRIPT_EOT
    sudo chmod 0755 "${backup_script}"

    sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT'
[Unit]
Description=Back up external Homelab Gitea to Debian host storage
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-gitea-backup.sh
SERVICE_EOT

    sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT'
[Unit]
Description=Run daily Homelab Gitea backups

[Timer]
OnCalendar=*-*-* 02:35:00
RandomizedDelaySec=20m
Persistent=true

[Install]
WantedBy=timers.target
TIMER_EOT

    sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT'
#!/usr/bin/env bash
set -euo pipefail

GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}"
GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}"

if ! command -v python3 >/dev/null 2>&1; then
    echo "python3 is required for Gitea restore drills." >&2
    exit 1
fi

latest_archive="$(
    { find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } |
        sort -nr |
        awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }'
)"

if [[ -z "${latest_archive}" ]]; then
    echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}."
    exit 0
fi

timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")"
tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")"
report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt"

cleanup() {
    rm -rf "${tmp_dir}"
    rm -f "${tmp_report}"
}
trap cleanup EXIT

python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY'
import os
import sys
import zipfile

archive_path, extract_dir, report_path = sys.argv[1:4]

with zipfile.ZipFile(archive_path) as archive:
    bad_member = archive.testzip()
    if bad_member:
        raise SystemExit(f"ZIP integrity check failed at {bad_member}")

    members = archive.infolist()
    if not members:
        raise SystemExit("ZIP archive is empty")

    extract_root = os.path.abspath(extract_dir)
    for member in members:
        target = os.path.abspath(os.path.join(extract_root, member.filename))
        if target != extract_root and not target.startswith(extract_root + os.sep):
            raise SystemExit(f"Unsafe archive path: {member.filename}")

    archive.extractall(extract_root)

file_count = 0
total_bytes = 0
for root, _, files in os.walk(extract_dir):
    for name in files:
        file_count += 1
        total_bytes += os.path.getsize(os.path.join(root, name))

if file_count == 0:
    raise SystemExit("Archive extracted no files")

with open(report_path, "w", encoding="utf-8") as handle:
    handle.write("Gitea restore drill report\n")
    handle.write(f"archive={archive_path}\n")
    handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n")
    handle.write(f"extracted_files={file_count}\n")
    handle.write(f"extracted_bytes={total_bytes}\n")
    handle.write("result=ok\n")
PY

sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}"
sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}"
sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete

echo "Created ${report_path}"
RESTORE_DRILL_SCRIPT_EOT
    sudo chmod 0755 "${restore_drill_script}"

    sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT'
[Unit]
Description=Run a non-destructive Gitea backup restore drill
After=network-online.target homelab-gitea-backup.service
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh
RESTORE_DRILL_SERVICE_EOT

    sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT'
[Unit]
Description=Run monthly Homelab Gitea restore drills

[Timer]
OnCalendar=monthly
RandomizedDelaySec=2h
Persistent=true

[Install]
WantedBy=timers.target
RESTORE_DRILL_TIMER_EOT

    sudo systemctl daemon-reload
    sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null
    sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null
}


backup_gitea() {
    require_debian_server "backup-gitea"

    install_gitea_backup_timer
    sudo /usr/local/sbin/homelab-gitea-backup.sh
}

drill_gitea_restore() {
    require_debian_server "drill-gitea-restore"

    install_gitea_backup_timer
    sudo /usr/local/sbin/homelab-gitea-restore-drill.sh
}

install_gitea_runner() {
    local runner_arch
    local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}"
    local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}"
    local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}"
    local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}"
    local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}"
    local runner_user="${GITEA_RUNNER_USER:-jv}"
    local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}"
    local missing_packages=()

    require_debian_server "install-gitea-runner"

    case "$(dpkg --print-architecture)" in
        amd64)
            runner_arch="linux-amd64"
            ;;
        arm64)
            runner_arch="linux-arm64"
            ;;
        *)
            echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2
            exit 1
            ;;
    esac

    for package in ca-certificates curl git nodejs python3; do
        if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
            missing_packages+=("$package")
        fi
    done
    if [[ ${#missing_packages[@]} -gt 0 ]]; then
        sudo apt-get update
        sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
    fi

    sudo curl -fsSL \
        -o /usr/local/bin/act_runner \
        "https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}"
    sudo chmod 0755 /usr/local/bin/act_runner
    sudo chown root:root /usr/local/bin/act_runner

    sudo -u "${runner_user}" mkdir -p "${runner_home}"

    if [[ ! -f "${runner_home}/.runner" ]]; then
        if [[ -z "${runner_token}" ]]; then
            echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2
            exit 1
        fi

        sudo -u "${runner_user}" env \
            HOME="/home/${runner_user}" \
            GITEA_RUNNER_HOME="${runner_home}" \
            GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \
            GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \
            GITEA_RUNNER_NAME="${runner_name}" \
            GITEA_RUNNER_LABELS="${runner_labels}" \
            bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"'
    else
        echo "Existing runner registration found at ${runner_home}/.runner; keeping it."
    fi

    sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null <<SERVICE_EOT
[Unit]
Description=Homelab Gitea Actions runner for my-homelab-configs
After=network-online.target docker.service
Wants=network-online.target

[Service]
Type=simple
User=${runner_user}
Group=${runner_user}
WorkingDirectory=${runner_home}
Environment=HOME=/home/${runner_user}
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ExecStart=/usr/local/bin/act_runner daemon
Restart=always
RestartSec=5

[Install]
WantedBy=multi-user.target
SERVICE_EOT

    sudo systemctl daemon-reload
    sudo systemctl enable --now homelab-gitea-runner.service >/dev/null
    sudo systemctl status homelab-gitea-runner.service --no-pager -l
}

recreate_pods_for_selector() {
    local namespace="$1"
    local selector="$2"
    local app="$3"

    if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then
        echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2
        dump_argocd_debug "${app}"
        dump_namespace_debug "${namespace}"
        exit 1
    fi
}

refresh_argocd_application() {
    local app="$1"

    kubectl --kubeconfig "${KUBECONFIG_PATH}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null
}

website_translation_model() {
    local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}"
    local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}"

    require_debian_server "website-translation-model"

    if ! command -v ollama >/dev/null 2>&1; then
        echo "ollama is not installed or not in PATH on this Debian server." >&2
        exit 1
    fi

    if [[ ! -r "${modelfile}" ]]; then
        echo "Ollama Modelfile is not readable: ${modelfile}" >&2
        exit 1
    fi

    echo "Creating or updating Ollama model ${model_name} from ${modelfile}..."
    ollama create "${model_name}" -f "${modelfile}"

    if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
        echo "Ollama model ${model_name} was not found after creation." >&2
        exit 1
    fi

    echo "Ollama model ${model_name} is ready."
}

website_ollama_listen() {
    local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}"
    local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
    local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}"
    local dropin_dir="/etc/systemd/system/ollama.service.d"
    local dropin_file="${dropin_dir}/homelab.conf"
    local waited=0

    require_debian_server "website-ollama-listen"

    bind_address="${bind_address#http://}"
    bind_address="${bind_address#https://}"

    if ! systemctl cat ollama.service >/dev/null 2>&1; then
        echo "ollama.service was not found on this Debian server." >&2
        exit 1
    fi

    echo "Configuring ollama.service to listen on ${bind_address}..."
    sudo mkdir -p "${dropin_dir}"
    sudo mkdir -p "${models_dir}"
    if id ollama >/dev/null 2>&1; then
        sudo chown ollama:ollama "${models_dir}"
    fi
    printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" |
        sudo tee "${dropin_file}" >/dev/null
    sudo systemctl daemon-reload
    sudo systemctl restart ollama.service
    sudo systemctl is-active --quiet ollama.service

    echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..."
    until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do
        waited=$((waited + 2))
        if ((waited >= 60)); then
            echo "Ollama service did not expose the local API after ${waited}s." >&2
            echo "Recent service logs:" >&2
            sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true
            exit 1
        fi
        sleep 2
    done
    echo "Ollama API is reachable on the Debian host."
}

ollama_linux_arch() {
    case "$(uname -m)" in
        x86_64 | amd64)
            printf 'amd64\n'
            ;;
        aarch64 | arm64)
            printf 'arm64\n'
            ;;
        *)
            echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2
            return 1
            ;;
    esac
}

ollama_write_systemd_service() {
    sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF'
[Unit]
Description=Ollama Service
After=network-online.target

[Service]
ExecStart=/usr/bin/ollama serve
User=ollama
Group=ollama
Restart=always
RestartSec=3
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

[Install]
WantedBy=default.target
EOF
}

ollama_install_from_tgz() {
    local tarball="$1"

    if [[ ! -r "${tarball}" ]]; then
        echo "Ollama tarball is not readable: ${tarball}" >&2
        exit 1
    fi

    echo "Installing Ollama from local tarball ${tarball}..."
    sudo tar -C /usr -xzf "${tarball}"
    if ! id ollama >/dev/null 2>&1; then
        sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
    fi
    if getent group render >/dev/null 2>&1; then
        sudo usermod -aG render ollama
    fi
    if getent group video >/dev/null 2>&1; then
        sudo usermod -aG video ollama
    fi
    ollama_write_systemd_service
}

ollama_setup() {
    local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}"
    local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
    local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
    local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}"
    local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}"
    local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}"
    local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}"
    local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}"
    local arch
    local tmp_installer
    local curl_ip_args=()

    require_debian_server "ollama-setup"

    if ! command -v curl >/dev/null 2>&1; then
        echo "curl is required to install Ollama." >&2
        exit 1
    fi
    if [[ -n "${curl_ip_version}" ]]; then
        curl_ip_args+=("${curl_ip_version}")
    fi

    if ! command -v ollama >/dev/null 2>&1; then
        if [[ -n "${local_tarball}" ]]; then
            ollama_install_from_tgz "${local_tarball}"
        else
            echo "Installing Ollama from ${installer_url}..."
            tmp_installer="$(mktemp)"
            if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then
                rm -f "${tmp_installer}"
                arch="$(ollama_linux_arch || true)"
                cat >&2 <<EOF
Could not download the Ollama installer from ${installer_url}.

If this Debian host cannot reach ollama.com, download the Linux tarball from a
machine with internet access and copy it to the Debian host:

  curl -L https://ollama.com/download/ollama-linux-${arch:-amd64}.tgz -o /tmp/ollama-linux-${arch:-amd64}.tgz
  scp /tmp/ollama-linux-${arch:-amd64}.tgz jv@${LAB_DEBIAN_LAN_IP:-192.168.100.73}:/tmp/

Then run on Debian:

  LAB_OLLAMA_TARBALL_FILE=/tmp/ollama-linux-${arch:-amd64}.tgz ./jeannie ollama-setup
EOF
                exit 1
            fi
            sudo sh "${tmp_installer}"
            rm -f "${tmp_installer}"
        fi
    else
        echo "Ollama is already installed: $(command -v ollama)"
    fi

    website_ollama_listen

    echo "Ensuring Ollama model ${model} is available..."
    if ollama list | awk -v model="${model}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
        echo "Ollama model ${model} is already present."
    else
        OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}"
    fi

    echo "Ollama homelab setup is ready."
}

require_argocd_application_controller() {
    local namespace="${TF_VAR_argocd_namespace:-argocd}"
    local expected_resources="10m|1|2Gi"
    local actual_resources

    if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then
        echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2
        kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide >&2 || true
        kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true
        exit 1
    fi

    actual_resources="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get statefulset argocd-application-controller \
        -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')"
    if [[ "${actual_resources}" != "${expected_resources}" ]]; then
        echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2
        echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2
        exit 1
    fi
}

require_pimox_app_worker_capacity() {
    local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}"
    local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
    local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}"
    local worker_key
    local worker_host
    local worker_user
    local worker_node
    local worker_key_path
    local capacity
    local ready
    local failures=0

    if disabled_value "${enabled}"; then
        echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}."
        return 0
    fi
    if [[ ! -s "${spec_file}" ]]; then
        echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate."
        return 0
    fi
    if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then
        echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2
        exit 1
    fi

    while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do
        [[ -n "${worker_node}" ]] || continue
        capacity="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)"
        ready="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
        if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then
            echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2
            failures=$((failures + 1))
        fi
    done <"${spec_file}"

    if ((failures > 0)); then
        echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2
        kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true
        exit 1
    fi
}

apps() {
    local buildx_builder_ready=false
    local demos_image_built=false
    local demos_image_ref
    local demos_image_state_file
    local demos_platforms
    local demos_registry_endpoint
    local demos_source_hash
    local registry_endpoint
    local website_image_built=false
    local website_image_ref
    local website_image_state_file
    local website_platforms
    local website_source_hash

    require_debian_server "apps"

    registry_endpoint="$(apps_registry_endpoint)"
    demos_registry_endpoint="$(demos_registry_endpoint)"
    demos_image_ref="${registry_endpoint}/demos-static:latest"
    demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state"
    demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
    demos_source_hash="$(demos_source_hash)"
    website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state"
    website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
    website_source_hash="$(website_source_hash)"
    website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}"
    export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}"
    export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}"
    export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
    export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"

    require_argocd_application_controller
    require_pimox_app_worker_capacity

    ensure_cosign_keypair
    publish_cosign_public_key_config_map

    if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then
        echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2
        exit 1
    fi

    if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then
        echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2
        exit 1
    fi

    if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then
        echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2
        exit 1
    fi

    if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then
        echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2
        exit 1
    fi
    ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}"

    echo "Deploying homelab applications..."

    run_tofu_stack "bootstrap/apps"

    refresh_argocd_application container-registry
    refresh_argocd_application supply-chain-policy

    wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300
    wait_for_namespace container-registry container-registry 300
    wait_for_namespaced_resource container-registry deployment local-registry container-registry 300
    wait_for_deployment_ready container-registry local-registry container-registry 300

    if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" &&
        image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then
        echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build."
    else
        echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..."
        ensure_docker_build_space
        if [[ "${buildx_builder_ready}" != "true" ]]; then
            prepare_buildx_builder "${registry_endpoint}"
            buildx_builder_ready=true
        fi

        docker buildx build \
            --network host \
            --platform "${website_platforms}" \
            --provenance=false \
            --sbom=true \
            --label "dev.homelab.website.source-hash=${website_source_hash}" \
            -t "${website_image_ref}" \
            -f "${REPO_ROOT}/apps/website/Dockerfile" \
            "${REPO_ROOT}/apps/website/" \
            --push
        website_image_built=true
    fi
    publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}"

    if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" &&
        image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then
        echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build."
    else
        echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..."
        ensure_docker_build_space
        if [[ "${buildx_builder_ready}" != "true" ]]; then
            prepare_buildx_builder "${registry_endpoint}"
            buildx_builder_ready=true
        fi

        docker buildx build \
            --network host \
            --platform "${demos_platforms}" \
            --provenance=false \
            --sbom=true \
            --label "dev.homelab.demos.source-hash=${demos_source_hash}" \
            -t "${demos_image_ref}" \
            -f "${REPO_ROOT}/apps/demos-static/Dockerfile" \
            "${REPO_ROOT}/apps/demos-static/" \
            --push
        demos_image_built=true
    fi
    publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}"

    refresh_argocd_application website-production
    wait_for_namespace website-production website-production 300
    wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300
    if [[ "${website_image_built}" == "true" ]]; then
        recreate_pods_for_selector website-production app=php-website website-production
    else
        echo "Skipping website pod restart because the image did not change."
    fi
    wait_for_deployment_ready website-production php-website-deployment website-production 300
    if [[ "${website_image_built}" == "true" ]]; then
        write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}"
    fi

    refresh_argocd_application demos-static
    wait_for_namespace demos-static demos-static 300
    wait_for_namespaced_resource demos-static deployment demos-static demos-static 300
    if [[ "${demos_image_built}" == "true" ]]; then
        recreate_pods_for_selector demos-static app=demos-static demos-static
    else
        echo "Skipping demos pod restart because the image did not change."
    fi
    wait_for_deployment_ready demos-static demos-static demos-static 300
    if [[ "${demos_image_built}" == "true" ]]; then
        write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}"
    fi

    refresh_argocd_application heimdall
    wait_for_namespaced_resource monitoring ingress grafana heimdall 300
    wait_for_namespaced_resource monitoring ingress prometheus heimdall 300
    wait_for_namespaced_resource monitoring ingress alertmanager heimdall 300
    wait_for_namespaced_resource argocd ingress argocd-server heimdall 300

    refresh_argocd_application n8n
    wait_for_namespace n8n n8n 300
    wait_for_namespaced_resource n8n deployment n8n n8n 300
    wait_for_deployment_ready n8n n8n n8n 300

    echo "Application deployment successfully completed."
}

platform_apply() {
    require_debian_server "platform"

    run_tofu_stack "bootstrap/platform"
}

edge_apply() {
    require_debian_server "edge"

    run_tofu_stack "bootstrap/edge"
}

edge_haproxy_stats_password_valid() {
    local password="${1:-}"

    [[ -n "${password}" ]] && ((${#password} >= 12)) && [[ "${password}" != "adminpassword" ]]
}

ensure_edge_tailscale_routes() {
    local enabled="${LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES:-true}"
    local route="${LAB_EDGE_TAILSCALE_SUBNET_ROUTE:-${LAB_LAN_CIDR:-192.168.100.0/24}}"
    local router="${LAB_EDGE_TAILSCALE_ROUTER:-debian}"
    local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
    local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
    local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}"
    local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}"
    local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"

    if disabled_value "${enabled}"; then
        return 0
    fi
    if ! truthy "${enabled}"; then
        echo "LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES must be true or false." >&2
        exit 1
    fi
    if ! validate_ipv4_cidr_or_host "${route}"; then
        echo "Invalid LAB_EDGE_TAILSCALE_SUBNET_ROUTE '${route}'." >&2
        exit 1
    fi
    if [[ -z "${edge_host}" ]]; then
        echo "LAB_EDGE_HOST or TF_VAR_edge_host is required to configure edge Tailscale routes." >&2
        exit 1
    fi

    case "${router}" in
        debian)
            echo "Ensuring Debian advertises ${route} and OCI edge accepts Tailscale routes..."
            if ! command -v tailscale >/dev/null 2>&1; then
                echo "tailscale is not installed on the Debian subnet router." >&2
                exit 1
            fi
            sudo mkdir -p /etc/sysctl.d
            printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null
            sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
            sudo tailscale set --advertise-routes="${route}"
            ;;
        rpi | rpi4 | raspberrypi)
            echo "Ensuring RPi advertises ${route} and OCI edge accepts Tailscale routes..."
            ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
if ! command -v tailscale >/dev/null 2>&1; then
  echo 'tailscale is not installed on the RPi subnet router.' >&2
  exit 1
fi
sysctl_bin=\"\$(command -v sysctl 2>/dev/null || true)\"
if [ -z \"\$sysctl_bin\" ] && [ -x /usr/sbin/sysctl ]; then
  sysctl_bin=/usr/sbin/sysctl
fi
if [ -z \"\$sysctl_bin\" ]; then
  echo 'sysctl is not installed on the RPi subnet router.' >&2
  exit 1
fi
sudo mkdir -p /etc/sysctl.d
printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null
sudo \"\$sysctl_bin\" -w net.ipv4.ip_forward=1 >/dev/null
sudo tailscale set --advertise-routes='${route}'
"
            ;;
        *)
            echo "LAB_EDGE_TAILSCALE_ROUTER must be debian or rpi." >&2
            exit 1
            ;;
    esac

    ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu
if ! command -v tailscale >/dev/null 2>&1; then
  echo 'tailscale is not installed on the OCI edge host.' >&2
  exit 1
fi
sudo tailscale set --accept-routes=true
"
}

edge_haproxy_stats_credentials_file() {
    printf '%s\n' "${LAB_EDGE_HAPROXY_STATS_CREDENTIALS_FILE:-${HOME}/.config/homelab/edge-haproxy.env}"
}

generate_edge_haproxy_stats_password() {
    if command -v openssl >/dev/null 2>&1; then
        openssl rand -hex 24
    elif command -v python3 >/dev/null 2>&1; then
        python3 - <<'PY'
import secrets

print(secrets.token_hex(24))
PY
    else
        echo "openssl or python3 is required to generate an edge HAProxy stats password." >&2
        exit 1
    fi
}

write_edge_haproxy_stats_credentials() {
    local credentials_file="$1"
    local password="$2"
    local credentials_dir

    credentials_dir="$(dirname "${credentials_file}")"
    mkdir -p "${credentials_dir}"
    chmod 0700 "${credentials_dir}"
    {
        printf 'TF_VAR_haproxy_stats_password='
        printf '%q' "${password}"
        printf '\n'
    } > "${credentials_file}"
    chmod 0600 "${credentials_file}"
}

recover_edge_haproxy_stats_password() {
    local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}"
    local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}"
    local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
    local edge_install_dir="${TF_VAR_edge_install_dir:-${LAB_EDGE_INSTALL_DIR:-/opt/homelab-edge}}"
    local remote_auth
    local remote_password

    if [[ -z "${edge_host}" || "${edge_install_dir}" == *"'"* ]]; then
        return 1
    fi

    remote_auth="$(ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
        "sudo sed -n 's/^[[:space:]]*stats auth //p' '${edge_install_dir}/config_files/haproxy.cfg' | head -n 1" 2>/dev/null || true)"
    remote_auth="${remote_auth//$'\r'/}"
    if [[ "${remote_auth}" != *:* ]]; then
        return 1
    fi

    remote_password="${remote_auth#*:}"
    if ! edge_haproxy_stats_password_valid "${remote_password}"; then
        return 1
    fi

    printf '%s\n' "${remote_password}"
}

ensure_edge_haproxy_stats_password() {
    local credentials_file
    local recovered_password

    if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then
        export TF_VAR_haproxy_stats_password
        return 0
    fi

    credentials_file="$(edge_haproxy_stats_credentials_file)"
    if [[ -r "${credentials_file}" ]]; then
        # shellcheck disable=SC1090
        source "${credentials_file}"
        if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then
            export TF_VAR_haproxy_stats_password
            return 0
        fi
        echo "Ignoring invalid edge HAProxy stats credentials in ${credentials_file}." >&2
    fi

    if recovered_password="$(recover_edge_haproxy_stats_password)"; then
        TF_VAR_haproxy_stats_password="${recovered_password}"
        export TF_VAR_haproxy_stats_password
        write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}"
        echo "Recovered edge HAProxy stats password from the live edge host and stored it at ${credentials_file}."
        return 0
    fi

    TF_VAR_haproxy_stats_password="$(generate_edge_haproxy_stats_password)"
    export TF_VAR_haproxy_stats_password
    write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}"
    echo "Generated edge HAProxy stats password at ${credentials_file}."
}

ensure_cluster_worker_var_file() {
    if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
        prepare_cluster_worker_var_file false
    fi
    if truthy "${TF_VAR_allow_empty_worker_nodes:-false}"; then
        return 0
    fi
    if ! cluster_worker_var_file_has_workers "${LAB_CLUSTER_VAR_FILE}"; then
        echo "Cluster worker var file has no worker_nodes: ${LAB_CLUSTER_VAR_FILE}" >&2
        echo "Run the Pimox worker stage first or set LAB_PIMOX_WORKER_COUNT and LAB_PIMOX_WORKER_STATIC_IPS." >&2
        return 1
    fi
}

run_up_parallel_stage() {
    local spec_file
    local parallelism="${JEANNIE_UP_PARALLELISM:-4}"
    local status

    if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then
        echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2
        return 1
    fi

    spec_file="$(mktemp)"
    python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY'
import json
import sys

spec_file, jeannie, parallelism = sys.argv[1:4]
spec = {
    "parallelism": int(parallelism),
    "tasks": [
        {
            "id": "gitea-deploy",
            "title": "Gitea deploy",
            "command": [jeannie, "__up-task", "gitea-deploy"],
        },
        {
            "id": "gitea-bootstrap",
            "title": "Gitea repo bootstrap",
            "needs": ["gitea-deploy"],
            "command": [jeannie, "__up-task", "gitea-bootstrap"],
        },
        {
            "id": "heimdall-deploy",
            "title": "Heimdall deploy",
            "command": [jeannie, "__up-task", "heimdall-deploy"],
        },
        {
            "id": "matrix-deploy",
            "title": "Matrix deploy",
            "command": [jeannie, "__up-task", "matrix-deploy"],
        },
        {
            "id": "omniroute-deploy",
            "title": "OmniRoute deploy",
            "command": [jeannie, "__up-task", "omniroute-deploy"],
        },
        {
            "id": "hermes-deploy",
            "title": "Hermes deploy",
            "needs": ["omniroute-deploy", "matrix-deploy"],
            "command": [jeannie, "__up-task", "hermes-deploy"],
        },
        {
            "id": "rpi-services",
            "title": "RPi services",
            "command": [jeannie, "__up-task", "rpi-services"],
        },
        {
            "id": "pimox-workers",
            "title": "Pimox provisioning and workers",
            "command": [jeannie, "__up-task", "pimox-workers"],
        },
        {
            "id": "openwrt",
            "title": "OpenWrt VM",
            "command": [jeannie, "__up-task", "openwrt"],
        },
        {
            "id": "edge-host",
            "title": "OCI edge host check",
            "command": [jeannie, "__up-task", "edge-host"],
        },
    ],
}
with open(spec_file, "w", encoding="utf-8") as handle:
    json.dump(spec, handle, indent=2)
    handle.write("\n")
PY
    set +e
    "${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}"
    status=$?
    set -e
    rm -f "${spec_file}"
    return "${status}"
}

graphiphy_map() {
    if ! command -v graphiphy >/dev/null 2>&1; then
        echo "graphiphy is not installed; skipping repo map generation."
        return 0
    fi

    graphiphy "${REPO_ROOT}"
}

up_task() {
    require_debian_server "__up-task"

    case "${1:-}" in
        gitea-deploy)
            deploy_gitea
            ;;
        gitea-bootstrap)
            bootstrap_gitea_repo
            ;;
        heimdall-deploy)
            deploy_heimdall
            ;;
        matrix-deploy)
            deploy_matrix
            ;;
        omniroute-deploy)
            deploy_omniroute
            ;;
        hermes-deploy)
            deploy_hermes
            ;;
        rpi-services)
            deploy_rpi_services
            ;;
        pimox-workers)
            run_pimox_pipeline
            ;;
        openwrt)
            run_openwrt_pipeline
            ;;
        edge-host)
            check_edge_ssh
            ;;
        *)
            echo "Unknown __up-task '${1:-}'." >&2
            return 1
            ;;
    esac
}

up() {
    require_debian_server "up"

    echo "Deploying the homelab infrastructure..."
    jeannie_log_start "up"
    jeannie_step_plan 12

    run_step "Early preflight" homelab_preflight early
    run_step "Go toolchain" ensure_go_toolchain
    run_step "Repo Map (Graphiphy)" graphiphy_map
    run_step "Independent host services" run_up_parallel_stage
    run_step "Full preflight" homelab_preflight full
    run_step "Pre-apply doctor" doctor_preapply
    run_step "Worker var file" ensure_cluster_worker_var_file
    run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
    run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
    run_step "Version report" doctor_versions_report
    run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
    run_step "Applications" apps
    run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"

    echo "Deployment successfully completed."
    echo "Log: ${JEANNIE_LOG_FILE}"
}

tofu_stack_from_plan_target() {
    case "$1" in
        provisioning)
            printf 'bootstrap/provisioning\n'
            ;;
        cluster)
            printf 'bootstrap/cluster\n'
            ;;
        platform)
            printf 'bootstrap/platform\n'
            ;;
        apps)
            printf 'bootstrap/apps\n'
            ;;
        edge)
            printf 'bootstrap/edge\n'
            ;;
        *)
            echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2
            return 1
            ;;
    esac
}