From 0bdce094977ca88db0d702d2031e72e07b7670fe Mon Sep 17 00:00:00 2001 From: jv Date: Sat, 19 Sep 2026 01:25:36 -0500 Subject: [PATCH] fix: restore corrupted jeannie script and correct KUBECONFIG variable --- jeannie | 8334 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 8319 insertions(+), 15 deletions(-) diff --git a/jeannie b/jeannie index d747bf7..9826600 100755 --- a/jeannie +++ b/jeannie @@ -27,11 +27,320 @@ source "${REPO_ROOT}/scripts/report-ui" trap 'rm -f "${BUILDX_CONFIG}"' EXIT +load_homelab_inventory_defaults() { + local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" + local rendered_defaults + + if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then + return 0 + fi + + rendered_defaults="$(python3 - "${inventory_file}" <<'PY' +import re +import json +import shlex +import sys + +inventory_file = sys.argv[1] +mapping = { + "domain.base": "LAB_DOMAIN", + "domain.public_url": "LAB_PUBLIC_URL", + "domain.gitea_url": "LAB_GITEA_ROOT_URL", + "network.lan_cidr": "LAB_LAN_CIDR", + "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX", + "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP", + "hosts.debian.user": "LAB_DEBIAN_USER", + "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP", + "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP", + "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT", + "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH_PATH", + "hosts.rpi4.user": "LAB_RPI_USER", + "hosts.rpi4.lan_ip": "LAB_RPI_HOST", + "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", + "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT", + "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", + "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", + "hosts.opi5_pimox.user": "LAB_PIMOX_USER", + "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST", + "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE", + "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE", + "hosts.oci_edge.user": "LAB_EDGE_USER", + "hosts.oci_edge.public_ip": "LAB_EDGE_HOST", + "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR", + "services.gitea.http_port": "LAB_GITEA_HTTP_PORT", + "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", + "services.gitea.root_url": "LAB_GITEA_ROOT_URL", + "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", + "services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR", + "services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT", + "services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL", + "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", + "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS", + "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR", + "services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE", + "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT", + "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT", + "ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER", + "ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED", + "ai_gateway.url": "LAB_AI_GATEWAY_URL", + "ai_gateway.model": "LAB_AI_GATEWAY_MODEL", + "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS", + "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR", + "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", + "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", +} + +def flatten(document, prefix=""): + values = {} + if isinstance(document, dict): + for key, value in document.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict): + values.update(flatten(value, path)) + elif isinstance(value, list): + values[path] = ",".join(str(item) for item in value) + elif value is not None: + values[path] = str(value) + return values + + +def parse_simple_inventory(path): + stack = [] + values = {} + pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") + + with open(path, encoding="utf-8") as handle: + for raw_line in handle: + if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): + continue + match = pattern.match(raw_line.rstrip("\n")) + if not match: + continue + indent = len(match.group(1)) + key = match.group(2) + value = (match.group(3) or "").strip() + while stack and stack[-1][0] >= indent: + stack.pop() + current_path = ".".join([item[1] for item in stack] + [key]) + if value == "": + stack.append((indent, key)) + continue + if " #" in value: + value = value.split(" #", 1)[0].strip() + values[current_path] = value.strip("\"'") + return values + + +try: + import yaml +except ImportError: + values = parse_simple_inventory(inventory_file) +else: + with open(inventory_file, encoding="utf-8") as handle: + values = flatten(yaml.safe_load(handle) or {}) + +for path, env_name in mapping.items(): + value = values.get(path) + if value: + print(f": ${{{env_name}:={shlex.quote(value)}}}") + +debian_ip = values.get("hosts.debian.lan_ip") +gitea_http = values.get("services.gitea.http_port") +if debian_ip: + print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}") +if debian_ip and gitea_http: + print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}") +gitea_ts = values.get("hosts.debian.tailscale_ip") +if gitea_ts: + print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}") +subdomains = values.get("domain.subdomains") +if subdomains: + if isinstance(subdomains, str): + subdomain_values = [item.strip() for item in subdomains.split(",") if item.strip()] + else: + subdomain_values = list(subdomains) + if subdomain_values: + print(f": ${{LAB_ADDITIONAL_SERVER_NAMES_JSON:={shlex.quote(json.dumps(subdomain_values))}}}") +PY +)" + + if [[ -n "${rendered_defaults}" ]]; then + eval "${rendered_defaults}" + fi +} + load_homelab_inventory_defaults +export_if_unset() { + local name="$1" + local value="$2" + + if [[ -z "${value}" ]]; then + return 0 + fi + if [[ -z "${!name:-}" ]]; then + printf -v "${name}" '%s' "${value}" + fi + # shellcheck disable=SC2163 + export "${name?}" +} + +export_homelab_inventory_tf_vars() { + export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH_PATH:-}" + export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}" + export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}" + export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}" + export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}" + export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}" + export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}" + export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}" + export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}" + export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}" + export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}" + export_if_unset TF_VAR_additional_server_names "${LAB_ADDITIONAL_SERVER_NAMES_JSON:-}" + export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" + export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" + export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" + export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" + export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}" + export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" + export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}" + export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}" + export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}" +} + export_homelab_inventory_tf_vars -KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}" +KUBECONFIG_PATH_PATH="${KUBECONFIG_PATH_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH_PATH:-/home/jv/.kube/config}}}" + +require_debian_server() { + local command_name="$1" + local os_id="" + + if [[ "$(uname -s)" != "Linux" ]]; then + echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2 + exit 1 + fi + + if [[ -r /etc/os-release ]]; then + os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)" + fi + + if [[ "${os_id}" != "debian" ]]; then + echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2 + exit 1 + fi +} + +tofu_state_has_resource() { + local stack="$1" + local resource_address="$2" + + tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1 +} + +helm_release_secret_exists() { + local namespace="$1" + local release_name="$2" + local secret_name + + secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get secrets \ + -l "owner=helm,name=${release_name}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + + [[ -n "${secret_name}" ]] +} + +kubernetes_resource_exists() { + local namespace="$1" + local resource_kind="$2" + local resource_name="$3" + + if [[ -n "${namespace}" ]]; then + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 + return $? + fi + + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 +} + +adopt_tofu_helm_release() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local release_name="$4" + local has_state=false + local has_release_secret=false + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + has_state=true + fi + if helm_release_secret_exists "${namespace}" "${release_name}"; then + has_release_secret=true + fi + + if [[ "${has_state}" == "true" && "${has_release_secret}" == "false" ]]; then + echo "Removing stale Helm release state for ${namespace}/${release_name} from ${stack} (${resource_address}) because no Helm release secret exists..." + tofu -chdir="${REPO_ROOT}/${stack}" state rm "${resource_address}" + return 0 + fi + + if [[ "${has_state}" == "true" ]]; then + return 0 + fi + if [[ "${has_release_secret}" == "false" ]]; then + return 0 + fi + + echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}" +} + +adopt_tofu_kubernetes_resource() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local resource_kind="$4" + local resource_name="$5" + local import_id="$6" + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + return 0 + fi + if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then + return 0 + fi + + echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" +} + +adopt_tofu_kubernetes_manifest() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local kubectl_kind="$4" + local api_version="$5" + local manifest_kind="$6" + local resource_name="$7" + local import_id + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + return 0 + fi + if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then + return 0 + fi + + import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}" + echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" +} adopt_platform_existing_resources() { local stack="bootstrap/platform" @@ -129,7 +438,7 @@ ensure_homelab_node_labels() { [[ -n "${node}" ]] || continue if [[ "${node}" == "${control_plane_node}" ]]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \ homelab.dev/node-role=control-plane \ homelab.dev/storage=local \ homelab.dev/workload-class=control-plane \ @@ -137,26 +446,26 @@ ensure_homelab_node_labels() { continue fi - kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \ node-role.kubernetes.io/worker=worker \ --overwrite if [[ "${node}" == pimox-worker-* ]]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \ homelab.dev/node-role=app \ homelab.dev/storage=ssd \ homelab.dev/workload-class=platform \ --overwrite elif [[ "${node}" == "${raspberry_node}" ]]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \ homelab.dev/node-role=edge-app \ homelab.dev/storage=local \ homelab.dev/workload-class=edge \ --overwrite fi - done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + done < <(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') - target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)" + target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -l "${prometheus_selector}" -o name)" if [[ -z "${target_nodes}" ]]; then echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2 exit 1 @@ -169,32 +478,7228 @@ delete_prometheus_stack_storage() { local pvc_names local pv_names - pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null | + pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null | awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')" - pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \ + pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get pv \ -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null | awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')" if [[ -n "${pvc_names}" ]]; then echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..." printf '%s\n' "${pvc_names}" | - xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s + xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s fi if [[ -n "${pv_names}" ]]; then echo "Deleting old prometheus-stack retained PV objects..." printf '%s\n' "${pv_names}" | - xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false + xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" delete pv --wait=false fi } run_tofu_stack() { local stack="$1" - local auto_approve="${L + local auto_approve="${LAB_AUTO_APPROVE:-true}" + local -a apply_args=() -... [OUTPUT TRUNCATED - 267,570 chars omitted out of 317,495 total] ... + if [[ "${stack}" == "bootstrap/cluster" ]]; then + ensure_cluster_worker_var_file + fi + if [[ "${stack}" == "bootstrap/edge" ]]; then + ensure_edge_tailscale_routes + ensure_edge_haproxy_stats_password + fi -s not installed. Run ./jeannie secrets-init first." >&2 + if truthy "${auto_approve}"; then + apply_args+=("-auto-approve") + elif ! disabled_value "${auto_approve}"; then + echo "LAB_AUTO_APPROVE must be true or false." >&2 + exit 1 + fi + + if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then + apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") + fi + + tofu -chdir="${REPO_ROOT}/${stack}" init + ensure_kubernetes_api_ready_for_tofu_stack "${stack}" + if [[ "${stack}" == "bootstrap/platform" ]]; then + adopt_platform_existing_resources + fi + if [[ "${stack}" == "bootstrap/apps" ]]; then + adopt_apps_existing_resources + fi + ensure_kubernetes_api_ready_for_tofu_stack "${stack}" + tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}" +} + +run_tofu_plan_stack() { + local stack="$1" + local -a plan_args=() + + if [[ "${stack}" == "bootstrap/cluster" ]]; then + ensure_cluster_worker_var_file + fi + + if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then + plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") + fi + + tofu -chdir="${REPO_ROOT}/${stack}" init + tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}" +} + +backup_tofu_state() { + local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}" + local timestamp + local archive + local -a paths=() + local path + + timestamp="$(date +%Y%m%d-%H%M%S)" + archive="${backup_dir}/tofu-state-${timestamp}.tgz" + mkdir -p "${backup_dir}" + + for path in \ + bootstrap/provisioning/terraform.tfstate \ + bootstrap/provisioning/terraform.tfstate.backup \ + bootstrap/cluster/terraform.tfstate \ + bootstrap/cluster/terraform.tfstate.backup \ + bootstrap/platform/terraform.tfstate \ + bootstrap/platform/terraform.tfstate.backup \ + bootstrap/apps/terraform.tfstate \ + bootstrap/apps/terraform.tfstate.backup \ + bootstrap/edge/terraform.tfstate \ + bootstrap/edge/terraform.tfstate.backup \ + .lab/cluster-workers.auto.tfvars.json \ + .lab/pimox-workers.tsv \ + .lab/manual-workers.tsv; do + if [[ -e "${REPO_ROOT}/${path}" ]]; then + paths+=("${path}") + fi + done + + if ((${#paths[@]} == 0)); then + echo "No local OpenTofu state files found to back up." + return 0 + fi + + tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}" + chmod 0600 "${archive}" + echo "Backed up local OpenTofu state to ${archive}." +} + +move_prometheus_stack_workers() { + local stack="bootstrap/platform" + local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}" + local auto_approve="${LAB_AUTO_APPROVE:-true}" + local -a approve_args=() + + require_debian_server "move-prometheus-stack-workers" + + if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then + cat >&2 <<'EOF' +move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data. +Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended. +EOF + exit 1 + fi + + if truthy "${auto_approve}"; then + approve_args+=("-auto-approve") + elif ! disabled_value "${auto_approve}"; then + echo "LAB_AUTO_APPROVE must be true or false." >&2 + exit 1 + fi + + export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH_PATH}}" + export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}" + + echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted." + ensure_homelab_node_labels + tofu -chdir="${REPO_ROOT}/${stack}" init + adopt_platform_existing_resources + tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}" + delete_prometheus_stack_storage "${namespace}" + tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}" + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get pods -o wide +} + +doctor_versions_report() { + local api_version + local api_minor + local containerd_major_count + local kubelet_minor_count + local node_rows + local status=0 + + require_debian_server "doctor-versions" + ensure_python3 + + api_version="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" version -o json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' + )" + api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" + node_rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' + )" + + if [[ -z "${node_rows}" ]]; then + echo "No Kubernetes nodes found." >&2 + return 1 + fi + + printf 'API server: %s\n\n' "${api_version}" + printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME" + printf '%s\n' "${node_rows}" | + awk -F '\t' '{ + version = $2 + gsub(/^v/, "", version) + split(version, parts, ".") + printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3 + }' + + kubelet_minor_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | + sort -u | + wc -l | + tr -d ' ' + )" + containerd_major_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | + sort -u | + wc -l | + tr -d ' ' + )" + + if [[ "${kubelet_minor_count}" != "1" ]]; then + echo + echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2 + status=1 + fi + + if printf '%s\n' "${node_rows}" | + awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then + echo + echo "At least one kubelet does not match the API server minor (${api_minor})." >&2 + status=1 + fi + + if [[ "${containerd_major_count}" -gt 1 ]]; then + echo + echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2 + status=1 + fi + + if [[ "${status}" -eq 0 ]]; then + echo + echo "Node Kubernetes and containerd versions are aligned." + fi + + return "${status}" +} + +doctor_versions_compact() { + local api_version + local api_minor + local node_rows + local kubelet_minor_count + local containerd_major_count + local mismatch_count + local failures=0 + + ensure_python3 + report_ui_begin + + if ! status_kubernetes_api_ok; then + report_ui_emit fail "Kubernetes" "API" "not reachable" "kubectl readyz failed" "./jeannie start-cluster" + report_ui_render "Doctor Versions" + report_ui_cleanup + return 1 + fi + + api_version="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" version -o json | + python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' + )" + api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" + node_rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' + )" + + if [[ -z "${node_rows}" ]]; then + report_ui_emit fail "Kubernetes" "Nodes" "none found" "kubectl returned no node rows" "./jeannie doctor-cluster" + report_ui_render "Doctor Versions" + report_ui_cleanup + return 1 + fi + + kubelet_minor_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | + sort -u | + wc -l | + tr -d ' ' + )" + mismatch_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) count++ } END { print count + 0 }' + )" + containerd_major_count="$( + printf '%s\n' "${node_rows}" | + awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | + sort -u | + wc -l | + tr -d ' ' + )" + + report_ui_emit ok "Kubernetes" "API version" "${api_version}" + if [[ "${kubelet_minor_count}" != "1" || "${mismatch_count}" != "0" ]]; then + report_ui_emit fail "Kubernetes" "Kubelet versions" "${mismatch_count} node(s) differ from ${api_minor}" "unique kubelet minors=${kubelet_minor_count}" "./jeannie doctor-versions --verbose" + failures=$((failures + 1)) + else + report_ui_emit ok "Kubernetes" "Kubelet versions" "aligned to ${api_minor}" + fi + if [[ "${containerd_major_count}" -gt 1 ]]; then + report_ui_emit warn "Runtime" "Containerd versions" "mixed major versions" "unique majors=${containerd_major_count}" "./jeannie doctor-versions --verbose" + else + report_ui_emit ok "Runtime" "Containerd versions" "single major version" + fi + + report_ui_render "Doctor Versions" + report_ui_cleanup + return "${failures}" +} + +doctor_versions() { + require_debian_server "doctor-versions" + if ! report_ui_parse_args "$@"; then + return 1 + fi + if [[ "${REPORT_UI_VERBOSE}" == "true" ]]; then + doctor_versions_report + exit "$?" + fi + doctor_versions_compact + exit "$?" +} + + + + + + + + + + + + +worker_index_is_skipped() { + local index="$1" + local skip_indexes="$2" + local skip_index + + skip_indexes="${skip_indexes//,/ }" + for skip_index in ${skip_indexes}; do + [[ -z "${skip_index}" ]] && continue + if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2 + exit 1 + fi + if ((skip_index == index)); then + return 0 + fi + done + + return 1 +} + +ensure_python3() { + if command -v python3 >/dev/null 2>&1; then + return 0 + fi + + sudo apt-get update + sudo apt-get install -y --no-install-recommends python3 +} + +ensure_go_toolchain() { + require_debian_server "go toolchain" + + if command -v go >/dev/null 2>&1; then + return 0 + fi + + echo "Installing Go toolchain for Jeannie core helpers..." + sudo apt-get update + sudo apt-get install -y --no-install-recommends golang-go + + if ! command -v go >/dev/null 2>&1; then + echo "go is still unavailable after installing golang-go." >&2 + exit 1 + fi +} + +detect_route_interface() { + local target="$1" + + ip route get "${target}" 2>/dev/null | awk ' + { + for (i = 1; i <= NF; i++) { + if ($i == "dev") { + print $(i + 1) + exit + } + } + } + ' +} + +pimox_ssh() { + local host="$1" + local user="$2" + local key_path="$3" + + shift 3 + ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@" +} + +pimox_guest_ipv4() { + local guest_json + local host="$1" + local user="$2" + local key_path="$3" + local vmid="$4" + local ip_prefix="$5" + local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" + + guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)" + if [[ -z "${guest_json}" ]]; then + return 1 + fi + + GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY' +import json +import os +import sys + +prefix = sys.argv[1] +try: + interfaces = json.loads(os.environ.get("GUEST_JSON", "")) +except Exception: + sys.exit(1) + +for iface in interfaces or []: + for address in iface.get("ip-addresses") or []: + if address.get("ip-address-type") != "ipv4": + continue + ip = address.get("ip-address", "") + if not ip or ip.startswith(("127.", "169.254.")): + continue + if prefix and not ip.startswith(prefix): + continue + print(ip) + sys.exit(0) +sys.exit(1) +PY +} + +pimox_worker_vm_debug() { + local host="$1" + local user="$2" + local key_path="$3" + local vmid="$4" + local qm_bin="$5" + local script + local encoded_script + + script="$(cat <<'EOF' +set +e +echo "-- addresses --" +ip -br addr +echo "-- routes --" +ip route +echo "-- units --" +systemctl is-active ssh sshd 2>/dev/null || true +echo "-- listeners --" +ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true +echo "-- sshd config test --" +sudo sshd -t 2>&1 || true +echo "-- recent ssh logs --" +journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true +EOF +)" + encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" + pimox_ssh "${host}" "${user}" "${key_path}" "set +e +echo 'Pimox VM ${vmid} status:' +sudo '${qm_bin}' status '${vmid}' +echo 'Pimox VM ${vmid} config summary:' +sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true +echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:' +sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true +echo 'Pimox VM ${vmid} guest SSH diagnostics:' +sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\" || true" >&2 || true +} + +pimox_worker_guest_agent_recovery_hint() { + local host="$1" + local user="$2" + local vmid="$3" + local qm_bin="$4" + + cat >&2 </dev/null | grep -q '^ssh[.]service'; then + ssh_unit=ssh.service +elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then + ssh_unit=sshd.service +else + echo "Neither ssh.service nor sshd.service exists in the guest" >&2 + exit 1 +fi + +sudo ssh-keygen -A +sudo install -d -m 0755 /run/sshd +sudo mkdir -p /etc/ssh/sshd_config.d +sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' +Port 22 +ListenAddress 0.0.0.0 +PubkeyAuthentication yes +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin no +UsePAM yes +SSHD_CONFIG + +sudo sshd -t +sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true +sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true +sudo systemctl enable "$ssh_unit" >/dev/null +sudo systemctl restart "$ssh_unit" +sleep 2 +sudo systemctl is-active "$ssh_unit" +if command -v ss >/dev/null 2>&1; then + ss -ltnp 2>/dev/null || ss -ltn + ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' +fi +EOF +)" + encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" + if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then + echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2 + printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 + return 1 + fi + + printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 + if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then + return 0 + fi + echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2 + if [[ -n "${repair_exitcode:-}" ]]; then + echo "Guest exit code: ${repair_exitcode}" >&2 + fi + return 1 +} + +wait_for_pimox_guest_ssh() { + local host="$1" + local user="$2" + local key_path="$3" + local vmid="$4" + local guest_user="$5" + local guest_key_path="$6" + local ip_prefix="$7" + local timeout_seconds="$8" + local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" + local deadline + local elapsed + local guest_ip + local ip_filter_description + local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" + local last_guest_ip="" + local last_known_hosts_ip="" + local last_ssh_output="" + local last_host_probe_output="" + local host_probe_failures=0 + local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-false}" + local next_log + local ssh_deadline=0 + local ssh_output + local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}" + local ssh_refused_since=0 + local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}" + local ssh_repair_attempted=false + + ip_filter_description="matching prefix ${ip_prefix}" + if [[ -z "${ip_prefix}" ]]; then + ip_filter_description="that is not loopback or link-local" + fi + if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then + echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2 + return 1 + fi + if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then + echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2 + return 1 + fi + if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then + echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2 + return 1 + fi + mkdir -p "$(dirname "${known_hosts_file}")" + touch "${known_hosts_file}" + chmod 0600 "${known_hosts_file}" + + deadline=$((SECONDS + timeout_seconds)) + next_log="${SECONDS}" + while ((SECONDS < deadline)); do + guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)" + if [[ -n "${guest_ip}" ]]; then + if ((ssh_deadline == 0)); then + ssh_deadline=$((SECONDS + ssh_timeout_seconds)) + elif ((SECONDS >= ssh_deadline)); then + break + fi + last_guest_ip="${guest_ip}" + if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then + ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true + last_known_hosts_ip="${guest_ip}" + fi + if truthy "${require_host_probe}"; then + if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then + host_probe_failures=0 + else + host_probe_failures=$((host_probe_failures + 1)) + last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}" + if ((host_probe_failures >= 3)); then + echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2 + echo "Latest host probe: ${last_host_probe_output}" >&2 + echo "Fast checks:" >&2 + echo " ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2 + echo " arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2 + echo " Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2 + pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" + return 1 + fi + sleep 10 + continue + fi + fi + if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then + printf '%s\n' "${guest_ip}" + return 0 + fi + last_ssh_output="${ssh_output}" + if [[ "${ssh_output}" == *"Connection refused"* ]]; then + if ((ssh_refused_since == 0)); then + ssh_refused_since="${SECONDS}" + fi + if [[ "${ssh_repair_attempted}" == "false" ]]; then + echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2 + pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true + ssh_repair_attempted=true + elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then + echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2 + echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2 + pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" + return 1 + fi + else + ssh_refused_since=0 + fi + fi + + if ((SECONDS >= next_log)); then + elapsed=$((timeout_seconds - (deadline - SECONDS))) + if [[ -n "${last_guest_ip}" ]]; then + echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2 + if [[ -n "${last_ssh_output}" ]]; then + echo "Last SSH failure: ${last_ssh_output}" >&2 + fi + else + echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2 + fi + next_log=$((SECONDS + 60)) + fi + sleep 10 + done + + if [[ -n "${last_guest_ip}" ]]; then + echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2 + if [[ -n "${last_ssh_output}" ]]; then + echo "Last SSH failure: ${last_ssh_output}" >&2 + fi + if [[ -n "${last_host_probe_output}" ]]; then + echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2 + fi + else + echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2 + fi + pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" + + return 1 +} + +pimox_generated_mac() { + local vmid="$1" + + printf '02:68:10:%02x:%02x:%02x\n' \ + $(((vmid >> 16) & 255)) \ + $(((vmid >> 8) & 255)) \ + $((vmid & 255)) +} + +pimox_worker_static_ip() { + local index="$1" + local static_ips="$2" + local current_index=1 + local ip + + static_ips="${static_ips//,/ }" + for ip in ${static_ips}; do + if ((current_index == index)); then + printf '%s\n' "${ip}" + return 0 + fi + current_index=$((current_index + 1)) + done + + return 1 +} + +pimox_worker_net0_config() { + local vmid="$1" + local bridge="$2" + local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-deterministic}" + local mac + + case "${mac_mode}" in + auto) + printf 'virtio,bridge=%s\n' "${bridge}" + ;; + deterministic) + mac="$(pimox_generated_mac "${vmid}")" + printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}" + ;; + *) + echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2 + return 1 + ;; + esac +} + +validate_ipv4_cidr_or_host() { + local value="$1" + + [[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]] +} + +configure_pimox_worker_guest_network() { + local pimox_host="$1" + local pimox_user="$2" + local pimox_key="$3" + local vmid="$4" + local qm_bin="$5" + local worker_ip="$6" + local gateway="$7" + local dns_servers="$8" + local interface_name="$9" + local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}" + local deadline + local script + local encoded_script + local network_output + local network_exitcode + local worker_address + local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}" + local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-false}" + local lan_probe_block="" + + if ! validate_ipv4_cidr_or_host "${worker_ip}"; then + echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2 + return 1 + fi + if [[ "${worker_ip}" == */* && "${worker_ip}" != */24 ]]; then + echo "LAB_PIMOX_WORKER_STATIC_IPS currently supports host IPv4 values or /24 CIDRs, got '${worker_ip}'." >&2 + return 1 + fi + if [[ "${worker_ip}" != */* ]]; then + worker_ip="${worker_ip}/24" + fi + worker_address="${worker_ip%%/*}" + if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2 + return 1 + fi + if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2 + return 1 + fi + if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2 + return 1 + fi + if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then + echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2 + return 1 + fi + if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then + echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2 + return 1 + fi + + if truthy "${require_lan_probe}"; then + lan_probe_block="$(cat </dev/null 2>&1 || true +done +sudo pkill -x dhclient >/dev/null 2>&1 || true +sudo pkill -x dhcpcd >/dev/null 2>&1 || true +sudo systemctl enable networking >/dev/null 2>&1 || true +sudo ifdown --force ${interface_name} 2>/dev/null || true +sudo ip addr flush dev ${interface_name} scope global || true +sudo ip route flush dev ${interface_name} || true +sudo ip link set ${interface_name} up +sudo ip addr add ${worker_ip} dev ${interface_name} +sudo ip route replace default via ${gateway} dev ${interface_name} src ${worker_address} +printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null +sudo ifup --force ${interface_name} 2>/dev/null || true +sudo ssh-keygen -A +sudo install -d -m 0755 /run/sshd +sudo mkdir -p /etc/ssh/sshd_config.d +sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' +Port 22 +ListenAddress 0.0.0.0 +PubkeyAuthentication yes +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin no +UsePAM yes +SSHD_CONFIG +sudo sshd -t +sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true +sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true +ssh_unit=ssh.service +if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then + ssh_unit=sshd.service +fi +sudo systemctl enable "\$ssh_unit" >/dev/null +sudo systemctl restart "\$ssh_unit" +ip -br addr show ${interface_name} +ip route +ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/' +${lan_probe_block} +echo 'guest-check: ssh service' +sudo systemctl is-active "\$ssh_unit" +if command -v ss >/dev/null 2>&1; then + ss -ltnp 2>/dev/null || ss -ltn + ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' +fi +EOF +)" + encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" + + deadline=$((SECONDS + timeout_seconds)) + while ((SECONDS < deadline)); do + if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then + printf '%s\n' "${network_output}" | sed 's/^/ guest-network: /' + if network_exitcode="$(pimox_guest_exec_exitcode "${network_output}")" && [[ "${network_exitcode}" == "0" ]]; then + return 0 + fi + echo "Static guest network validation failed inside Pimox VM ${vmid}." >&2 + if [[ -n "${network_exitcode:-}" ]]; then + echo "Guest exit code: ${network_exitcode}" >&2 + fi + return 1 + fi + sleep 5 + done + + echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2 + if [[ -n "${network_output:-}" ]]; then + printf '%s\n' "${network_output}" | sed 's/^/ /' >&2 + fi + pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}" + return 1 +} + +configure_pimox_worker_forwarding() { + local pimox_host="$1" + local pimox_user="$2" + local pimox_key="$3" + local worker_ip="$4" + local enabled="${LAB_PIMOX_WORKER_CONFIGURE_FORWARDING:-true}" + local backup_dir="${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}" + local worker_address + + if disabled_value "${enabled}"; then + return 0 + fi + if ! truthy "${enabled}"; then + echo "LAB_PIMOX_WORKER_CONFIGURE_FORWARDING must be true or false." >&2 + return 1 + fi + if ! validate_ipv4_cidr_or_host "${worker_ip}"; then + echo "Invalid Pimox worker forwarding IP '${worker_ip}'." >&2 + return 1 + fi + worker_address="${worker_ip%%/*}" + + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\" +if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then + iptables_bin=/usr/sbin/iptables +fi +if [ -z \"\$iptables_bin\" ]; then + if command -v apt-get >/dev/null 2>&1; then + echo 'iptables not found on Pimox host; installing iptables for worker forwarding rules.' + sudo apt-get update + sudo apt-get install -y --no-install-recommends iptables + fi +fi +iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\" +if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then + iptables_bin=/usr/sbin/iptables +fi +if [ -z \"\$iptables_bin\" ]; then + echo 'iptables not found on Pimox host and could not be installed; cannot configure worker forwarding rules.' >&2 + exit 1 +fi +iptables_wait_seconds='${LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS:-60}' +case \"\$iptables_wait_seconds\" in + ''|*[!0-9]*) + echo \"LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS must be a non-negative integer, got '\$iptables_wait_seconds'.\" >&2 + exit 1 + ;; +esac +iptables_wait_args= +if \"\$iptables_bin\" -h 2>&1 | grep -q -- ' -w'; then + iptables_wait_args=\"-w \$iptables_wait_seconds\" +else + echo 'Pimox host iptables does not support -w; proceeding without xtables lock wait.' +fi +iptables_cmd() { + sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\" +} +iptables_save_bin=\"\$(command -v iptables-save 2>/dev/null || true)\" +if [ -z \"\$iptables_save_bin\" ] && [ -x /usr/sbin/iptables-save ]; then + iptables_save_bin=/usr/sbin/iptables-save +fi +backup_dir='${backup_dir}' +case \"\$backup_dir\" in + '~'|'~/') backup_dir=\"\$HOME\" ;; + '~/'*) backup_dir=\"\$HOME/\${backup_dir#~/}\" ;; +esac +mkdir -p \"\$backup_dir\" +backup_file=\"\$backup_dir/iptables-before-jeannie-workers-\$(date +%Y%m%d-%H%M%S).rules\" +if [ -n \"\$iptables_save_bin\" ]; then + sudo \"\$iptables_save_bin\" >\"\$backup_file\" + echo \"Saved Pimox iptables backup: \$backup_file\" +fi +sudo mkdir -p /etc/sysctl.d +printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-pimox-worker-forwarding.conf >/dev/null +sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null +chain=FORWARD +if iptables_cmd -S DOCKER-USER >/dev/null 2>&1; then + chain=DOCKER-USER +fi +if ! iptables_cmd -C \"\$chain\" -s '${worker_address}/32' -j ACCEPT 2>/dev/null; then + iptables_cmd -I \"\$chain\" 1 -s '${worker_address}/32' -j ACCEPT +fi +if ! iptables_cmd -C \"\$chain\" -d '${worker_address}/32' -j ACCEPT 2>/dev/null; then + iptables_cmd -I \"\$chain\" 1 -d '${worker_address}/32' -j ACCEPT +fi +echo \"Pimox worker forwarding rules active in \$chain for ${worker_address}\"" +} + +cpuset_cpu_count() { + local cpuset="$1" + local count=0 + local part + local start + local end + local -a parts + + IFS=',' read -r -a parts <<<"${cpuset}" + for part in "${parts[@]}"; do + if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then + start="${BASH_REMATCH[1]}" + end="${BASH_REMATCH[2]}" + if ((end < start)); then + return 1 + fi + count=$((count + end - start + 1)) + elif [[ "${part}" =~ ^[0-9]+$ ]]; then + count=$((count + 1)) + else + return 1 + fi + done + + printf '%s\n' "${count}" +} + +pimox_worker_cpu_affinity() { + local index="$1" + local affinities="$2" + local worker_cores="$3" + local affinity + local affinity_index=1 + local cpu_count + + for affinity in ${affinities}; do + if ((affinity_index == index)); then + if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then + echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 + exit 1 + fi + if ((cpu_count != worker_cores)); then + echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2 + exit 1 + fi + printf '%s\n' "${affinity}" + return 0 + fi + affinity_index=$((affinity_index + 1)) + done + + echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2 + exit 1 +} + +pimox_start_vm_with_cpuset() { + local host="$1" + local user="$2" + local key_path="$3" + local qm_bin="$4" + local vmid="$5" + local cpuset="$6" + + if [[ -z "${cpuset}" ]]; then + pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'" + return + fi + if ! cpuset_cpu_count "${cpuset}" >/dev/null; then + echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2 + exit 1 + fi + + # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated + # command under taskset so PSCI brings secondary guest CPUs online reliably. + pimox_ssh "${host}" "${user}" "${key_path}" "set -eu +vmid='${vmid}' +qm_bin='${qm_bin}' +cpuset='${cpuset}' + +if ! command -v taskset >/dev/null 2>&1; then + echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2 + exit 1 +fi +if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then + echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2 + exit 1 +fi + +start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\") +cleanup() { + rm -f \"\$start_script\" +} +trap cleanup 0 + +sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\" +if ! awk -v vmid=\"\$vmid\" ' + NR == 1 && (\$1 == "/usr/bin/kvm" || \$1 ~ /qemu-system-aarch64$/) { kvm = 1 } + \$1 == "-id" && \$2 == vmid { id = 1 } + END { exit (kvm && id) ? 0 : 1 } +' \"\$start_script\"; then + echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2 + exit 1 +fi +smp_count=\$(awk '\$1 == "-smp" { count++ } END { print count + 0 }' \"\$start_script\") +if [ \"\$smp_count\" -ne 1 ]; then + echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2 + exit 1 +fi + +sudo taskset -c \"\$cpuset\" bash \"\$start_script\"" +} + +pimox_shutdown_vm_gracefully() { + local host="$1" + local user="$2" + local key_path="$3" + local qm_bin="$4" + local vmid="$5" + + pimox_ssh "${host}" "${user}" "${key_path}" "set -eu +sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true +elapsed=0 +while [ \"\$elapsed\" -lt 300 ]; do + if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + exit 0 + fi + sleep 5 + elapsed=\$((elapsed + 5)) +done +echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2 +exit 1" +} + +pimox_guest_cpu_count() { + local guest_ip="$1" + local guest_user="$2" + local guest_key_path="$3" + local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" + + ssh -i "${guest_key_path}" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${known_hosts_file}" \ + "${guest_user}@${guest_ip}" \ + 'nproc --all' +} + +ensure_pimox_worker_node() { + local index="$1" + local spec_file="$2" + local pimox_host="$3" + local pimox_user="$4" + local pimox_key="$5" + local template_vmid="$6" + local bridge="$7" + local worker_base_vmid="$8" + local worker_name_prefix="$9" + local worker_node_prefix="${10}" + local worker_key_prefix="${11}" + local worker_cores="${12}" + local worker_memory="${13}" + local worker_user="${14}" + local worker_key_path="${15}" + local ip_prefix="${16}" + local timeout_seconds="${17}" + local qm_bin="${18}" + local worker_storage="${19}" + local worker_replace_existing="${20}" + local worker_cpu_affinity="${21}" + local worker_kvm_cpuset="${22}" + local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" + local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" + local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" + local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" + local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" + local padded + local vmid + local worker_key + local worker_name + local node_name + local net0_config + local guest_ip + local guest_cpu_count + local static_ip + + printf -v padded '%02d' "${index}" + vmid=$((worker_base_vmid + index - 1)) + worker_key="${worker_key_prefix}${padded}" + worker_name="${worker_name_prefix}-${padded}" + node_name="${worker_node_prefix}-${padded}" + if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then + exit 1 + fi + if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then + if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then + echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2 + exit 1 + fi + if ! truthy "${worker_replace_existing}" && + ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then + echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}." + worker_replace_existing=true + fi + if truthy "${worker_replace_existing}"; then + echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..." + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true +elapsed=0 +while [ \"\$elapsed\" -lt 300 ]; do + if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + break + fi + sleep 5 + elapsed=\$((elapsed + 5)) +done +sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" + else + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +was_running=false +had_legacy_args=false +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then + was_running=true +fi +if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then + had_legacy_args=true +fi +sudo '${qm_bin}' set '${vmid}' \ + --agent enabled=1 \ + --bios ovmf \ + --boot 'order=scsi0;net0' \ + --cores '${worker_cores}' \ + --memory '${worker_memory}' \ + --net0 '${net0_config}' \ + --numa 0 \ + --ostype l26 \ + --scsihw virtio-scsi-pci \ + --sockets 1 \ + --vga virtio +if [ \"\$had_legacy_args\" = true ]; then + sudo '${qm_bin}' set '${vmid}' --delete args +fi +if [ -n '${worker_cpu_affinity}' ]; then + affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { + case \"\$affinity_output\" in + *'Unknown option: affinity'*) + echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' + ;; + *) + printf '%s\n' \"\$affinity_output\" >&2 + exit 1 + ;; + esac + } +fi +if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then + echo 'Restarting VM ${vmid} to apply removal of legacy qm args.' + sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true + elapsed=0 + while [ \"\$elapsed\" -lt 300 ]; do + if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + break + fi + sleep 5 + elapsed=\$((elapsed + 5)) + done + if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2 + exit 1 + fi +fi" + fi + fi + + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! ip link show '${bridge}' >/dev/null 2>&1; then + echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2 + exit 1 +fi +pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" +if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z \"\$pvesm_cmd\" ]; then + echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2 + exit 1 +fi +if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then + echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2 + exit 1 +fi +sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' +sudo '${qm_bin}' set '${vmid}' \ + --agent enabled=1 \ + --bios ovmf \ + --boot 'order=scsi0;net0' \ + --cores '${worker_cores}' \ + --memory '${worker_memory}' \ + --net0 '${net0_config}' \ + --numa 0 \ + --ostype l26 \ + --scsihw virtio-scsi-pci \ + --sockets 1 \ + --vga virtio +if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then + sudo '${qm_bin}' set '${vmid}' --delete args +fi +if [ -n '${worker_cpu_affinity}' ]; then + affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { + case \"\$affinity_output\" in + *'Unknown option: affinity'*) + echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' + ;; + *) + printf '%s\n' \"\$affinity_output\" >&2 + exit 1 + ;; + esac +} +fi +sudo '${qm_bin}' set '${vmid}' --onboot 1" + fi + + if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then + pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" + elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then + echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2 + exit 1 + fi + + case "${worker_network_mode}" in + static) + if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then + echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2 + exit 1 + fi + echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..." + configure_pimox_worker_forwarding \ + "${pimox_host}" \ + "${pimox_user}" \ + "${pimox_key}" \ + "${static_ip}" + if ! configure_pimox_worker_guest_network \ + "${pimox_host}" \ + "${pimox_user}" \ + "${pimox_key}" \ + "${vmid}" \ + "${qm_bin}" \ + "${static_ip}" \ + "${worker_gateway}" \ + "${worker_dns_servers}" \ + "${worker_interface}"; then + pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}" + exit 1 + fi + ;; + dhcp) + ;; + *) + echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2 + exit 1 + ;; + esac + + if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then + echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2 + exit 1 + fi + + guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" + if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then + echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2 + pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" + pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" + if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then + echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2 + exit 1 + fi + guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" + fi + if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then + echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2 + echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2 + exit 1 + fi + + printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" +} + +write_cluster_worker_var_file() { + local var_file="$1" + local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}' + local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}" + local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}" + shift + + LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \ + LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \ + LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \ + LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \ + LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \ + LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \ + LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \ + LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \ + python3 - "${var_file}" "$@" <<'PY' +import json +import os +from pathlib import Path +import sys + +var_file = sys.argv[1] +spec_files = sys.argv[2:] +nodes = {} +node_labels = {} + +try: + raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"]) + manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"]) + pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"]) +except json.JSONDecodeError as exc: + raise SystemExit(f"Invalid node label JSON: {exc}") from exc + +if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}: + nodes["raspberrypi"] = { + "host": os.environ["LAB_RASPBERRY_HOST"], + "user": os.environ["LAB_RASPBERRY_USER"], + "node_name": os.environ["LAB_RASPBERRY_NODE_NAME"], + "ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"], + } + node_labels["raspberrypi"] = raspberry_labels + +for spec_file in spec_files: + path = Path(spec_file) + if not path.exists() or path.stat().st_size == 0: + continue + labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels + with path.open(encoding="utf-8") as handle: + for line_number, line in enumerate(handle, start=1): + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + try: + key, host, user, node_name, ssh_key_path = line.split("\t") + except ValueError as exc: + raise SystemExit( + f"{spec_file}:{line_number}: expected tab-separated " + "key, host, user, node_name, ssh_key_path" + ) from exc + nodes[key] = { + "host": host, + "user": user, + "node_name": node_name, + "ssh_key_path": ssh_key_path, + } + node_labels[key] = labels + +with open(var_file, "w", encoding="utf-8") as handle: + json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2) + handle.write("\n") +PY +} + +prepare_cluster_worker_var_file() { + local include_raspberry_default="$1" + local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv" + local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" + local include_manual="${LAB_INCLUDE_MANUAL_WORKERS:-false}" + local -a spec_files=() + + export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}" + + mkdir -p "${REPO_ROOT}/.lab" + ensure_static_pimox_worker_spec_file "${pimox_spec_file}" + if truthy "${include_manual}"; then + spec_files+=("${manual_spec_file}") + elif ! disabled_value "${include_manual}"; then + echo "LAB_INCLUDE_MANUAL_WORKERS must be true or false." >&2 + return 1 + fi + spec_files+=("${pimox_spec_file}") + write_cluster_worker_var_file "${var_file}" "${spec_files[@]}" + export LAB_CLUSTER_VAR_FILE="${var_file}" +} + +ensure_static_pimox_worker_spec_file() { + local spec_file="$1" + local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" + local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" + local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" + local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" + local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" + local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" + local index + local padded + local worker_key + local node_name + local static_ip + + if [[ -s "${spec_file}" || "${worker_network_mode}" != "static" ]]; then + return 0 + fi + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 + return 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + mkdir -p "$(dirname "${spec_file}")" + : >"${spec_file}" + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + continue + fi + if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then + echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2 + return 1 + fi + printf -v padded '%02d' "${index}" + worker_key="${worker_key_prefix}${padded}" + node_name="${worker_node_prefix}-${padded}" + printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${static_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" + done +} + +cluster_worker_var_file_has_workers() { + local var_file="$1" + + [[ -s "${var_file}" ]] || return 1 + python3 - "${var_file}" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as handle: + document = json.load(handle) +sys.exit(0 if document.get("worker_nodes") else 1) +PY +} + +preflight_check() { + local description="$1" + local output + shift + + printf '%s %s... ' '-->' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + return 0 + fi + + printf 'failed\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed 's/^/ /' >&2 + fi + return 1 +} + +preflight_warn() { + local description="$1" + local output + shift + + printf '%s %s... ' '-->' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + return 0 + fi + + printf 'warning\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed 's/^/ /' >&2 + fi + return 0 +} + +check_gitea_reachable() { + local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" + local output + + if output="$(curl -fsS --max-time 10 "${gitea_url}" 2>&1 >/dev/null)"; then + return 0 + fi + + printf '%s\n' "${output}" >&2 + cat >&2 <<'EOF' +Gitea local HTTP failed. Run ./jeannie doctor-gitea --verbose. +If the local Gitea container is down or unhealthy, run ./jeannie deploy-gitea. +EOF + return 1 +} + +check_debian_docker_root() { + local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" + local actual_root + + if ! command -v docker >/dev/null 2>&1; then + echo "docker command was not found on Debian host" >&2 + return 1 + fi + actual_root="$(sudo docker info --format '{{.DockerRootDir}}')" + if [[ "${actual_root}" != "${expected_root}" ]]; then + echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2 + return 1 + fi +} + +fix_debian_docker_root() { + local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" + local daemon_file="/etc/docker/daemon.json" + local current_root="" + + require_debian_server "fix-debian-docker-root" + + if ! command -v docker >/dev/null 2>&1; then + echo "docker command was not found on Debian host" >&2 + exit 1 + fi + + current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" + if [[ "${current_root}" == "${expected_root}" ]]; then + echo "Debian Docker root is already ${expected_root}." + return 0 + fi + + echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..." + sudo mkdir -p "${expected_root}" /etc/docker + + echo "Stopping Docker services..." + sudo systemctl stop docker 2>/dev/null || true + sudo systemctl stop containerd 2>/dev/null || true + + if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then + echo "Copying existing Docker data to ${expected_root}..." + sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/" + fi + + sudo python3 - "${daemon_file}" "${expected_root}" <<'PY' +import json +import os +import sys + +path, expected_root = sys.argv[1:3] +document = {} +if os.path.exists(path) and os.path.getsize(path) > 0: + with open(path, encoding="utf-8") as handle: + document = json.load(handle) +document["data-root"] = expected_root +with open(path, "w", encoding="utf-8") as handle: + json.dump(document, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + + echo "Starting Docker services..." + sudo systemctl start containerd + sudo systemctl start docker + + check_debian_docker_root + echo "Debian Docker root is now ${expected_root}." +} + +check_debian_tailscale_ip() { + local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}" + + if [[ -z "${expected_ip}" ]]; then + return 0 + fi + if ! command -v tailscale >/dev/null 2>&1; then + echo "tailscale command was not found on Debian host" >&2 + return 1 + fi + if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then + echo "Debian Tailscale IP does not include ${expected_ip}" >&2 + tailscale ip -4 >&2 || true + return 1 + fi +} + +check_edge_ssh() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true" +} + +check_rpi_ssh() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true" +} + +check_rpi_docker_root_state() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" + local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu +actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" +if [ \"\$actual_root\" = '${nvme_root}' ]; then + mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; } + exit 0 +fi +if [ \"\$actual_root\" = '${fallback_root}' ]; then + echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2 + exit 0 +fi +echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2 +exit 1" +} + +check_rpi_tailscale_ip() { + local expected_ip="${LAB_RPI_TAILSCALE_IP:-}" + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + if [[ -z "${expected_ip}" ]]; then + return 0 + fi + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'" +} + +check_pimox_storage() { + local mode="${LAB_PIMOX_PIPELINE:-true}" + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" + + if disabled_value "${mode}"; then + return 0 + fi + + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" +if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z \"\$pvesm_cmd\" ]; then + echo 'pvesm was not found' >&2 + exit 1 +fi +sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'" +} + +check_local_disk_free_gib() { + local path="$1" + local min_gib="$2" + local available_kib + local min_kib + + available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')" + min_kib=$((min_gib * 1024 * 1024)) + if ((available_kib < min_kib)); then + echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2 + return 1 + fi +} + +check_systemd_active() { + local unit="$1" + + systemctl is-active --quiet "${unit}" +} + +check_rpi_docker_writable() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu +root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" +sudo mkdir -p \"\$root/.homelab-check\" +sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\" +sudo rm -f \"\$root/.homelab-check/write-test\"" +} + +check_edge_disk_free() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu +available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\" +min_kib=$((min_gib * 1024 * 1024)) +if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then + echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2 + exit 1 +fi" +} + +check_pihole_dns_query() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}" + local output + + if command -v dig >/dev/null 2>&1; then + output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || { + printf '%s\n' "${output}" >&2 + echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2 + return 1 + } + if [[ -z "${output}" ]]; then + echo "Pi-hole returned no A records for ${query_name}" >&2 + return 1 + fi + return 0 + fi + if command -v nslookup >/dev/null 2>&1; then + nslookup "${query_name}" "${rpi_host}" >/dev/null + return $? + fi + + echo "dig or nslookup is required to test Pi-hole DNS directly." >&2 + return 1 +} + +homelab_preflight() { + local phase="${1:-full}" + local failures=0 + + require_debian_server "preflight" + + if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then + echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}." + return 0 + fi + + case "${phase}" in + early | full) + ;; + *) + echo "Unsupported preflight phase '${phase}'." >&2 + exit 1 + ;; + esac + + echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..." + + preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1)) + preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1)) + preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1)) + if [[ "${phase}" == "full" ]]; then + preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1)) + preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1)) + fi + preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip + preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) + preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" check_edge_ssh || failures=$((failures + 1)) + + if ((failures > 0)); then + echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2 + exit 1 + fi + + echo "${phase^} preflight checks passed." +} + +doctor_preapply() { + local failures=0 + + require_debian_server "doctor-preapply" + + echo "Running pre-apply readiness checks..." + preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1)) + preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1)) + preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1)) + preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1)) + preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1)) + preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) + preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1)) + preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1)) + + if ((failures > 0)); then + echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2 + exit 1 + fi + + echo "Pre-apply doctor checks passed." +} + +inventory_check() { + "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" +} + +run_pimox_pipeline() { + local mode="${LAB_PIMOX_PIPELINE:-true}" + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}" + local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}" + local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" + local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" + local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}" + local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" + local provisioning_interface + local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}" + local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" + local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" + local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}" + local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" + local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}" + local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}" + local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}" + local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" + local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" + local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" + local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" + local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}" + local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}" + local worker_wait_timeout_seconds="${LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS:-600}" + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" + local index + local readiness_output + local readiness_status + local worker_cpu_affinity + local cpuset_cpu_total + local cpuset_name + local cpuset_value + local required_cores + + if disabled_value "${mode}"; then + return 0 + fi + + if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then + mode="true" + fi + + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 + exit 1 + fi + if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2 + exit 1 + fi + if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2 + exit 1 + fi + if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then + echo "Pimox template and worker CPU and memory values must be greater than zero." >&2 + exit 1 + fi + for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do + cpuset_value="${!cpuset_name}" + if [[ -z "${cpuset_value}" ]]; then + continue + fi + if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then + echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2 + exit 1 + fi + required_cores="${template_cores}" + if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then + required_cores="${worker_cores}" + fi + if ((cpuset_cpu_total < required_cores)); then + echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2 + exit 1 + fi + done + if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then + echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 + fi + if ! [[ "${worker_wait_timeout_seconds}" =~ ^[0-9]+$ && "${worker_wait_timeout_seconds}" -gt 0 ]]; then + echo "LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS must be a positive integer." >&2 + exit 1 + fi + if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then + echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2 + exit 1 + fi + if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2 + exit 1 + fi + if [[ "${worker_storage}" == "local" ]]; then + echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2 + exit 1 + fi + + set +e + readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then + echo 'qm was not found in PATH and ${qm_bin} is not executable' + exit 1 +fi +if ! ip link show '${bridge}' >/dev/null 2>&1; then + echo 'bridge ${bridge} was not found' + exit 1 +fi +if ! sudo -n true >/dev/null 2>&1; then + echo 'passwordless sudo is not available for ${pimox_user}' + exit 1 +fi +pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" +if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z \"\$pvesm_cmd\" ]; then + echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' + exit 1 +fi +if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then + echo 'Pimox worker storage ${worker_storage} was not found or is not active' + exit 1 +fi" 2>&1)" + readiness_status=$? + set -e + if ((readiness_status != 0)); then + if [[ "${mode}" == "auto" ]]; then + echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready." + return 0 + fi + echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2 + exit 1 + fi + + ensure_python3 + provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}" + if [[ -z "${provisioning_interface}" ]]; then + echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2 + exit 1 + fi + + export TF_VAR_provisioning_interface="${provisioning_interface}" + export TF_VAR_pimox_host="${pimox_host}" + export TF_VAR_pimox_user="${pimox_user}" + export TF_VAR_pimox_ssh_key_path="${pimox_key}" + export TF_VAR_pimox_qm_bin="${qm_bin}" + export TF_VAR_pimox_template_bridge="${bridge}" + export TF_VAR_pimox_template_vmid="${template_vmid}" + export TF_VAR_pimox_template_name="${template_name}" + export TF_VAR_pimox_template_cores="${template_cores}" + export TF_VAR_pimox_template_memory="${template_memory}" + export TF_VAR_pimox_template_replace_existing="${template_replace_existing}" + export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}" + export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}" + export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}" + export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}" + export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}" + export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}" + + echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..." + run_tofu_stack "bootstrap/provisioning" + + if ((worker_count == 0)); then + return 0 + fi + + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then + echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2 + exit 1 + fi + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1" + echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage." + + mkdir -p "${REPO_ROOT}/.lab" + : >"${spec_file}" + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + + worker_cpu_affinity="" + if [[ -n "${worker_cpu_affinities}" ]]; then + worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" + fi + ensure_pimox_worker_node \ + "${index}" \ + "${spec_file}" \ + "${pimox_host}" \ + "${pimox_user}" \ + "${pimox_key}" \ + "${template_vmid}" \ + "${bridge}" \ + "${worker_base_vmid}" \ + "${worker_name_prefix}" \ + "${worker_node_prefix}" \ + "${worker_key_prefix}" \ + "${worker_cores}" \ + "${worker_memory}" \ + "${worker_user}" \ + "${worker_key_path}" \ + "${ip_prefix}" \ + "${worker_wait_timeout_seconds}" \ + "${qm_bin}" \ + "${worker_storage}" \ + "${worker_replace_existing}" \ + "${worker_cpu_affinity}" \ + "${worker_kvm_cpuset}" + done + + write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}" + export LAB_CLUSTER_VAR_FILE="${var_file}" +} + +run_openwrt_pipeline() { + local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}" + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local vmid="${LAB_OPENWRT_VMID:-9100}" + local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}" + local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}" + local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}" + local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}" + local cores="${LAB_OPENWRT_CORES:-2}" + local memory="${LAB_OPENWRT_MEMORY:-512}" + local version="${LAB_OPENWRT_VERSION:-24.10.6}" + local image_url="${LAB_OPENWRT_IMAGE_URL:-}" + local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}" + local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}" + local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}" + local start_vm="${LAB_OPENWRT_START:-false}" + local onboot="${LAB_OPENWRT_ONBOOT:-false}" + local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}" + local root_key_b64="" + local lan_dhcp_ignore="1" + local start_vm_flag="false" + local onboot_flag="0" + + if disabled_value "${mode}"; then + return 0 + fi + if ! truthy "${mode}"; then + echo "LAB_OPENWRT_VM must be true or false." >&2 + exit 1 + fi + + if [[ -z "${image_url}" ]]; then + image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz" + fi + + if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then + echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2 + exit 1 + fi + for value_name in storage wan_bridge lan_bridge vm_name; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2 + exit 1 + fi + done + if [[ "${storage}" == "local" ]]; then + echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2 + exit 1 + fi + if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then + echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2 + exit 1 + fi + if truthy "${lan_dhcp_enabled}"; then + lan_dhcp_ignore="0" + fi + if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then + echo "LAB_OPENWRT_START must be true or false." >&2 + exit 1 + fi + if truthy "${start_vm}"; then + start_vm_flag="true" + fi + if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then + echo "LAB_OPENWRT_ONBOOT must be true or false." >&2 + exit 1 + fi + if truthy "${onboot}"; then + onboot_flag="1" + fi + if [[ -r "${root_key_path}" ]]; then + root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')" + fi + + echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..." + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <&2 + exit 1 +fi + +pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)" +if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z "\$pvesm_cmd" ]; then + echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2 + exit 1 +fi + +if ! sudo -n true >/dev/null 2>&1; then + echo "passwordless sudo is required for OpenWrt VM automation" >&2 + exit 1 +fi +if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then + echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2 + exit 1 +fi +if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then + echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2 + exit 1 +fi +if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then + echo "Pimox storage \$storage was not found." >&2 + exit 1 +fi + +if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then + if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then + echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2 + exit 1 + fi + sudo "\$qm_cmd" set "\$vmid" \\ + --net0 "virtio,bridge=\$wan_bridge" \\ + --net1 "virtio,bridge=\$lan_bridge" \\ + --cores "\$cores" \\ + --memory "\$memory" \\ + --onboot "\$onboot" + if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then + sudo "\$qm_cmd" start "\$vmid" + fi + exit 0 +fi + +for required_cmd in curl gzip losetup mount umount awk sed; do + if ! command -v "\$required_cmd" >/dev/null 2>&1; then + echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2 + exit 1 + fi +done + +tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)" +mnt_dir="\$tmp_dir/root" +loopdev="" +cleanup() { + if mountpoint -q "\$mnt_dir" 2>/dev/null; then + sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true + fi + if [ -n "\$loopdev" ]; then + sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true + fi + rm -rf "\$tmp_dir" +} +trap cleanup EXIT + +mkdir -p "\$mnt_dir" +curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz" +gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img" + +loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")" +root_part="\${loopdev}p2" +if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then + root_part="\${loopdev}p2" +fi +if [ ! -b "\$root_part" ]; then + echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2 + exit 1 +fi + +sudo mount "\$root_part" "\$mnt_dir" +sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh" + +cat >"\$tmp_dir/network" <"\$tmp_dir/dhcp" <"\$tmp_dir/firewall" <<'FIREWALL' +config defaults + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option synflood_protect '1' + +config zone + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + +config zone + option name 'wan' + list network 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config forwarding + option src 'lan' + option dest 'wan' + +config rule + option name 'Allow-DHCP-Renew' + option src 'wan' + option proto 'udp' + option dest_port '68' + option target 'ACCEPT' + option family 'ipv4' + +config rule + option name 'Allow-Ping' + option src 'wan' + option proto 'icmp' + option icmp_type 'echo-request' + option family 'ipv4' + option target 'ACCEPT' +FIREWALL + +cat >"\$tmp_dir/system" <"\$tmp_dir/authorized_keys" + sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys" + sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" + sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" +fi +sync +sudo umount "\$mnt_dir" +sudo losetup -d "\$loopdev" +loopdev="" + +sudo "\$qm_cmd" create "\$vmid" \\ + --name "\$vm_name" \\ + --bios ovmf \\ + --cores "\$cores" \\ + --memory "\$memory" \\ + --net0 "virtio,bridge=\$wan_bridge" \\ + --net1 "virtio,bridge=\$lan_bridge" \\ + --numa 0 \\ + --ostype l26 \\ + --scsihw virtio-scsi-pci \\ + --sockets 1 \\ + --vga virtio \\ + --onboot "\$onboot" + +sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0" +sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null +disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')" +if [ -z "\$disk_volume" ]; then + echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2 + exit 1 +fi +sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume" +sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0" + +if [ "\$start_vm" = "true" ]; then + sudo "\$qm_cmd" start "\$vmid" +fi +EOF +} + +openwrt() { + require_debian_server "openwrt" + + LAB_OPENWRT_VM=true run_openwrt_pipeline +} + +cleanup_calico_links() { + ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true + sudo ip link delete vxlan.calico 2>/dev/null || true + sudo ip link delete tunl0 2>/dev/null || true + sudo ip link delete cni0 2>/dev/null || true + sudo ip link delete kube-ipvs0 2>/dev/null || true + ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true +} + +cleanup_iptables() { + sudo iptables -F || true + sudo iptables -X || true + sudo iptables -t nat -F || true + sudo iptables -t nat -X || true + sudo iptables -t mangle -F || true + sudo iptables -t mangle -X || true + sudo iptables -t raw -F || true + sudo iptables -t raw -X || true + if command -v ipvsadm >/dev/null 2>&1; then + sudo ipvsadm --clear || true + fi +} + +cleanup_calico_runtime_files() { + local path + + for path in /run/calico /var/run/calico; do + if sudo test -e "${path}"; then + sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true + sudo rmdir "${path}" 2>/dev/null || true + fi + done +} + +restore_node_dns() { + sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf + if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then + sudo rm -f /etc/resolv.conf + sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf + fi + sudo systemctl restart systemd-resolved 2>/dev/null || true +} + +cleanup_mounts() { + if command -v findmnt >/dev/null 2>&1; then + local mount_root + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <( + for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do + findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true + done | sort -ru + ) + fi + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) + sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true +} + +cleanup_node() { + sudo kubeadm reset --force || true + sudo systemctl stop kubelet 2>/dev/null || true + sudo systemctl stop containerd 2>/dev/null || true + sudo killall containerd-shim-runc-v2 2>/dev/null || true + + cleanup_mounts + + sudo rm -rf \ + /etc/kubernetes/ \ + /var/lib/etcd/ \ + /var/lib/kubelet/ \ + /var/lib/cni/ \ + /etc/cni/net.d \ + /run/flannel \ + /var/lib/calico \ + /var/log/calico \ + /var/lib/containerd/* \ + /run/containerd/* \ + /etc/containerd/certs.d \ + /etc/containerd/config.toml + cleanup_calico_runtime_files + sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam + + cleanup_iptables + cleanup_calico_links + restore_node_dns + + sudo mkdir -p /etc/containerd/certs.d + sudo systemctl reset-failed kubelet containerd 2>/dev/null || true + sudo systemctl start containerd 2>/dev/null || true +} + +image_ref_tag() { + local image_ref="$1" + local tag + + tag="${image_ref##*:}" + if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then + echo "Image reference ${image_ref} must include an immutable tag." >&2 + exit 1 + fi + + printf '%s\n' "${tag}" +} + +website_image_tag() { + local source_hash="$1" + + printf 'src-%s\n' "${source_hash:0:12}" +} + +apps_registry_endpoint() { + if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then + printf '%s\n' "${TF_VAR_registry_endpoint}" + return 0 + fi + + demos_registry_endpoint +} + +demos_registry_endpoint() { + local image + + image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")" + if [[ -z "${image}" || "${image}" != */* ]]; then + echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2 + exit 1 + fi + + printf '%s\n' "${image%%/*}" +} + +website_source_hash() { + ( + cd "${REPO_ROOT}" + find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' + ) +} + +demos_source_hash() { + ( + cd "${REPO_ROOT}" + find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' + ) +} + +registry_image_exists() { + local registry_endpoint="$1" + local repository="$2" + local tag="$3" + local accept_header + + if ! command -v curl >/dev/null 2>&1; then + return 1 + fi + + accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" + curl -fsS \ + -H "Accept: ${accept_header}" \ + "http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null +} + +image_state_value() { + local state_file="$1" + local key="$2" + + awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true +} + +website_image_is_current() { + local state_file="$1" + local source_hash="$2" + local platforms="$3" + local image_ref="$4" + local registry_endpoint="$5" + local image_tag + local saved_hash + local saved_platforms + local saved_image + + [[ -f "${state_file}" ]] || return 1 + + saved_hash="$(image_state_value "${state_file}" source_hash)" + saved_platforms="$(image_state_value "${state_file}" platforms)" + saved_image="$(image_state_value "${state_file}" image)" + + [[ "${saved_hash}" == "${source_hash}" ]] || return 1 + [[ "${saved_platforms}" == "${platforms}" ]] || return 1 + [[ "${saved_image}" == "${image_ref}" ]] || return 1 + + image_tag="$(image_ref_tag "${image_ref}")" + registry_image_exists "${registry_endpoint}" php-website "${image_tag}" +} + +ensure_website_image_tag_not_reused() { + local state_file="$1" + local source_hash="$2" + local image_ref="$3" + local saved_hash + local saved_image + + [[ -f "${state_file}" ]] || return 0 + + saved_hash="$(image_state_value "${state_file}" source_hash)" + saved_image="$(image_state_value "${state_file}" image)" + + if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then + echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2 + echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2 + exit 1 + fi +} + +demos_image_is_current() { + local state_file="$1" + local source_hash="$2" + local platforms="$3" + local image_ref="$4" + local registry_endpoint="$5" + local saved_hash + local saved_platforms + local saved_image + + [[ -f "${state_file}" ]] || return 1 + + saved_hash="$(image_state_value "${state_file}" source_hash)" + saved_platforms="$(image_state_value "${state_file}" platforms)" + saved_image="$(image_state_value "${state_file}" image)" + + [[ "${saved_hash}" == "${source_hash}" ]] || return 1 + [[ "${saved_platforms}" == "${platforms}" ]] || return 1 + [[ "${saved_image}" == "${image_ref}" ]] || return 1 + + registry_image_exists "${registry_endpoint}" demos-static latest +} + +write_website_image_state() { + local state_file="$1" + local source_hash="$2" + local platforms="$3" + local image_ref="$4" + + mkdir -p "$(dirname "${state_file}")" + { + printf 'source_hash=%s\n' "${source_hash}" + printf 'platforms=%s\n' "${platforms}" + printf 'image=%s\n' "${image_ref}" + } > "${state_file}" +} + +write_demos_image_state() { + local state_file="$1" + local source_hash="$2" + local platforms="$3" + local image_ref="$4" + + mkdir -p "$(dirname "${state_file}")" + { + printf 'source_hash=%s\n' "${source_hash}" + printf 'platforms=%s\n' "${platforms}" + printf 'image=%s\n' "${image_ref}" + } > "${state_file}" +} + +path_available_mb() { + local path="$1" + + while [[ ! -e "${path}" && "${path}" != "/" ]]; do + path="$(dirname "${path}")" + done + + df -Pm "${path}" | awk 'NR == 2 {print $4}' +} + +docker_root_dir() { + docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n' +} + +prune_unused_docker_build_data() { + docker buildx rm lab-builder 2>/dev/null || true + docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true + docker builder prune -af 2>/dev/null || true + docker system prune -af 2>/dev/null || true +} + +ensure_docker_build_space() { + local docker_root + local free_mb + local min_free_mb + + min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}" + docker_root="$(docker_root_dir)" + free_mb="$(path_available_mb "${docker_root}")" + + if (( free_mb >= min_free_mb )); then + return 0 + fi + + echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..." + prune_unused_docker_build_data + free_mb="$(path_available_mb "${docker_root}")" + + if (( free_mb < min_free_mb )); then + echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2 + echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2 + echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2 + exit 1 + fi +} + +prepare_buildx_builder() { + local registry_endpoint="$1" + + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + cat < "${BUILDX_CONFIG}" +[registry."${registry_endpoint}"] + http = true + insecure = true +[registry."127.0.0.1:30500"] + http = true + insecure = true +[registry."localhost:30500"] + http = true + insecure = true +EOF + + docker buildx rm lab-builder 2>/dev/null || true + docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use + docker buildx inspect --bootstrap +} + +ensure_cosign_available() { + local arch + local cosign_url + local target_dir + local target_path + local tmp_path + + if [[ -n "${COSIGN_BIN}" ]]; then + if [[ -x "${COSIGN_BIN}" ]]; then + return 0 + fi + echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2 + exit 1 + fi + + if command -v cosign >/dev/null 2>&1; then + COSIGN_BIN="$(command -v cosign)" + return 0 + fi + + case "$(uname -m)" in + x86_64 | amd64) + arch="amd64" + ;; + aarch64 | arm64) + arch="arm64" + ;; + *) + echo "Unsupported Cosign install architecture: $(uname -m)" >&2 + exit 1 + ;; + esac + + target_dir="${HOMELAB_STATE_DIR}/bin" + target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}" + if [[ -x "${target_path}" ]]; then + COSIGN_BIN="${target_path}" + return 0 + fi + + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2 + exit 1 + fi + + mkdir -p "${target_dir}" + tmp_path="${target_path}.tmp" + cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" + echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..." + curl -fsSL "${cosign_url}" -o "${tmp_path}" + chmod 0755 "${tmp_path}" + mv "${tmp_path}" "${target_path}" + COSIGN_BIN="${target_path}" +} + +ensure_cosign_password() { + if [[ -n "${COSIGN_PASSWORD:-}" ]]; then + return 0 + fi + + if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then + COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" + export COSIGN_PASSWORD + return 0 + fi + + mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")" + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}" + elif command -v python3 >/dev/null 2>&1; then + python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}" +import secrets + +print(secrets.token_hex(32)) +PY + else + echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2 + exit 1 + fi + chmod 0600 "${COSIGN_PASSWORD_FILE}" + COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" + export COSIGN_PASSWORD +} + +run_cosign() { + ensure_cosign_available + ensure_cosign_password + + COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@" +} + +ensure_cosign_keypair() { + ensure_cosign_available + ensure_cosign_password + + if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then + return 0 + fi + + if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then + echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2 + exit 1 + fi + + mkdir -p "$(dirname "${COSIGN_KEY_PATH}")" + echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..." + COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}" + chmod 0600 "${COSIGN_KEY_PATH}" + chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}" +} + +publish_cosign_public_key_config_map() { + kubectl --kubeconfig "${KUBECONFIG_PATH}" create namespace kyverno --dry-run=client -o yaml | + kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f - + + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \ + --from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \ + --dry-run=client -o yaml | + kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f - +} + +cosign_registry_flags() { + local registry_endpoint="$1" + local image_ref="$2" + + if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then + printf '%s\n' "--allow-http-registry" + fi +} + +image_supply_chain_metadata_exists() { + local image_ref="$1" + local registry_endpoint="$2" + local digest_ref + local -a registry_flags=() + + if ! digest_ref="$(image_digest_ref "${image_ref}")"; then + return 1 + fi + + mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") + + run_cosign verify \ + --key "${COSIGN_PUBLIC_KEY_PATH}" \ + --insecure-ignore-tlog=true \ + "${registry_flags[@]}" \ + "${digest_ref}" >/dev/null 2>&1 && + run_cosign verify-attestation \ + --key "${COSIGN_PUBLIC_KEY_PATH}" \ + --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ + --insecure-ignore-tlog=true \ + "${registry_flags[@]}" \ + "${digest_ref}" >/dev/null 2>&1 +} + +write_image_sbom_predicate() { + local image_ref="$1" + local output_file="$2" + local source_hash="$3" + + docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}" + python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY' +from datetime import datetime, timezone +import json +import re +import sys + +path = sys.argv[1] +image_ref = sys.argv[2] +source_hash = sys.argv[3] + +try: + with open(path, encoding="utf-8") as handle: + document = json.load(handle) +except json.JSONDecodeError: + document = None + +if not isinstance(document, dict): + safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-") + document = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": f"homelab image SBOM for {image_ref}", + "documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}", + "creationInfo": { + "created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": ["Tool: jeannie"], + }, + "packages": [ + { + "name": image_ref, + "SPDXID": "SPDXRef-Image", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": False, + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "copyrightText": "NOASSERTION", + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-Image", + } + ], + } + with open(path, "w", encoding="utf-8") as handle: + json.dump(document, handle, sort_keys=True) + handle.write("\n") + +if not isinstance(document, dict): + raise SystemExit("image SBOM predicate is not a JSON object") +if document.get("SPDXID") != "SPDXRef-DOCUMENT": + raise SystemExit("image SBOM predicate is not an SPDX document") +if not document.get("packages"): + raise SystemExit("image SBOM predicate does not include packages") +PY +} + +image_digest_ref() { + local image_ref="$1" + local digest + local repository + + if [[ "${image_ref}" == *@sha256:* ]]; then + printf '%s\n' "${image_ref}" + return 0 + fi + + digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')" + if [[ -z "${digest}" || "${digest}" == "" ]]; then + echo "Unable to resolve image digest for ${image_ref}." >&2 + return 1 + fi + + if [[ "${image_ref##*/}" == *:* ]]; then + repository="${image_ref%:*}" + else + repository="${image_ref}" + fi + printf '%s@%s\n' "${repository}" "${digest}" +} + +publish_image_supply_chain_metadata() { + local image_ref="$1" + local registry_endpoint="$2" + local source_hash="$3" + local digest_ref + local sbom_file + local -a registry_flags=() + + if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then + echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM." + return 0 + fi + + digest_ref="$(image_digest_ref "${image_ref}")" + mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") + sbom_file="$(mktemp)" + write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}" + + echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..." + run_cosign sign \ + --yes \ + --key "${COSIGN_KEY_PATH}" \ + --tlog-upload=false \ + -a "homelab.dev/source-hash=${source_hash}" \ + "${registry_flags[@]}" \ + "${digest_ref}" + run_cosign attest \ + --yes \ + --key "${COSIGN_KEY_PATH}" \ + --predicate "${sbom_file}" \ + --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ + --tlog-upload=false \ + "${registry_flags[@]}" \ + "${digest_ref}" + rm -f "${sbom_file}" + + if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then + echo "Cosign verification failed after signing ${digest_ref}." >&2 + exit 1 + fi +} + +dump_argocd_debug() { + local app="$1" + + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" -o yaml || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd describe application "${app}" || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get pods -o wide || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs deployment/argocd-repo-server --tail=120 || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true +} + +dump_namespace_debug() { + local namespace="$1" + + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get all -o wide || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o wide || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe pods || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true +} + +wait_for_namespace() { + local namespace="$1" + local app="$2" + local timeout_seconds="$3" + local elapsed=0 + + until kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; do + if ((elapsed >= timeout_seconds)); then + echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2 + dump_argocd_debug "${app}" + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +wait_for_namespaced_resource() { + local namespace="$1" + local kind="$2" + local name="$3" + local app="$4" + local timeout_seconds="$5" + local elapsed=0 + + until kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do + if ((elapsed >= timeout_seconds)); then + echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2 + dump_argocd_debug "${app}" + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +wait_for_cluster_resource() { + local kind="$1" + local name="$2" + local app="$3" + local timeout_seconds="$4" + local elapsed=0 + local api_resource + + api_resource="${kind%%/*}" + if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get crd "${api_resource}" >/dev/null 2>&1; then + echo "Required CRD ${api_resource} is not installed; cannot wait for ${kind}/${name} from Argo CD app ${app}." >&2 + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" \ + -o jsonpath='sync={.status.sync.status} health={.status.health.status} message={.status.operationState.message}{"\n"}' 2>/dev/null || true + dump_argocd_debug "${app}" + exit 1 + fi + + until kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${kind}/${name}" >/dev/null 2>&1; do + if ((elapsed >= timeout_seconds)); then + echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2 + dump_argocd_debug "${app}" + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +wait_for_deployment_ready() { + local namespace="$1" + local deployment="$2" + local app="$3" + local timeout_seconds="$4" + local desired_replicas + local ready_replicas + local elapsed=0 + + desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" + desired_replicas="${desired_replicas:-1}" + + until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \ + (( ${ready_replicas:-0} >= desired_replicas )); do + if ((elapsed >= timeout_seconds)); then + echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2 + dump_argocd_debug "${app}" + dump_namespace_debug "${namespace}" + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +deploy_gitea() { + local mode="${LAB_GITEA_DEPLOY:-true}" + local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}" + local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}" + local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}" + local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}" + local http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" + local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}}" + local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" + local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml" + local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}" + + require_debian_server "deploy-gitea" + + if disabled_value "${mode}"; then + install_gitea_backup_timer + return 0 + fi + + if [[ ! -s "${compose_file}" ]]; then + echo "Missing ${compose_file}" >&2 + exit 1 + fi + + echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..." + + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea" + scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml" + + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu +install_dir='${install_dir}' +install_docker='${install_docker}' + +install_missing_packages() { + missing_packages='' + for package in \"\$@\"; do + if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then + missing_packages=\"\$missing_packages \$package\" + fi + done + if [ -n \"\$missing_packages\" ]; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends \$missing_packages + fi +} + +install_missing_packages ca-certificates curl iptables + +if ! command -v docker >/dev/null 2>&1; then + if [ \"\$install_docker\" = 'true' ]; then + curl -fsSL https://get.docker.com | sudo sh + else + echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2 + exit 1 + fi +fi + +if ! sudo docker compose version >/dev/null 2>&1; then + install_missing_packages docker-compose-plugin +fi + +repair_docker_iptables() { + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...' + sudo systemctl restart docker + sleep 3 + + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo 'Docker NAT chain is still missing after restarting Docker.' >&2 + sudo iptables -t nat -S >&2 || true + sudo systemctl status docker --no-pager -l >&2 || true + exit 1 +} + +repair_docker_iptables + +sudo mkdir -p \"\$install_dir/data\" +sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\" +sudo chown -R 1000:1000 \"\$install_dir/data\" +sudo tee \"\$install_dir/.env\" >/dev/null </dev/null | grep -q "install ok installed"; then + missing_packages+=("${package}") + fi + done + + if ((${#missing_packages[@]})); then + sudo apt-get update + sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi +} + +ensure_local_docker_compose() { + local command_name="$1" + local install_docker="$2" + + install_missing_debian_packages ca-certificates curl iptables + + if ! command -v docker >/dev/null 2>&1; then + if [[ "${install_docker}" == "true" ]]; then + curl -fsSL https://get.docker.com | sudo sh + else + echo "Docker is not installed on the Debian host. Install Docker through host bootstrap first, or rerun with LAB_${command_name}_INSTALL_DOCKER=true to allow get.docker.com installation." >&2 + exit 1 + fi + fi + + if ! sudo docker compose version >/dev/null 2>&1; then + install_missing_debian_packages docker-compose-plugin + fi +} + +repair_local_docker_iptables() { + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo "Docker NAT chain is missing on the Debian host; restarting Docker once to restore iptables state..." + sudo systemctl restart docker + sleep 3 + + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo "Docker NAT chain is still missing after restarting Docker." >&2 + sudo iptables -t nat -S >&2 || true + sudo systemctl status docker --no-pager -l >&2 || true + exit 1 +} + +deploy_heimdall() { + local mode="${LAB_HEIMDALL_DEPLOY:-true}" + local install_dir="${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" + local data_dir="${LAB_HEIMDALL_DATA_DIR:-${install_dir}/data}" + local config_dir="${LAB_HEIMDALL_CONFIG_DIR:-${data_dir}/config}" + local http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" + local bind_ip="${LAB_HEIMDALL_BIND_IP:-0.0.0.0}" + local public_url="${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:-lab2025.duckdns.org}/}" + local image="${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347}" + local container_name="${HEIMDALL_CONTAINER_NAME:-homelab-heimdall}" + local seeder_container_name="${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder}" + local network="${HEIMDALL_NETWORK:-homelab-heimdall}" + local install_docker="${LAB_HEIMDALL_INSTALL_DOCKER:-false}" + local source_dir="${REPO_ROOT}/infra/heimdall" + local elapsed=0 + local value_name + local value + + require_debian_server "deploy-heimdall" + + if disabled_value "${mode}"; then + return 0 + fi + + for value_name in install_dir data_dir config_dir http_port bind_ip public_url image container_name seeder_container_name network install_docker; do + value="${!value_name}" + if [[ "${value}" == *$'\n'* ]]; then + echo "${value_name} cannot contain a newline." >&2 + exit 1 + fi + done + + if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/links.json" || ! -s "${source_dir}/seed-heimdall.py" ]]; then + echo "Missing Heimdall source files in ${source_dir}" >&2 + exit 1 + fi + + echo "Deploying external Heimdall on the Debian host at ${install_dir}..." + + ensure_local_docker_compose "HEIMDALL" "${install_docker}" + repair_local_docker_iptables + + sudo mkdir -p "${install_dir}" "${config_dir}" + sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" + sudo cp "${source_dir}/links.json" "${install_dir}/links.json" + sudo cp "${source_dir}/seed-heimdall.py" "${install_dir}/seed-heimdall.py" + sudo chown -R 1000:1000 "${data_dir}" + sudo tee "${install_dir}/.env" >/dev/null </dev/null; do + if ((elapsed >= 180)); then + echo "Timed out waiting for Heimdall on http://127.0.0.1:${http_port}/" >&2 + (cd "${install_dir}" && sudo docker compose ps && sudo docker compose logs --tail=120) >&2 || true + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + echo "Heimdall is reachable at http://127.0.0.1:${http_port}/ and ${public_url%/}/." +} + +deploy_matrix() { + local mode="${LAB_MATRIX_DEPLOY:-true}" + local install_dir="${LAB_MATRIX_INSTALL_DIR:-/home/jv/matrix-server}" + local image="" + local container_name="" + local http_port="" + local ssh_port="" + local domain="" + local install_docker="false" + local source_dir="${REPO_ROOT}/infra/matrix-server" + local value_name + local value + + require_debian_server "deploy-matrix" + + if disabled_value "${mode}"; then + return 0 + fi + + for value_name in install_dir; do + value="${!value_name}" + if [[ "${value}" == *$'\n'* ]]; then + echo "${value_name} cannot contain a newline." >&2 + exit 1 + fi + done + + if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/homeserver.yaml.tpl" || ! -s "${source_dir}/bootstrap.sh" ]]; then + echo "Missing Matrix source files in ${source_dir}" >&2 + exit 1 + fi + + echo "Deploying Matrix stack on the Debian host at ${install_dir}..." + + ensure_local_docker_compose "MATRIX" "${install_docker}" + repair_local_docker_iptables + + sudo mkdir -p "${install_dir}" + sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" + sudo cp "${source_dir}/homeserver.yaml.tpl" "${install_dir}/homeserver.yaml.tpl" + sudo cp "${source_dir}/log_config.yaml" "${install_dir}/log_config.yaml" + sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" 2>/dev/null || { + sudo mkdir -p "${install_dir}/element-config" + sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" + } + sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh" + sudo chmod +x "${install_dir}/bootstrap.sh" + + # Secrets come from the SOPS-encrypted repo file, decrypted at deploy time. + # (Env vars override the file, for manual/ops overrides.) + if ! sops_load_secrets "${REPO_ROOT}/infra/matrix-server/matrix.secret.yaml"; then + exit 1 + fi + + if [[ -z "${MATRIX_POSTGRES_PASSWORD:-}" || -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]]; then + echo "MATRIX_POSTGRES_PASSWORD and MATRIX_REGISTRATION_SHARED_SECRET missing after decrypt." >&2 + exit 1 + fi + + local matrix_server_name="${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}" + local matrix_synapse_port="${LAB_MATRIX_SYNAPSE_PORT:-8008}" + local matrix_element_port="${LAB_MATRIX_ELEMENT_PORT:-8081}" + local postgres_db="${LAB_MATRIX_POSTGRES_DB:-synapse}" + local postgres_user="${LAB_MATRIX_POSTGRES_USER:-synapse}" + + sudo tee "${install_dir}/.env" >/dev/null <&2 + exit 1 + fi + + echo "Deploying OmniRoute LLM gateway on the Debian host at ${install_dir}..." + + ensure_local_docker_compose "OMNIROUTE" "${install_docker}" + repair_local_docker_iptables + + sudo mkdir -p "${install_dir}" + sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" + sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh" + sudo chmod +x "${install_dir}/bootstrap.sh" + + sudo tee "${install_dir}/.env" >/dev/null <&2 + exit 1 + fi + done + + echo "Deploying Hermes agent on the Debian host at ${install_dir}..." + + ensure_local_docker_compose "HERMES" "${install_docker}" + + # Install uv if missing. + if [[ ! -x "${uv_bin}" ]]; then + mkdir -p "$(dirname "${uv_bin}")" + echo "Installing uv..." + curl -LsSf https://astral.sh/uv/install.sh | bash + # uv installs to ~/.local/bin; copy/alias into HERMES bin if that path differs. + if command -v uv >/dev/null 2>&1 && [[ ! -x "${uv_bin}" ]]; then + cp "$(command -v uv)" "${uv_bin}" + fi + fi + + # Install Hermes via its official installer (git method, managed venv layout). + if [[ ! -x "${hermes_home}/hermes-agent/venv/bin/python" ]]; then + echo "Installing Hermes agent..." + curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash + fi + + # Ensure the Matrix platform dependencies (E2EE) are present in the venv. + if [[ -x "${uv_bin}" && -f "${install_dir}/pyproject.toml" ]]; then + cd "${install_dir}" + "${uv_bin}" pip install 'mautrix[encryption]' -p venv/bin/python + fi + + # Secret env for the gateway (.env under HERMES_HOME, mode 600, never committed). + # Pull the hermes secrets from the SOPS-encrypted repo file; env vars override. + if ! sops_load_secrets "${REPO_ROOT}/infra/hermes.secret.yaml"; then + echo "WARNING: could not load hermes secrets; MATRIX_PASSWORD may be blank." >&2 + fi + + local secrets_file="${hermes_home}/.env" + sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "${hermes_home}" + if [[ -f "${secrets_file}" ]]; then + sudo cp "${secrets_file}" "${secrets_file}.bak-pre-jeannie" + fi + sudo tee "${secrets_file}" >/dev/null </dev/null 2>&1 && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^matrix-synapse$'; then + if [[ -n "${MATRIX_BOT_PASSWORD:-}" ]]; then + if ! docker exec matrix-synapse register_new_matrix_user -c /data/homeserver.yaml \ + -u hermes -p "${MATRIX_BOT_PASSWORD}" --no-admin >/dev/null 2>&1; then + echo "Hermes bot account already exists (expected on re-run) or auto-registration skipped." >&2 + else + echo "Hermes Matrix bot account ready." + fi + else + echo "MATRIX_BOT_PASSWORD not set; skipping Hermes bot account registration (needed only on fresh install)." >&2 + fi + fi + + # Install/start the gateway systemd user service (Hermes manages the unit). + "${hermes_home}/hermes-agent/venv/bin/python" "${hermes_home}/hermes-agent/hermes" gateway install + systemctl --user daemon-reload + systemctl --user enable --now hermes-gateway 2>/dev/null || true + + echo "Hermes agent deployed. Run 'hermes' to configure the model, or check gateway logs at ${hermes_home}/logs/gateway.log." +} + +deploy_rpi_services() { + local mode="${LAB_RPI_SERVICES_DEPLOY:-true}" + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" + local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}" + local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" + local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" + local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}" + local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}" + local source_dir="${REPO_ROOT}/infra/rpi-services" + local value_name + local value + + require_debian_server "rpi-services" + + if disabled_value "${mode}"; then + return 0 + fi + + if [[ ! -d "${source_dir}" ]]; then + echo "Missing ${source_dir}" >&2 + exit 1 + fi + + for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do + value="${!value_name}" + if [[ "${value}" == *"'"* ]]; then + echo "${value_name} cannot contain a single quote." >&2 + exit 1 + fi + done + + echo "Deploying RPi services on ${rpi_user}@${rpi_host}..." + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services" + scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "${source_dir}/docker-compose.yml" \ + "${source_dir}/adlists.txt" \ + "${source_dir}/local-dns-records.txt" \ + "${source_dir}/cname-records.txt" \ + "${source_dir}/static-dhcp-hosts.txt" \ + "${source_dir}/uptime-kuma-monitors.json" \ + "${source_dir}/seed-uptime-kuma.js" \ + "${source_dir}/bootstrap.sh" \ + "${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " +LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \ +LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \ +LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \ +LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \ +LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \ +LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \ +LAB_RPI_INSTALL_DOCKER='${install_docker}' \ +PIHOLE_WEBPASSWORD='${pihole_webpassword}' \ +bash /tmp/homelab-rpi-services/bootstrap.sh" +} + +gitea_bootstrap_password() { + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 32 + return 0 + fi + + python3 - <<'PY' +import secrets + +print(secrets.token_hex(32)) +PY +} + +gitea_api_base_url() { + local gitea_host="$1" + local http_port="$2" + local candidate + local api_base_override="${LAB_GITEA_API_BASE_URL:-}" + + if [[ -n "${api_base_override}" ]]; then + printf '%s\n' "${api_base_override%/}" + return 0 + fi + + for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do + if curl -fsS "${candidate}/version" >/dev/null 2>&1; then + printf '%s\n' "${candidate}" + return 0 + fi + done + + echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2 + exit 1 +} + +gitea_repo_exists() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local status + + status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")" + case "${status}" in + 200) + return 0 + ;; + 404) + return 1 + ;; + 401 | 403) + echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2 + exit 1 + ;; + *) + echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2 + exit 1 + ;; + esac +} + +gitea_branch_exists() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local branch="$6" + local status + + status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")" + case "${status}" in + 200) + return 0 + ;; + 404) + return 1 + ;; + 401 | 403) + echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2 + exit 1 + ;; + *) + echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2 + exit 1 + ;; + esac +} + +create_gitea_repo() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local repo_name="$4" + local default_branch="$5" + local payload + + payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY' +import json +import sys + +repo_name, default_branch = sys.argv[1:3] +print(json.dumps({ + "name": repo_name, + "private": False, + "auto_init": False, + "default_branch": default_branch, + "description": "Homelab infrastructure configuration", +})) +PY +)" + + curl -fsS \ + -u "${auth_user}:${auth_password}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "${payload}" \ + "${api_base}/user/repos" >/dev/null +} + +gitea_public_key_registered() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local public_key_path="$6" + local repo_keys + local user_keys + + user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")" + repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")" + + GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ + GITEA_USER_KEYS="${user_keys}" \ + GITEA_REPO_KEYS="${repo_keys}" \ + python3 - <<'PY' +import json +import os +import sys + +public_key = os.environ["GITEA_PUBLIC_KEY"].strip() +for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"): + for key in json.loads(os.environ[env_name]) or []: + if key.get("key", "").strip() == public_key: + sys.exit(0) +sys.exit(1) +PY +} + +create_gitea_repo_deploy_key() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local title="$6" + local public_key_path="$7" + local read_only="$8" + local payload + + payload="$( + GITEA_DEPLOY_KEY_TITLE="${title}" \ + GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ + GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \ + python3 - <<'PY' +import json +import os + +print(json.dumps({ + "title": os.environ["GITEA_DEPLOY_KEY_TITLE"], + "key": os.environ["GITEA_PUBLIC_KEY"].strip(), + "read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true", +})) +PY + )" + + curl -fsS \ + -u "${auth_user}:${auth_password}" \ + -H "Content-Type: application/json" \ + -X POST \ + -d "${payload}" \ + "${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null +} + +ensure_gitea_repo_ssh_access() { + local api_base="$1" + local auth_user="$2" + local auth_password="$3" + local owner="$4" + local repo_name="$5" + local ssh_host="$6" + local ssh_port="$7" + local key_path="$8" + local key_title="$9" + local key_read_only="${10}" + local key_dir + local known_hosts + local public_key_path + local read_only_json="false" + local ssh_repo_url + + if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then + echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2 + exit 1 + fi + + key_dir="$(dirname "${key_path}")" + public_key_path="${key_path}.pub" + mkdir -p "${key_dir}" + chmod 0700 "${key_dir}" + + if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then + ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null + elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then + ssh-keygen -y -f "${key_path}" >"${public_key_path}" + elif [[ ! -s "${key_path}" ]]; then + echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2 + exit 1 + fi + + chmod 0600 "${key_path}" + chmod 0644 "${public_key_path}" + + if truthy "${key_read_only}"; then + read_only_json="true" + fi + + if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then + echo "Gitea already has Debian host SSH key ${public_key_path}." + else + create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}" + echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}." + fi + + known_hosts="${HOME}/.ssh/known_hosts" + touch "${known_hosts}" + chmod 0644 "${known_hosts}" + if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then + ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null + fi + + ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git" + git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null || + git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}" + git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" + git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null + echo "Gitea SSH remote: ${ssh_repo_url}" +} + +bootstrap_gitea_repo() { + local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}" + local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" + local gitea_user="${LAB_GITEA_USER:-jv}" + local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" + local http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" + local root_url="${LAB_GITEA_ROOT_URL:?LAB_GITEA_ROOT_URL is required from homelab.yml}" + local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}" + local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}" + local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}" + local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}" + local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}" + local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}" + local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}" + local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}" + local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}" + local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" + local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}" + local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}" + local api_base + local public_repo_url + local direct_repo_url + local push_url + local askpass + local credentials_dir + local remote_status + local worktree_status + + require_debian_server "bootstrap-gitea-repo" + + if disabled_value "${mode}"; then + return 0 + fi + + ensure_python3 + for value_name in repo_owner repo_name default_branch bootstrap_user; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "${value_name} contains unsupported characters." >&2 + exit 1 + fi + done + if [[ "${bootstrap_email}" == *"'"* ]]; then + echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2 + exit 1 + fi + if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then + echo "LAB_GITEA_SSH_PORT must be numeric." >&2 + exit 1 + fi + + if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then + # shellcheck disable=SC1090 + source "${credentials_file}" + bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}" + bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}" + bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}" + fi + + if [[ -z "${bootstrap_password}" ]]; then + bootstrap_password="$(gitea_bootstrap_password)" + credentials_dir="$(dirname "${credentials_file}")" + mkdir -p "${credentials_dir}" + chmod 0700 "${credentials_dir}" + { + printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}" + printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}" + printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}" + } > "${credentials_file}" + chmod 0600 "${credentials_file}" + echo "Generated Gitea bootstrap credentials at ${credentials_file}." + fi + + for value_name in repo_owner repo_name default_branch bootstrap_user; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + echo "${value_name} contains unsupported characters." >&2 + exit 1 + fi + done + for value_name in bootstrap_email bootstrap_password; do + local value="${!value_name}" + if [[ "${value}" == *"'"* ]]; then + echo "${value_name} cannot contain a single quote." >&2 + exit 1 + fi + done + + echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..." + + # shellcheck disable=SC2087 + ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" </dev/null 2>&1; then + echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2 + exit 1 +fi + +for attempt in \$(seq 1 60); do + if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 || + curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then + break + fi + if [ "\${attempt}" = "60" ]; then + echo "Timed out waiting for Gitea API inside \${container_name}." >&2 + exit 1 + fi + sleep 2 +done + +if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \ + --username "\${bootstrap_user}" \ + --password "\${bootstrap_password}" \ + --email "\${bootstrap_email}" \ + --admin \ + --must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then + if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then + sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \ + --username "\${bootstrap_user}" \ + --password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || { + cat /tmp/homelab-gitea-user-password.log >&2 + exit 1 + } + else + cat /tmp/homelab-gitea-user-create.log >&2 + exit 1 + fi +fi +EOF + + api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")" + + if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then + echo "Gitea repository ${repo_owner}/${repo_name} already exists." + else + if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then + echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2 + exit 1 + fi + create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}" + echo "Created Gitea repository ${repo_owner}/${repo_name}." + fi + + public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git" + if [[ "${api_base}" == */git/api/v1 ]]; then + direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git" + else + direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git" + fi + push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}" + + git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null + git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null || + git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}" + + if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then + echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged." + else + worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)" + if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then + echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2 + echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2 + exit 1 + fi + + askpass="$(mktemp)" + trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT + cat > "${askpass}" </dev/null </dev/null 2>&1 || true" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +ssh_gitea "set -eu +sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true +sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories' +sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}' +sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}' +sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}' +sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" + +scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ + "\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}" + +sudo mkdir -p "\${GITEA_BACKUP_DIR}" +sudo chown jv:jv "\${GITEA_BACKUP_DIR}" +sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}" +sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete + +echo "Created \${backup_archive}" +BACKUP_SCRIPT_EOT + sudo chmod 0755 "${backup_script}" + + sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT' +[Unit] +Description=Back up external Homelab Gitea to Debian host storage +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-gitea-backup.sh +SERVICE_EOT + + sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT' +[Unit] +Description=Run daily Homelab Gitea backups + +[Timer] +OnCalendar=*-*-* 02:35:00 +RandomizedDelaySec=20m +Persistent=true + +[Install] +WantedBy=timers.target +TIMER_EOT + + sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT' +#!/usr/bin/env bash +set -euo pipefail + +GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" +GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}" +GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}" + +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required for Gitea restore drills." >&2 + exit 1 +fi + +latest_archive="$( + { find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } | + sort -nr | + awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }' +)" + +if [[ -z "${latest_archive}" ]]; then + echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}." + exit 0 +fi + +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")" +tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")" +report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt" + +cleanup() { + rm -rf "${tmp_dir}" + rm -f "${tmp_report}" +} +trap cleanup EXIT + +python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY' +import os +import sys +import zipfile + +archive_path, extract_dir, report_path = sys.argv[1:4] + +with zipfile.ZipFile(archive_path) as archive: + bad_member = archive.testzip() + if bad_member: + raise SystemExit(f"ZIP integrity check failed at {bad_member}") + + members = archive.infolist() + if not members: + raise SystemExit("ZIP archive is empty") + + extract_root = os.path.abspath(extract_dir) + for member in members: + target = os.path.abspath(os.path.join(extract_root, member.filename)) + if target != extract_root and not target.startswith(extract_root + os.sep): + raise SystemExit(f"Unsafe archive path: {member.filename}") + + archive.extractall(extract_root) + +file_count = 0 +total_bytes = 0 +for root, _, files in os.walk(extract_dir): + for name in files: + file_count += 1 + total_bytes += os.path.getsize(os.path.join(root, name)) + +if file_count == 0: + raise SystemExit("Archive extracted no files") + +with open(report_path, "w", encoding="utf-8") as handle: + handle.write("Gitea restore drill report\n") + handle.write(f"archive={archive_path}\n") + handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n") + handle.write(f"extracted_files={file_count}\n") + handle.write(f"extracted_bytes={total_bytes}\n") + handle.write("result=ok\n") +PY + +sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}" +sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}" +sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete + +echo "Created ${report_path}" +RESTORE_DRILL_SCRIPT_EOT + sudo chmod 0755 "${restore_drill_script}" + + sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT' +[Unit] +Description=Run a non-destructive Gitea backup restore drill +After=network-online.target homelab-gitea-backup.service +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh +RESTORE_DRILL_SERVICE_EOT + + sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT' +[Unit] +Description=Run monthly Homelab Gitea restore drills + +[Timer] +OnCalendar=monthly +RandomizedDelaySec=2h +Persistent=true + +[Install] +WantedBy=timers.target +RESTORE_DRILL_TIMER_EOT + + sudo systemctl daemon-reload + sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null + sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null +} + +backup_gitea() { + require_debian_server "backup-gitea" + + install_gitea_backup_timer + sudo /usr/local/sbin/homelab-gitea-backup.sh +} + +drill_gitea_restore() { + require_debian_server "drill-gitea-restore" + + install_gitea_backup_timer + sudo /usr/local/sbin/homelab-gitea-restore-drill.sh +} + +install_gitea_runner() { + local runner_arch + local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}" + local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}" + local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}" + local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}" + local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}" + local runner_user="${GITEA_RUNNER_USER:-jv}" + local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}" + local missing_packages=() + + require_debian_server "install-gitea-runner" + + case "$(dpkg --print-architecture)" in + amd64) + runner_arch="linux-amd64" + ;; + arm64) + runner_arch="linux-arm64" + ;; + *) + echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2 + exit 1 + ;; + esac + + for package in ca-certificates curl git nodejs python3; do + if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then + missing_packages+=("$package") + fi + done + if [[ ${#missing_packages[@]} -gt 0 ]]; then + sudo apt-get update + sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi + + sudo curl -fsSL \ + -o /usr/local/bin/act_runner \ + "https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}" + sudo chmod 0755 /usr/local/bin/act_runner + sudo chown root:root /usr/local/bin/act_runner + + sudo -u "${runner_user}" mkdir -p "${runner_home}" + + if [[ ! -f "${runner_home}/.runner" ]]; then + if [[ -z "${runner_token}" ]]; then + echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2 + exit 1 + fi + + sudo -u "${runner_user}" env \ + HOME="/home/${runner_user}" \ + GITEA_RUNNER_HOME="${runner_home}" \ + GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \ + GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \ + GITEA_RUNNER_NAME="${runner_name}" \ + GITEA_RUNNER_LABELS="${runner_labels}" \ + bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"' + else + echo "Existing runner registration found at ${runner_home}/.runner; keeping it." + fi + + sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null </dev/null + sudo systemctl status homelab-gitea-runner.service --no-pager -l +} + +recreate_pods_for_selector() { + local namespace="$1" + local selector="$2" + local app="$3" + + if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then + echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2 + dump_argocd_debug "${app}" + dump_namespace_debug "${namespace}" + exit 1 + fi +} + +refresh_argocd_application() { + local app="$1" + + kubectl --kubeconfig "${KUBECONFIG_PATH}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null +} + +website_translation_model() { + local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}" + local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}" + + require_debian_server "website-translation-model" + + if ! command -v ollama >/dev/null 2>&1; then + echo "ollama is not installed or not in PATH on this Debian server." >&2 + exit 1 + fi + + if [[ ! -r "${modelfile}" ]]; then + echo "Ollama Modelfile is not readable: ${modelfile}" >&2 + exit 1 + fi + + echo "Creating or updating Ollama model ${model_name} from ${modelfile}..." + ollama create "${model_name}" -f "${modelfile}" + + if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then + echo "Ollama model ${model_name} was not found after creation." >&2 + exit 1 + fi + + echo "Ollama model ${model_name} is ready." +} + +website_ollama_listen() { + local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}" + local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" + local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}" + local dropin_dir="/etc/systemd/system/ollama.service.d" + local dropin_file="${dropin_dir}/homelab.conf" + local waited=0 + + require_debian_server "website-ollama-listen" + + bind_address="${bind_address#http://}" + bind_address="${bind_address#https://}" + + if ! systemctl cat ollama.service >/dev/null 2>&1; then + echo "ollama.service was not found on this Debian server." >&2 + exit 1 + fi + + echo "Configuring ollama.service to listen on ${bind_address}..." + sudo mkdir -p "${dropin_dir}" + sudo mkdir -p "${models_dir}" + if id ollama >/dev/null 2>&1; then + sudo chown ollama:ollama "${models_dir}" + fi + printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" | + sudo tee "${dropin_file}" >/dev/null + sudo systemctl daemon-reload + sudo systemctl restart ollama.service + sudo systemctl is-active --quiet ollama.service + + echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..." + until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do + waited=$((waited + 2)) + if ((waited >= 60)); then + echo "Ollama service did not expose the local API after ${waited}s." >&2 + echo "Recent service logs:" >&2 + sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true + exit 1 + fi + sleep 2 + done + echo "Ollama API is reachable on the Debian host." +} + +ollama_linux_arch() { + case "$(uname -m)" in + x86_64 | amd64) + printf 'amd64\n' + ;; + aarch64 | arm64) + printf 'arm64\n' + ;; + *) + echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2 + return 1 + ;; + esac +} + +ollama_write_systemd_service() { + sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF' +[Unit] +Description=Ollama Service +After=network-online.target + +[Service] +ExecStart=/usr/bin/ollama serve +User=ollama +Group=ollama +Restart=always +RestartSec=3 +Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +[Install] +WantedBy=default.target +EOF +} + +ollama_install_from_tgz() { + local tarball="$1" + + if [[ ! -r "${tarball}" ]]; then + echo "Ollama tarball is not readable: ${tarball}" >&2 + exit 1 + fi + + echo "Installing Ollama from local tarball ${tarball}..." + sudo tar -C /usr -xzf "${tarball}" + if ! id ollama >/dev/null 2>&1; then + sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama + fi + if getent group render >/dev/null 2>&1; then + sudo usermod -aG render ollama + fi + if getent group video >/dev/null 2>&1; then + sudo usermod -aG video ollama + fi + ollama_write_systemd_service +} + +ollama_setup() { + local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}" + local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" + local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" + local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}" + local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}" + local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}" + local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}" + local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}" + local arch + local tmp_installer + local curl_ip_args=() + + require_debian_server "ollama-setup" + + if ! command -v curl >/dev/null 2>&1; then + echo "curl is required to install Ollama." >&2 + exit 1 + fi + if [[ -n "${curl_ip_version}" ]]; then + curl_ip_args+=("${curl_ip_version}") + fi + + if ! command -v ollama >/dev/null 2>&1; then + if [[ -n "${local_tarball}" ]]; then + ollama_install_from_tgz "${local_tarball}" + else + echo "Installing Ollama from ${installer_url}..." + tmp_installer="$(mktemp)" + if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then + rm -f "${tmp_installer}" + arch="$(ollama_linux_arch || true)" + cat >&2 < 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then + echo "Ollama model ${model} is already present." + else + OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}" + fi + + echo "Ollama homelab setup is ready." +} + +require_argocd_application_controller() { + local namespace="${TF_VAR_argocd_namespace:-argocd}" + local expected_resources="10m|1|2Gi" + local actual_resources + + if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then + echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2 + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide >&2 || true + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true + exit 1 + fi + + actual_resources="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get statefulset argocd-application-controller \ + -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')" + if [[ "${actual_resources}" != "${expected_resources}" ]]; then + echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2 + echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2 + exit 1 + fi +} + +require_pimox_app_worker_capacity() { + local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}" + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}" + local worker_key + local worker_host + local worker_user + local worker_node + local worker_key_path + local capacity + local ready + local failures=0 + + if disabled_value "${enabled}"; then + echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}." + return 0 + fi + if [[ ! -s "${spec_file}" ]]; then + echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate." + return 0 + fi + if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then + echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2 + exit 1 + fi + + while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do + [[ -n "${worker_node}" ]] || continue + capacity="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)" + ready="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then + echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2 + failures=$((failures + 1)) + fi + done <"${spec_file}" + + if ((failures > 0)); then + echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2 + kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true + exit 1 + fi +} + +apps() { + local buildx_builder_ready=false + local demos_image_built=false + local demos_image_ref + local demos_image_state_file + local demos_platforms + local demos_registry_endpoint + local demos_source_hash + local registry_endpoint + local website_image_built=false + local website_image_ref + local website_image_state_file + local website_platforms + local website_source_hash + + require_debian_server "apps" + + registry_endpoint="$(apps_registry_endpoint)" + demos_registry_endpoint="$(demos_registry_endpoint)" + demos_image_ref="${registry_endpoint}/demos-static:latest" + demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state" + demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" + demos_source_hash="$(demos_source_hash)" + website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state" + website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" + website_source_hash="$(website_source_hash)" + website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}" + export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}" + export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}" + export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH_PATH}}" + export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}" + + require_argocd_application_controller + require_pimox_app_worker_capacity + + ensure_cosign_keypair + publish_cosign_public_key_config_map + + if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then + echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2 + exit 1 + fi + + if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then + echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2 + exit 1 + fi + + if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then + echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2 + exit 1 + fi + + if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then + echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2 + exit 1 + fi + ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}" + + echo "Deploying homelab applications..." + + run_tofu_stack "bootstrap/apps" + + refresh_argocd_application container-registry + refresh_argocd_application supply-chain-policy + + wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300 + wait_for_namespace container-registry container-registry 300 + wait_for_namespaced_resource container-registry deployment local-registry container-registry 300 + wait_for_deployment_ready container-registry local-registry container-registry 300 + + if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" && + image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then + echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build." + else + echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..." + ensure_docker_build_space + if [[ "${buildx_builder_ready}" != "true" ]]; then + prepare_buildx_builder "${registry_endpoint}" + buildx_builder_ready=true + fi + + docker buildx build \ + --network host \ + --platform "${website_platforms}" \ + --provenance=false \ + --sbom=true \ + --label "dev.homelab.website.source-hash=${website_source_hash}" \ + -t "${website_image_ref}" \ + -f "${REPO_ROOT}/apps/website/Dockerfile" \ + "${REPO_ROOT}/apps/website/" \ + --push + website_image_built=true + fi + publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}" + + if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" && + image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then + echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build." + else + echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..." + ensure_docker_build_space + if [[ "${buildx_builder_ready}" != "true" ]]; then + prepare_buildx_builder "${registry_endpoint}" + buildx_builder_ready=true + fi + + docker buildx build \ + --network host \ + --platform "${demos_platforms}" \ + --provenance=false \ + --sbom=true \ + --label "dev.homelab.demos.source-hash=${demos_source_hash}" \ + -t "${demos_image_ref}" \ + -f "${REPO_ROOT}/apps/demos-static/Dockerfile" \ + "${REPO_ROOT}/apps/demos-static/" \ + --push + demos_image_built=true + fi + publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}" + + refresh_argocd_application website-production + wait_for_namespace website-production website-production 300 + wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300 + if [[ "${website_image_built}" == "true" ]]; then + recreate_pods_for_selector website-production app=php-website website-production + else + echo "Skipping website pod restart because the image did not change." + fi + wait_for_deployment_ready website-production php-website-deployment website-production 300 + if [[ "${website_image_built}" == "true" ]]; then + write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" + fi + + refresh_argocd_application demos-static + wait_for_namespace demos-static demos-static 300 + wait_for_namespaced_resource demos-static deployment demos-static demos-static 300 + if [[ "${demos_image_built}" == "true" ]]; then + recreate_pods_for_selector demos-static app=demos-static demos-static + else + echo "Skipping demos pod restart because the image did not change." + fi + wait_for_deployment_ready demos-static demos-static demos-static 300 + if [[ "${demos_image_built}" == "true" ]]; then + write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" + fi + + refresh_argocd_application heimdall + wait_for_namespaced_resource monitoring ingress grafana heimdall 300 + wait_for_namespaced_resource monitoring ingress prometheus heimdall 300 + wait_for_namespaced_resource monitoring ingress alertmanager heimdall 300 + wait_for_namespaced_resource argocd ingress argocd-server heimdall 300 + + refresh_argocd_application n8n + wait_for_namespace n8n n8n 300 + wait_for_namespaced_resource n8n deployment n8n n8n 300 + wait_for_deployment_ready n8n n8n n8n 300 + + echo "Application deployment successfully completed." +} + +platform_apply() { + require_debian_server "platform" + + run_tofu_stack "bootstrap/platform" +} + +edge_apply() { + require_debian_server "edge" + + run_tofu_stack "bootstrap/edge" +} + +edge_haproxy_stats_password_valid() { + local password="${1:-}" + + [[ -n "${password}" ]] && ((${#password} >= 12)) && [[ "${password}" != "adminpassword" ]] +} + +ensure_edge_tailscale_routes() { + local enabled="${LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES:-true}" + local route="${LAB_EDGE_TAILSCALE_SUBNET_ROUTE:-${LAB_LAN_CIDR:-192.168.100.0/24}}" + local router="${LAB_EDGE_TAILSCALE_ROUTER:-debian}" + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}" + local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}" + local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + + if disabled_value "${enabled}"; then + return 0 + fi + if ! truthy "${enabled}"; then + echo "LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES must be true or false." >&2 + exit 1 + fi + if ! validate_ipv4_cidr_or_host "${route}"; then + echo "Invalid LAB_EDGE_TAILSCALE_SUBNET_ROUTE '${route}'." >&2 + exit 1 + fi + if [[ -z "${edge_host}" ]]; then + echo "LAB_EDGE_HOST or TF_VAR_edge_host is required to configure edge Tailscale routes." >&2 + exit 1 + fi + + case "${router}" in + debian) + echo "Ensuring Debian advertises ${route} and OCI edge accepts Tailscale routes..." + if ! command -v tailscale >/dev/null 2>&1; then + echo "tailscale is not installed on the Debian subnet router." >&2 + exit 1 + fi + sudo mkdir -p /etc/sysctl.d + printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null + sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null + sudo tailscale set --advertise-routes="${route}" + ;; + rpi | rpi4 | raspberrypi) + echo "Ensuring RPi advertises ${route} and OCI edge accepts Tailscale routes..." + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu +if ! command -v tailscale >/dev/null 2>&1; then + echo 'tailscale is not installed on the RPi subnet router.' >&2 + exit 1 +fi +sysctl_bin=\"\$(command -v sysctl 2>/dev/null || true)\" +if [ -z \"\$sysctl_bin\" ] && [ -x /usr/sbin/sysctl ]; then + sysctl_bin=/usr/sbin/sysctl +fi +if [ -z \"\$sysctl_bin\" ]; then + echo 'sysctl is not installed on the RPi subnet router.' >&2 + exit 1 +fi +sudo mkdir -p /etc/sysctl.d +printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null +sudo \"\$sysctl_bin\" -w net.ipv4.ip_forward=1 >/dev/null +sudo tailscale set --advertise-routes='${route}' +" + ;; + *) + echo "LAB_EDGE_TAILSCALE_ROUTER must be debian or rpi." >&2 + exit 1 + ;; + esac + + ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu +if ! command -v tailscale >/dev/null 2>&1; then + echo 'tailscale is not installed on the OCI edge host.' >&2 + exit 1 +fi +sudo tailscale set --accept-routes=true +" +} + +edge_haproxy_stats_credentials_file() { + printf '%s\n' "${LAB_EDGE_HAPROXY_STATS_CREDENTIALS_FILE:-${HOME}/.config/homelab/edge-haproxy.env}" +} + +generate_edge_haproxy_stats_password() { + if command -v openssl >/dev/null 2>&1; then + openssl rand -hex 24 + elif command -v python3 >/dev/null 2>&1; then + python3 - <<'PY' +import secrets + +print(secrets.token_hex(24)) +PY + else + echo "openssl or python3 is required to generate an edge HAProxy stats password." >&2 + exit 1 + fi +} + +write_edge_haproxy_stats_credentials() { + local credentials_file="$1" + local password="$2" + local credentials_dir + + credentials_dir="$(dirname "${credentials_file}")" + mkdir -p "${credentials_dir}" + chmod 0700 "${credentials_dir}" + { + printf 'TF_VAR_haproxy_stats_password=' + printf '%q' "${password}" + printf '\n' + } > "${credentials_file}" + chmod 0600 "${credentials_file}" +} + +recover_edge_haproxy_stats_password() { + local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}" + local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}" + local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local edge_install_dir="${TF_VAR_edge_install_dir:-${LAB_EDGE_INSTALL_DIR:-/opt/homelab-edge}}" + local remote_auth + local remote_password + + if [[ -z "${edge_host}" || "${edge_install_dir}" == *"'"* ]]; then + return 1 + fi + + remote_auth="$(ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "sudo sed -n 's/^[[:space:]]*stats auth //p' '${edge_install_dir}/config_files/haproxy.cfg' | head -n 1" 2>/dev/null || true)" + remote_auth="${remote_auth//$'\r'/}" + if [[ "${remote_auth}" != *:* ]]; then + return 1 + fi + + remote_password="${remote_auth#*:}" + if ! edge_haproxy_stats_password_valid "${remote_password}"; then + return 1 + fi + + printf '%s\n' "${remote_password}" +} + +ensure_edge_haproxy_stats_password() { + local credentials_file + local recovered_password + + if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then + export TF_VAR_haproxy_stats_password + return 0 + fi + + credentials_file="$(edge_haproxy_stats_credentials_file)" + if [[ -r "${credentials_file}" ]]; then + # shellcheck disable=SC1090 + source "${credentials_file}" + if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then + export TF_VAR_haproxy_stats_password + return 0 + fi + echo "Ignoring invalid edge HAProxy stats credentials in ${credentials_file}." >&2 + fi + + if recovered_password="$(recover_edge_haproxy_stats_password)"; then + TF_VAR_haproxy_stats_password="${recovered_password}" + export TF_VAR_haproxy_stats_password + write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}" + echo "Recovered edge HAProxy stats password from the live edge host and stored it at ${credentials_file}." + return 0 + fi + + TF_VAR_haproxy_stats_password="$(generate_edge_haproxy_stats_password)" + export TF_VAR_haproxy_stats_password + write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}" + echo "Generated edge HAProxy stats password at ${credentials_file}." +} + +ensure_cluster_worker_var_file() { + if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then + prepare_cluster_worker_var_file false + fi + if truthy "${TF_VAR_allow_empty_worker_nodes:-false}"; then + return 0 + fi + if ! cluster_worker_var_file_has_workers "${LAB_CLUSTER_VAR_FILE}"; then + echo "Cluster worker var file has no worker_nodes: ${LAB_CLUSTER_VAR_FILE}" >&2 + echo "Run the Pimox worker stage first or set LAB_PIMOX_WORKER_COUNT and LAB_PIMOX_WORKER_STATIC_IPS." >&2 + return 1 + fi +} + +run_up_parallel_stage() { + local spec_file + local parallelism="${JEANNIE_UP_PARALLELISM:-4}" + local status + + if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then + echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2 + return 1 + fi + + spec_file="$(mktemp)" + python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY' +import json +import sys + +spec_file, jeannie, parallelism = sys.argv[1:4] +spec = { + "parallelism": int(parallelism), + "tasks": [ + { + "id": "gitea-deploy", + "title": "Gitea deploy", + "command": [jeannie, "__up-task", "gitea-deploy"], + }, + { + "id": "gitea-bootstrap", + "title": "Gitea repo bootstrap", + "needs": ["gitea-deploy"], + "command": [jeannie, "__up-task", "gitea-bootstrap"], + }, + { + "id": "heimdall-deploy", + "title": "Heimdall deploy", + "command": [jeannie, "__up-task", "heimdall-deploy"], + }, + { + "id": "matrix-deploy", + "title": "Matrix deploy", + "command": [jeannie, "__up-task", "matrix-deploy"], + }, + { + "id": "omniroute-deploy", + "title": "OmniRoute deploy", + "command": [jeannie, "__up-task", "omniroute-deploy"], + }, + { + "id": "hermes-deploy", + "title": "Hermes deploy", + "needs": ["omniroute-deploy", "matrix-deploy"], + "command": [jeannie, "__up-task", "hermes-deploy"], + }, + { + "id": "rpi-services", + "title": "RPi services", + "command": [jeannie, "__up-task", "rpi-services"], + }, + { + "id": "pimox-workers", + "title": "Pimox provisioning and workers", + "command": [jeannie, "__up-task", "pimox-workers"], + }, + { + "id": "openwrt", + "title": "OpenWrt VM", + "command": [jeannie, "__up-task", "openwrt"], + }, + { + "id": "edge-host", + "title": "OCI edge host check", + "command": [jeannie, "__up-task", "edge-host"], + }, + ], +} +with open(spec_file, "w", encoding="utf-8") as handle: + json.dump(spec, handle, indent=2) + handle.write("\n") +PY + set +e + "${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}" + status=$? + set -e + rm -f "${spec_file}" + return "${status}" +} + +up_task() { + require_debian_server "__up-task" + + case "${1:-}" in + gitea-deploy) + deploy_gitea + ;; + gitea-bootstrap) + bootstrap_gitea_repo + ;; + heimdall-deploy) + deploy_heimdall + ;; + matrix-deploy) + deploy_matrix + ;; + omniroute-deploy) + deploy_omniroute + ;; + hermes-deploy) + deploy_hermes + ;; + rpi-services) + deploy_rpi_services + ;; + pimox-workers) + run_pimox_pipeline + ;; + openwrt) + run_openwrt_pipeline + ;; + edge-host) + check_edge_ssh + ;; + *) + echo "Unknown __up-task '${1:-}'." >&2 + return 1 + ;; + esac +} + +up() { + require_debian_server "up" + + echo "Deploying the homelab infrastructure..." + jeannie_log_start "up" + jeannie_step_plan 12 + + run_step "Early preflight" homelab_preflight early + run_step "Go toolchain" ensure_go_toolchain + run_step "Repo Map (Graphiphy)" graphiphy_map + run_step "Independent host services" run_up_parallel_stage + run_step "Full preflight" homelab_preflight full + run_step "Pre-apply doctor" doctor_preapply + run_step "Worker var file" ensure_cluster_worker_var_file + run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up + run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" + run_step "Version report" doctor_versions_report + run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" + run_step "Applications" apps + run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" + + echo "Deployment successfully completed." + echo "Log: ${JEANNIE_LOG_FILE}" +} + +tofu_stack_from_plan_target() { + case "$1" in + provisioning) + printf 'bootstrap/provisioning\n' + ;; + cluster) + printf 'bootstrap/cluster\n' + ;; + platform) + printf 'bootstrap/platform\n' + ;; + apps) + printf 'bootstrap/apps\n' + ;; + edge) + printf 'bootstrap/edge\n' + ;; + *) + echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2 + return 1 + ;; + esac +} + +plan_homelab() { + local target="${1:-all}" + local stack + local -a stacks=() + + require_debian_server "plan" + + if [[ "${target}" == "all" ]]; then + if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then + prepare_cluster_worker_var_file false + fi + stacks=( + "bootstrap/provisioning" + "bootstrap/cluster" + "bootstrap/platform" + "bootstrap/apps" + "bootstrap/edge" + ) + else + stack="$(tofu_stack_from_plan_target "${target}")" + if [[ "${stack}" == "bootstrap/cluster" && -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then + prepare_cluster_worker_var_file false + fi + stacks=("${stack}") + fi + + for stack in "${stacks[@]}"; do + echo + echo "Planning ${stack}..." + run_tofu_plan_stack "${stack}" + done +} + +rebuild_cluster() { + require_debian_server "rebuild-cluster" + + export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}" + export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" + export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}" + export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}" + export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}" + export TF_VAR_force_worker_rejoin="${TF_VAR_force_worker_rejoin:-true}" + + echo "Rebuilding the Kubernetes cluster without touching external Gitea..." + jeannie_log_start "rebuild-cluster" + jeannie_step_plan 10 + + run_step "Preflight" homelab_preflight + run_step "Nuke existing cluster state" nuke_for_rebuild + run_step "Pimox provisioning and workers" run_pimox_pipeline + run_step "OpenWrt VM" run_openwrt_pipeline + run_step "Worker var file" ensure_cluster_worker_var_file + run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" + run_step "Version report" doctor_versions_report + run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" + run_step "Applications" apps + run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" + + echo "Cluster rebuild successfully completed." + echo "Log: ${JEANNIE_LOG_FILE}" +} + +nuke_for_rebuild() { + LAB_NUKE_SKIP_CONFIRM=true nuke +} + +cluster_worker_targets() { + local worker_ssh_targets="${WORKER_SSH_TARGETS-}" + local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" + + CLUSTER_WORKER_TARGETS=() + read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}" + + if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then + while IFS= read -r target; do + [[ -n "${target}" ]] || continue + if [[ ! " ${CLUSTER_WORKER_TARGETS[*]-} " =~ [[:space:]]${target}[[:space:]] ]]; then + CLUSTER_WORKER_TARGETS+=("${target}") + fi + done < <(python3 - "${var_file}" <<'PY' +import json +import sys + +var_file = sys.argv[1] +with open(var_file, encoding="utf-8") as handle: + document = json.load(handle) + +for key, node in sorted((document.get("worker_nodes") or {}).items()): + host = node.get("host") + user = node.get("user") + if host and user: + print(f"{user}@{host}") +PY +) + fi +} + +stop_local_kubernetes_services() { + echo "--> Stopping local Kubernetes services on the Debian control plane..." + sudo systemctl stop kubelet 2>/dev/null || true + if command -v crictl >/dev/null 2>&1; then + sudo crictl stop --all 2>/dev/null || true + elif command -v ctr >/dev/null 2>&1; then + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | + while IFS= read -r task; do + [[ -n "${task}" ]] || continue + sudo ctr -n k8s.io tasks kill -s SIGTERM "${task}" 2>/dev/null || true + done + sleep 3 + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | + while IFS= read -r task; do + [[ -n "${task}" ]] || continue + sudo ctr -n k8s.io tasks kill -s SIGKILL "${task}" 2>/dev/null || true + sudo ctr -n k8s.io tasks rm -f "${task}" 2>/dev/null || true + done + fi + if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then + sudo systemctl stop containerd 2>/dev/null || true + sudo systemctl reset-failed containerd 2>/dev/null || true + fi + sudo systemctl reset-failed kubelet 2>/dev/null || true +} + +start_local_kubernetes_services() { + echo "--> Starting local Kubernetes services on the Debian control plane..." + sudo systemctl start containerd + sudo systemctl start docker 2>/dev/null || true + sudo systemctl start kubelet +} + +stop_remote_kubernetes_services() { + local target="$1" + + echo "--> Stopping Kubernetes services on remote worker ${target}..." + if ! ssh -o ConnectTimeout=5 "${target}" \ + "sudo systemctl stop kubelet 2>/dev/null || true +if command -v crictl >/dev/null 2>&1; then + sudo crictl stop --all 2>/dev/null || true +elif command -v ctr >/dev/null 2>&1; then + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do + [ -n \"\$task\" ] || continue + sudo ctr -n k8s.io tasks kill -s SIGTERM \"\$task\" 2>/dev/null || true + done + sleep 3 + sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do + [ -n \"\$task\" ] || continue + sudo ctr -n k8s.io tasks kill -s SIGKILL \"\$task\" 2>/dev/null || true + sudo ctr -n k8s.io tasks rm -f \"\$task\" 2>/dev/null || true + done +fi +if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi +sudo systemctl reset-failed kubelet 2>/dev/null || true"; then + echo "Warning: could not SSH to ${target}; continuing because Pimox VM shutdown follows." >&2 + fi +} + +start_remote_kubernetes_services() { + local target="$1" + + echo "--> Starting Kubernetes services on remote worker ${target}..." + ssh -o ConnectTimeout=5 "${target}" \ + "sudo systemctl start containerd && sudo systemctl start kubelet" +} + +pimox_worker_count_default() { + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" + local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" + local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" + local max_count=0 + local count + local state_count + + if [[ -s "${spec_file}" ]]; then + count="$(awk -F'\t' -v key_prefix="${worker_key_prefix}" -v node_prefix="${worker_node_prefix}-" ' + $1 ~ "^" key_prefix "[0-9]+$" { + worker_index = substr($1, length(key_prefix) + 1) + 0 + if (worker_index > max) max = worker_index + } + $4 ~ "^" node_prefix "[0-9]+$" { + worker_index = substr($4, length(node_prefix) + 1) + 0 + if (worker_index > max) max = worker_index + } + END { print max + 0 } + ' "${spec_file}")" + if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then + max_count="${count}" + fi + fi + + if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then + count="$(python3 - "${var_file}" "${worker_key_prefix}" "${worker_node_prefix}" <<'PY' +import json +import re +import sys + +var_file, worker_key_prefix, worker_node_prefix = sys.argv[1:4] +with open(var_file, encoding="utf-8") as handle: + document = json.load(handle) +nodes = document.get("worker_nodes") or {} +highest = 0 +key_pattern = re.compile(rf"^{re.escape(worker_key_prefix)}(\d+)$") +node_pattern = re.compile(rf"^{re.escape(worker_node_prefix)}-(\d+)$") +for key, node in nodes.items(): + for candidate in (key, str(node.get("node_name", ""))): + match = key_pattern.match(candidate) or node_pattern.match(candidate) + if match: + highest = max(highest, int(match.group(1))) +print(highest) +PY +)" + if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then + max_count="${count}" + fi + fi + + state_count="$(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" state show null_resource.worker_nodes_required 2>/dev/null | + awk -F'"' '/"worker_count"[[:space:]]*=/ { print $4; found = 1 } END { exit found ? 0 : 1 }' || true)" + if [[ "${state_count}" =~ ^[0-9]+$ && "${state_count}" -gt "${max_count}" ]]; then + max_count="${state_count}" + fi + + count="${LAB_PIMOX_DEFAULT_WORKER_COUNT:-1}" + if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then + max_count="${count}" + fi + + printf '%s\n' "${max_count}" +} + +pimox_worker_count_effective() { + local configured="${LAB_PIMOX_WORKER_COUNT:-}" + local detected + + detected="$(pimox_worker_count_default)" + if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then + printf '%s\n' "${configured}" + return 0 + fi + printf '%s\n' "${detected}" +} + +stop_pimox_worker_vms() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}" + local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}" + local index + local vmid + + worker_count="$(pimox_worker_count_effective)" + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 + exit 1 + fi + if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then + echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2 + exit 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + echo "--> Stopping Pimox worker VMs on ${pimox_host}..." + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + vmid=$((worker_base_vmid + index - 1)) + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then + echo 'Pimox worker VM ${vmid} does not exist; skipping.' + exit 0 +fi +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + echo 'Pimox worker VM ${vmid} is already stopped.' + exit 0 +fi +echo 'Gracefully shutting down Pimox worker VM ${vmid}...' +if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then + exit 0 +fi +if [ '${force_stop}' = 'true' ]; then + echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.' + sudo '${qm_bin}' stop '${vmid}' +else + echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2 + exit 1 +fi" + done +} + +verify_pimox_worker_vms_stopped() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + local failures=0 + + worker_count="$(pimox_worker_count_effective)" + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + continue + fi + vmid=$((worker_base_vmid + index - 1)) + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' 2>/dev/null | grep -q 'status: stopped'"; then + echo "Pimox worker VM ${vmid} is not stopped." >&2 + failures=$((failures + 1)) + fi + done + + ((failures == 0)) +} + +destroy_pimox_worker_vms() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + + if ! truthy "${LAB_NUKE_DESTROY_PIMOX_WORKERS:-true}"; then + echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}." + return 0 + fi + worker_count="$(pimox_worker_count_effective)" + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 + exit 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + echo "--> Destroying Pimox worker VMs on ${pimox_host}..." + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + vmid=$((worker_base_vmid + index - 1)) + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then + echo 'Pimox worker VM ${vmid} does not exist; skipping.' + exit 0 +fi +if sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'; then + echo 'Pimox VM ${vmid} is a template; refusing to destroy it as a worker.' >&2 + exit 1 +fi +sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true +sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" + done +} + +start_pimox_worker_vms() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + + worker_count="$(pimox_worker_count_effective)" + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 + exit 1 + fi + if ((worker_count == 0)); then + return 0 + fi + + echo "--> Starting Pimox worker VMs on ${pimox_host}..." + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." + continue + fi + vmid=$((worker_base_vmid + index - 1)) + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then + echo 'Pimox worker VM ${vmid} does not exist; skipping.' + exit 0 +fi +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then + echo 'Pimox worker VM ${vmid} is already running.' + exit 0 +fi +echo 'Starting Pimox worker VM ${vmid}...' +sudo '${qm_bin}' start '${vmid}'" + done +} + +stop_cluster() { + local target + local failures=0 + declare -a CLUSTER_WORKER_TARGETS=() + + require_debian_server "stop-cluster" + cluster_worker_targets + + echo "Stopping Kubernetes cluster runtime without destroying state..." + for target in "${CLUSTER_WORKER_TARGETS[@]}"; do + stop_remote_kubernetes_services "${target}" + done + stop_pimox_worker_vms + stop_local_kubernetes_services + if kubernetes_api_reachable; then + echo "Kubernetes API is still reachable after stop; control-plane containers may still be running." >&2 + failures=$((failures + 1)) + fi + if ! verify_pimox_worker_vms_stopped; then + failures=$((failures + 1)) + fi + if ((failures > 0)); then + echo "Cluster stop finished with ${failures} verification failure(s)." >&2 + return 1 + fi + echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact." +} + +start_cluster() { + local target + declare -a CLUSTER_WORKER_TARGETS=() + + require_debian_server "start-cluster" + cluster_worker_targets + + echo "Starting Kubernetes cluster runtime without rebuilding state..." + start_local_kubernetes_services + start_pimox_worker_vms + for target in "${CLUSTER_WORKER_TARGETS[@]}"; do + start_remote_kubernetes_services "${target}" + done + echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness." +} + +kubernetes_api_reachable() { + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get --raw=/readyz >/dev/null 2>&1 +} + +wait_for_kubernetes_api_stable() { + local timeout_seconds="${1:-300}" + local stable_seconds="${2:-30}" + local interval_seconds="${3:-5}" + local elapsed=0 + local stable_elapsed=0 + + echo "Waiting for Kubernetes API to stay ready for ${stable_seconds}s..." + until ((stable_elapsed >= stable_seconds)); do + if kubernetes_api_reachable; then + stable_elapsed=$((stable_elapsed + interval_seconds)) + else + stable_elapsed=0 + fi + + if ((stable_elapsed >= stable_seconds)); then + return 0 + fi + if ((elapsed >= timeout_seconds)); then + echo "Kubernetes API did not stay ready for ${stable_seconds}s within ${timeout_seconds}s." >&2 + return 1 + fi + + sleep "${interval_seconds}" + elapsed=$((elapsed + interval_seconds)) + done +} + +ensure_kubernetes_api_ready_for_tofu_stack() { + local stack="$1" + + case "${stack}" in + bootstrap/platform|bootstrap/apps) + wait_for_kubernetes_api_stable \ + "${LAB_KUBERNETES_API_WAIT_TIMEOUT_SECONDS:-300}" \ + "${LAB_KUBERNETES_API_STABLE_SECONDS:-30}" \ + "${LAB_KUBERNETES_API_STABLE_INTERVAL_SECONDS:-5}" + ;; + esac +} + +cluster_control_plane_tracked() { + tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane" +} + +cluster_admin_kubeconfig_present() { + [[ -s "${KUBECONFIG_PATH_PATH}" ]] || + sudo test -s /etc/kubernetes/admin.conf 2>/dev/null || + [[ -n "${KUBECONFIG_PATH:-}" && -s "${KUBECONFIG_PATH}" ]] +} + +wait_for_kubernetes_api() { + local timeout_seconds="${1:-180}" + local elapsed=0 + + until kubernetes_api_reachable; do + if ((elapsed >= timeout_seconds)); then + echo "Kubernetes API did not become reachable after ${timeout_seconds}s." >&2 + return 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done +} + +ensure_existing_cluster_started_for_up() { + if ! cluster_control_plane_tracked; then + echo "No tracked kubeadm control plane found; bootstrap/cluster will create one." + return 0 + fi + + if ! cluster_admin_kubeconfig_present; then + cat >&2 <&2 + fi +} + +check_pimox_worker_tailnet_egress() { + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local probe_host="${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" + local probe_port="${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}" + local failures=0 + local found=0 + local worker_key + local host + local user + local node_name + local key_path + local output + + if [[ ! -s "${spec_file}" ]]; then + echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE." + return 1 + fi + if [[ -z "${probe_host}" ]]; then + echo "No tailnet probe host configured; set LAB_PIMOX_WORKER_TAILNET_PROBE_HOST or LAB_DEBIAN_TAILSCALE_IP." + return 1 + fi + + while IFS=$'\t' read -r worker_key host user node_name key_path; do + [[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue + found=1 + printf '%-28s ' "${node_name}" + if output="$(ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" " +set -eu +command -v tailscale >/dev/null || { echo 'tailscale command missing'; exit 1; } +sudo systemctl is-active --quiet tailscaled || { echo 'tailscaled is not active'; exit 1; } +tailscale_ip=\"\$(tailscale ip -4 2>/dev/null | head -n 1 || true)\" +[ -n \"\$tailscale_ip\" ] || { echo 'worker is not joined to Tailscale'; exit 1; } +timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' || { echo 'tailnet TCP probe failed'; exit 1; } +echo \"tailscale_ip=\$tailscale_ip\" +" 2>&1)"; then + printf 'ok - %s:%s reachable over tailnet (%s)\n' "${probe_host}" "${probe_port}" "${output}" + else + printf 'fail - %s\n' "$(printf '%s' "${output}" | head -n 1)" + failures=$((failures + 1)) + fi + done <"${spec_file}" + + if ((found == 0)); then + echo "No Pimox workers were listed in ${spec_file}." + return 1 + fi + + ((failures == 0)) +} + +status_http() { + local name="$1" + local url="$2" + + if command -v curl >/dev/null 2>&1; then + printf '%-28s ' "${name}" + curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code} %{url_effective}\n' "${url}" || + printf 'unreachable %s\n' "${url}" + else + printf '%-28s curl not installed\n' "${name}" + fi +} + +status_http_ok() { + local url="$1" + local status + + if ! command -v curl >/dev/null 2>&1; then + echo "curl not installed" >&2 + return 1 + fi + + status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { + echo "unreachable ${url}" >&2 + return 1 + } + case "${status}" in + 2* | 3*) + return 0 + ;; + *) + echo "HTTP ${status} from ${url}" >&2 + return 1 + ;; + esac +} + +status_http_reachable() { + local url="$1" + local status + + if ! command -v curl >/dev/null 2>&1; then + echo "curl not installed" >&2 + return 1 + fi + + status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { + echo "unreachable ${url}" >&2 + return 1 + } + case "${status}" in + 2* | 3* | 4*) + return 0 + ;; + *) + echo "HTTP ${status} from ${url}" >&2 + return 1 + ;; + esac +} + +status_gitea_public_ok() { + local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + + status_http_reachable "${root_url}" +} + +status_cascade_check() { + local description="$1" + local output + shift + + if truthy "${JEANNIE_REPORT_MODE:-false}"; then + report_ui_check fail "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@" || return 1 + return 0 + fi + + printf '%-36s ' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 + fi + + printf 'fail\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /' + fi + return 1 +} + +status_cascade_warn() { + local description="$1" + local output + shift + + if truthy "${JEANNIE_REPORT_MODE:-false}"; then + report_ui_check warn "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@" + return 0 + fi + + printf '%-36s ' "${description}" + if output="$("$@" 2>&1)"; then + printf 'ok\n' + if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 + fi + + printf 'warn\n' + if [[ -n "${output}" ]]; then + printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' + fi + return 0 +} + +status_container_running() { + local container="$1" + + if ! command -v docker >/dev/null 2>&1; then + echo "docker not installed" >&2 + return 1 + fi + if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then + echo "container ${container} is not running" >&2 + return 1 + fi +} + +status_kubernetes_api_ok() { + if ! command -v kubectl >/dev/null 2>&1; then + echo "kubectl not installed" >&2 + return 1 + fi + if [[ ! -s "${KUBECONFIG_PATH_PATH}" ]]; then + echo "kubeconfig missing: ${KUBECONFIG_PATH_PATH}" >&2 + return 1 + fi + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get --raw=/readyz >/dev/null +} + +status_kubernetes_nodes_ready() { + local nodes + local not_ready + + nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes --no-headers 2>/dev/null)" || return 1 + if [[ -z "${nodes}" ]]; then + echo "no Kubernetes nodes found" >&2 + return 1 + fi + not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")" + if [[ -n "${not_ready}" ]]; then + echo "not ready: ${not_ready}" >&2 + return 1 + fi +} + +status_kubernetes_deployment_ready() { + local namespace="$1" + local deployment="$2" + + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null +} + +status_no_problem_pods() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get pods -A --no-headers 2>/dev/null | + awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_recent_deployments() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get deployments -A \ + -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.readyReplicas,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,AGE:.metadata.creationTimestamp' \ + --no-headers 2>/dev/null | + awk '$3 != $4 || $5 != $4 || $6 != $4 { print }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_pod_restart_pressure() { + local threshold="${LAB_STATUS_RESTART_THRESHOLD:-3}" + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get pods -A --no-headers 2>/dev/null | + awk -v threshold="${threshold}" '$5 + 0 >= threshold { print $1 "/" $2 ": restarts=" $5 " age=" $6 }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_node_pressure() { + local rows + + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}{"="}{.status}{" "}{end}{"\n"}{end}' 2>/dev/null | + awk '/MemoryPressure=True|DiskPressure=True|PIDPressure=True|NetworkUnavailable=True/ { print }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_disk_pressure() { + local threshold="${LAB_STATUS_DISK_USE_THRESHOLD:-85}" + local rows + + rows="$(df -P / /data 2>/dev/null | awk -v threshold="${threshold}" 'NR > 1 { gsub(/%/, "", $5); if ($5 + 0 >= threshold) print $6 ": " $5 "% used" }')" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" >&2 + return 1 + fi +} + +status_traefik_error_rates() { + local since="${LAB_STATUS_LOG_SINCE:-30m}" + local sample + local count_502 + local count_404 + + sample="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n traefik logs -l app.kubernetes.io/name=traefik --since="${since}" --tail=2000 2>/dev/null || true)" + if [[ -z "${sample}" ]]; then + echo "no recent Traefik logs available" >&2 + return 1 + fi + + count_502="$(grep -Eac '(^|[^0-9])50[234]([^0-9]|$)|level=error' <<<"${sample}" || true)" + count_404="$(grep -Eac '(^|[^0-9])404([^0-9]|$)' <<<"${sample}" || true)" + if ((count_502 > 0)); then + printf '5xx/error lines in last %s: %s\n' "${since}" "${count_502}" >&2 + return 1 + fi + printf '404 lines in last %s: %s\n' "${since}" "${count_404}" >&2 +} + +status_gitea_recent_errors() { + local since="${LAB_STATUS_LOG_SINCE:-30m}" + local container="${GITEA_CONTAINER_NAME:-homelab-gitea}" + local sample + local matches + + if ! command -v docker >/dev/null 2>&1; then + echo "docker not installed" >&2 + return 1 + fi + if ! sudo docker inspect "${container}" >/dev/null 2>&1; then + echo "container ${container} not found" >&2 + return 1 + fi + + sample="$(sudo docker logs --since "${since}" "${container}" 2>&1 || true)" + matches="$(grep -Eai 'error|panic|fatal|authentication failed|denied| 50[0-9] ' <<<"${sample}" | tail -10 || true)" + if [[ -n "${matches}" ]]; then + printf '%s\n' "${matches}" >&2 + return 1 + fi +} + +status_registry_ok() { + status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/" +} + +status_rpi_uptime_kuma_ok() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" + + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \ + "curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null" +} + +status_pimox_workers_running() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + local failures=0 + + worker_count="$(pimox_worker_count_effective)" + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + echo "invalid worker count ${worker_count}" >&2 + return 1 + fi + + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + continue + fi + vmid=$((worker_base_vmid + index - 1)) + if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then + echo "VM ${vmid} is not running" >&2 + failures=$((failures + 1)) + fi + done + + ((failures == 0)) +} + +status_cascade_report() { + local failures=0 + + status_section "Status Cascade" + status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1)) + status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1)) + status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1)) + status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1)) + status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1)) + + status_section "Git And Local Services" + status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1)) + status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1)) + status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/" + status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/" + status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/" + + status_section "RPi Bootstrap And DNS" + status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1)) + status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1)) + status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1)) + status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip + status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok + + status_section "Pimox And Cluster" + status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1)) + status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1)) + status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1)) + status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1)) + status_cascade_warn "No problem pods" status_no_problem_pods + + status_section "What Broke Signals" + status_cascade_warn "Backup freshness" backup_status + status_cascade_warn "Recent deployments healthy" status_recent_deployments + status_cascade_warn "Pod restart pressure" status_pod_restart_pressure + status_cascade_warn "Node pressure" status_node_pressure + status_cascade_warn "Disk usage" status_disk_pressure + status_cascade_warn "Traefik 5xx/404 signals" status_traefik_error_rates + status_cascade_warn "Gitea recent errors" status_gitea_recent_errors + + status_section "Platform And Apps" + status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1)) + status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1)) + status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1)) + status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server + + status_section "Edge And Public URLs" + status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1)) + status_cascade_check "Traefik LoadBalancer HTTP" status_http_reachable "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" || failures=$((failures + 1)) + status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1)) + status_cascade_check "Gitea public route" status_gitea_public_ok || failures=$((failures + 1)) + + if truthy "${JEANNIE_REPORT_MODE:-false}"; then + return "${failures}" + fi + + if ((failures > 0)); then + printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2 + return 1 + fi + + printf '\nStatus cascade passed.\n' +} + +backstage_brain_enabled() { + local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}" + local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}" + + [[ "${provider,,}" == "ollama" ]] && truthy "${enabled}" +} + +backstage_brain_note() { + local topic="$1" + local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" + local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" + local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}" + local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" + local context + local retrieved_context="" + + if ! backstage_brain_enabled; then + return 0 + fi + if ! command -v python3 >/dev/null 2>&1; then + return 0 + fi + + context="$(cat)" + if [[ -s "${index_dir}/index.json" ]]; then + retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)" + fi + if [[ -n "${retrieved_context}" ]]; then + context="${context} + +Retrieved homelab knowledge: +${retrieved_context}" + fi + BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true +import json +import os +import sys +import urllib.error +import urllib.request + +endpoint, model, timeout, topic = sys.argv[1:5] +context = os.environ.get("BACKSTAGE_CONTEXT", "").strip() +try: + timeout_seconds = int(timeout) +except ValueError: + timeout_seconds = 20 + +prompt = f"""You are the backstage helper for a personal homelab script named jeannie. +Be concise and operational. Do not invent facts. Use only the supplied context. +Return at most 5 bullets. Focus on likely cause, next command, and risk. + +Topic: {topic} + +Context: +{context} +""" + +payload = { + "model": model, + "prompt": prompt, + "stream": False, + "options": { + "temperature": 0.1, + "num_predict": 220, + }, +} + +try: + tags_request = urllib.request.Request(f"{endpoint.rstrip('/')}/api/tags") + with urllib.request.urlopen(tags_request, timeout=3) as response: + tags = json.loads(response.read().decode("utf-8")) + available = {item.get("name", "") for item in tags.get("models", [])} + if model not in available and f"{model}:latest" not in available: + print(f"\n== Backstage Notes ==\nOllama is reachable, but model '{model}' is not pulled. Run: ollama pull {model}") + raise SystemExit(0) + + request = urllib.request.Request( + f"{endpoint.rstrip('/')}/api/generate", + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + result = json.loads(response.read().decode("utf-8")) +except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError): + raise SystemExit(0) + +note = (result.get("response") or "").strip() +if note: + print("\n== Backstage Notes ==") + print(note) +PY +} + +status_systemd_units() { + local unit + + for unit in "$@"; do + if systemctl list-unit-files --no-legend "${unit}.service" 2>/dev/null | grep -q .; then + printf '%-28s %s\n' "${unit}" "$(systemctl is-active "${unit}" 2>/dev/null || true)" + else + printf '%-28s not-installed\n' "${unit}" + fi + done +} + +status_compose_stack() { + local name="$1" + local directory="$2" + + printf '\n-- Docker Compose: %s (%s)\n' "${name}" "${directory}" + if [[ ! -d "${directory}" ]]; then + printf 'missing directory\n' + return 0 + fi + if ! command -v docker >/dev/null 2>&1; then + printf 'docker not installed\n' + return 0 + fi + if ! sudo docker compose version >/dev/null 2>&1; then + printf 'docker compose unavailable\n' + return 0 + fi + + (cd "${directory}" && sudo docker compose ps) || true +} + +status_kubernetes() { + status_section "Kubernetes" + + if ! command -v kubectl >/dev/null 2>&1; then + printf 'kubectl not installed\n' + return 0 + fi + if [[ ! -s "${KUBECONFIG_PATH_PATH}" ]]; then + printf 'kubeconfig missing: %s\n' "${KUBECONFIG_PATH_PATH}" + return 0 + fi + if ! kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get --raw=/readyz >/dev/null 2>&1; then + printf 'API server not reachable through %s\n' "${KUBECONFIG_PATH_PATH}" + return 0 + fi + + status_run "nodes" kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -o wide + status_kubernetes_problem_pods + status_run "traefik services" kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n traefik get svc,pods -o wide + status_run "website pods" kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n website-production get pods -o wide +} + +status_kubernetes_problem_pods() { + local rows + + printf '\n-- pods not Running/Completed\n' + rows="$( + kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get pods -A --no-headers -o wide | + awk '$4 != "Running" && $4 != "Completed" { print }' + )" + if [[ -n "${rows}" ]]; then + printf '%s\n' "${rows}" + else + printf 'none\n' + fi +} + +status_pimox_workers() { + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local worker_count + local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" + local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" + local index + local vmid + + status_section "Pimox Workers" + + worker_count="$(pimox_worker_count_effective)" + if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then + printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}" + return 0 + fi + if ((worker_count == 0)); then + printf 'worker count is 0\n' + return 0 + fi + + for ((index = 1; index <= worker_count; index++)); do + if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then + printf 'worker index %s skipped\n' "${index}" + continue + fi + vmid=$((worker_base_vmid + index - 1)) + printf 'VM %-6s ' "${vmid}" + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}'" 2>/dev/null || + printf 'unreachable or missing\n' + done +} + +status_rpi_services() { + local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" + local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" + local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" + local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" + local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" + local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" + local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" + local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}" + local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" + + status_section "RPi Services" + ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " +set +e +dns_query() { + server=\"\$1\" + port=\"\$2\" + name=\"\$3\" + python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY' +import random +import socket +import struct +import sys + +server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.') +query_id = random.randrange(0, 65536) +packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0) +for label in name.split('.'): + packet += bytes([len(label)]) + label.encode('ascii') +packet += b'\x00' + struct.pack('!HH', 1, 1) +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.settimeout(5) +sock.sendto(packet, (server, port)) +data, _ = sock.recvfrom(512) +if len(data) < 12: + raise SystemExit('short DNS response') +response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12]) +rcode = flags & 0x000f +if response_id != query_id: + raise SystemExit('mismatched DNS response id') +if rcode != 0: + raise SystemExit(f'DNS rcode {rcode}') +if an < 1: + raise SystemExit('DNS response had no answers') +print(f'ok answers={an}') +PY +} +check() { + label=\"\$1\" + shift + printf '%-34s ' \"\$label\" + output=\"\$("\$@" 2>&1)\" + status=\$? + if [ \"\$status\" -eq 0 ]; then + printf 'ok' + if [ -n \"\$output\" ]; then + printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\" + fi + printf '\n' + else + printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\" + fi + return 0 +} +docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\" +printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\" +if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then + if mountpoint -q '${docker_nvme_root}'; then + printf '%-34s ok - nvme mount active\n' 'Docker root class' + else + printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class' + fi +elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then + printf '%-34s warn - fallback root active; expected ${docker_nvme_root}\n' 'Docker root class' +else + printf '%-34s warn - unexpected docker root\n' 'Docker root class' +fi +if [ -d '${install_dir}' ]; then + cd '${install_dir}' + sudo docker compose ps || true + unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\" + check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com + if [ -n \"\$unbound_ip\" ]; then + check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com + else + printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query' + fi + upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\" + if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then + printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config' + else + printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config' + fi + check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com + check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com + check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\" +else + echo 'RPi services install directory is missing: ${install_dir}' +fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" +} + +status_report() { + local cascade_status=0 + local heal_mode="" + local status_json + local -a status_args=() + + require_debian_server "status" + + while (($# > 0)); do + case "$1" in + --heal-plan) + heal_mode="plan" + shift + ;; + --heal) + heal_mode="apply" + shift + ;; + *) + status_args+=("$1") + shift + ;; + esac + done + + if ! report_ui_parse_args "${status_args[@]}"; then + return 1 + fi + + if [[ -n "${heal_mode}" ]]; then + status_json="$(mktemp "${TMPDIR:-/tmp}/jeannie-status-json.XXXXXX")" + # shellcheck disable=SC2034 + REPORT_UI_JSON=true + # shellcheck disable=SC2034 + REPORT_UI_ONLY=problems + report_ui_begin + JEANNIE_REPORT_MODE=true status_cascade_report >/dev/null || cascade_status=$? + report_ui_render "Jeannie Status" >"${status_json}" + report_ui_cleanup + if [[ "${heal_mode}" == "plan" ]]; then + "${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}" --ai + else + "${REPO_ROOT}/scripts/heal" apply --status-json "${status_json}" --ai + fi + rm -f "${status_json}" + return "${cascade_status}" + fi + + if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then + report_ui_begin + JEANNIE_REPORT_MODE=true status_cascade_report || cascade_status=$? + report_ui_render "Jeannie Status" + report_ui_cleanup + return "${cascade_status}" + fi + + status_cascade_report || cascade_status=$? + + if ! truthy "${LAB_STATUS_DETAILS:-false}" && [[ "${REPORT_UI_DETAILS}" != "true" ]]; then + return "${cascade_status}" + fi + + status_section "Host" + printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)" + printf 'date: %s\n' "$(date -Is)" + printf 'uptime: %s\n' "$(uptime -p 2>/dev/null || uptime)" + status_run "memory" free -h + status_run "disk" df -h / /data /data/openebs/local + + status_section "Systemd" + status_systemd_units ssh docker containerd kubelet tailscaled homelab-gitea-runner + + status_section "Docker" + if command -v docker >/dev/null 2>&1; then + status_run "docker containers" sudo docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' + else + printf 'docker not installed\n' + fi + status_compose_stack "gitea" "/data/homelab-gitea" + status_compose_stack "heimdall" "${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" + status_compose_stack "arr-stack" "${REPO_ROOT}/infra/arr-stack" + + status_kubernetes + status_pimox_workers + status_rpi_services + + status_section "Tailscale" + if command -v tailscale >/dev/null 2>&1; then + status_run "tailscale ip" tailscale ip -4 + status_run "tailscale peers" tailscale status + else + printf 'tailscale not installed\n' + fi + + status_section "HTTP" + status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" + status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" + status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" + status_http "heimdall local" "http://127.0.0.1:${LAB_HEIMDALL_HTTP_PORT:-8082}/" + status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" + status_http "arr radarr" "http://127.0.0.1:7878/" + status_http "arr sonarr" "http://127.0.0.1:8989/" + status_http "arr prowlarr" "http://127.0.0.1:9696/" + + return "${cascade_status}" +} + +check_edge_traefik_backend() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${traefik_ip}:80/' | grep -Eq '^[234]'" +} + +check_edge_gitea_backend() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}" + local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${gitea_ts_ip}:${gitea_http_port}/' | grep -Eq '^[234]'" +} + +check_edge_heimdall_backend() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}}" + local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${heimdall_host}:${heimdall_http_port}/' | grep -Eq '^[234]'" +} + +check_gitea_git_remote() { + git ls-remote gitea main >/dev/null +} + +check_gitea_ssh() { + local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" + local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}" + + ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" 2>&1 | + grep -Eiq 'successfully authenticated|Hi |shell access is disabled' +} + +doctor_edge() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}" + local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}" + local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${gitea_ts_ip}}" + local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" + local failures=0 + + require_debian_server "doctor-edge" + if ! report_ui_parse_args "$@"; then + return 1 + fi + + if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then + report_ui_begin + report_ui_check fail "Public URLs" "Website public" "./jeannie doctor-edge --verbose" "" "" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1)) + report_ui_check fail "Public URLs" "Gitea public" "./jeannie doctor-gitea" "" "" status_gitea_public_ok || failures=$((failures + 1)) + report_ui_check fail "Edge" "OCI edge SSH" "./jeannie preflight" "" "" check_edge_ssh || failures=$((failures + 1)) + report_ui_check fail "Backends" "Traefik from edge" "./jeannie doctor-cluster" "" "" check_edge_traefik_backend || failures=$((failures + 1)) + report_ui_check fail "Backends" "Gitea from edge" "./jeannie doctor-gitea" "" "" check_edge_gitea_backend || failures=$((failures + 1)) + report_ui_check fail "Backends" "Heimdall from edge" "./jeannie deploy-heimdall" "" "" check_edge_heimdall_backend || failures=$((failures + 1)) + report_ui_render "Doctor Edge" + report_ui_cleanup + return "${failures}" + fi + + status_section "Doctor Edge" + status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" + status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" + + printf '\n-- edge backend reachability\n' + if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " +set +e +printf '%-28s ' 'traefik backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable +printf '%-28s ' 'gitea backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable +printf '%-28s ' 'heimdall backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${heimdall_host}:${heimdall_http_port}/' || echo unreachable +printf '\n-- tailscale\n' +tailscale status || true +"; then + printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}" + fi + + cat <<'EOF' + +Next steps: +- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster. +- If only Gitea is unreachable, run ./jeannie doctor-gitea. +- If only Heimdall is unreachable, run ./jeannie deploy-heimdall. +- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge. +- Runbook: docs/runbooks/edge-failures.md +EOF + + backstage_brain_note "doctor-edge" </dev/null 2>&1; then + sudo crictl ps -a | head -50 || true + else + printf 'crictl not installed\n' + fi + + cat <<'EOF' + +Next steps: +- If API server is unreachable after a stop, run ./jeannie start-cluster. +- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node. +- If Pimox workers are stopped, start them with ./jeannie start-cluster. +- If kubelet versions drift, run ./jeannie doctor-versions. +- Runbook: docs/runbooks/cluster-stop-start-failures.md +EOF + + backstage_brain_note "doctor-cluster" <&2 <<'EOF' +nuke destroys Kubernetes state and Pimox worker VMs. +Rerun with LAB_CONFIRM_NUKE=homelab when that destructive action is intended. +EOF + exit 1 + fi + + echo "Brutally nuking the homelab infrastructure..." + cluster_worker_targets + + if ((${#CLUSTER_WORKER_TARGETS[@]} > 0)); then + mkdir -p "$(dirname "${worker_known_hosts_file}")" + touch "${worker_known_hosts_file}" + chmod 0600 "${worker_known_hosts_file}" + fi + + echo "--> Terminating local OpenTofu tasks..." + killall tofu terraform 2>/dev/null || true + + echo "--> Eviscerating local Kubernetes components..." + cleanup_node + sudo rm -f "${KUBECONFIG_PATH_PATH}" + + for target in "${CLUSTER_WORKER_TARGETS[@]}"; do + echo "--> Eviscerating remote Kubernetes components (${target})..." + target_host="${target#*@}" + target_host="${target_host%%:*}" + ssh-keygen -R "${target_host}" -f "${worker_known_hosts_file}" >/dev/null 2>&1 || true + if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${worker_known_hosts_file}" "${target}" "bash -s" <<'EOF' +set -euo pipefail + +cleanup_calico_links() { + ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true + sudo ip link delete vxlan.calico 2>/dev/null || true + sudo ip link delete tunl0 2>/dev/null || true + sudo ip link delete cni0 2>/dev/null || true + sudo ip link delete kube-ipvs0 2>/dev/null || true + ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true +} + +cleanup_iptables() { + sudo iptables -F || true + sudo iptables -X || true + sudo iptables -t nat -F || true + sudo iptables -t nat -X || true + sudo iptables -t mangle -F || true + sudo iptables -t mangle -X || true + sudo iptables -t raw -F || true + sudo iptables -t raw -X || true + if command -v ipvsadm >/dev/null 2>&1; then + sudo ipvsadm --clear || true + fi +} + +cleanup_calico_runtime_files() { + local path + + for path in /run/calico /var/run/calico; do + if sudo test -e "${path}"; then + sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true + sudo rmdir "${path}" 2>/dev/null || true + fi + done +} + +restore_node_dns() { + sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf + if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then + sudo rm -f /etc/resolv.conf + sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf + fi + sudo systemctl restart systemd-resolved 2>/dev/null || true +} + +cleanup_mounts() { + if command -v findmnt >/dev/null 2>&1; then + local mount_root + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <( + for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do + findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true + done | sort -ru + ) + fi + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) + sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true +} + +sudo kubeadm reset --force || true +sudo systemctl stop kubelet 2>/dev/null || true +sudo systemctl stop containerd 2>/dev/null || true +sudo killall containerd-shim-runc-v2 2>/dev/null || true + +cleanup_mounts + +sudo rm -rf \ + /etc/kubernetes/ \ + /var/lib/etcd/ \ + /var/lib/kubelet/ \ + /var/lib/cni/ \ + /etc/cni/net.d \ + /run/flannel \ + /var/lib/calico \ + /var/log/calico \ + /var/lib/containerd/* \ + /run/containerd/* \ + /etc/containerd/certs.d \ + /etc/containerd/config.toml +cleanup_calico_runtime_files +sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam + +cleanup_iptables +cleanup_calico_links +restore_node_dns + +sudo mkdir -p /etc/containerd/certs.d +sudo systemctl reset-failed kubelet containerd 2>/dev/null || true +sudo systemctl start containerd 2>/dev/null || true +EOF + then + echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 + exit 1 + fi + done + + destroy_pimox_worker_vms + + docker buildx rm lab-builder 2>/dev/null || true + docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true + rm -f "${BUILDX_CONFIG}" || true + + echo "--> Backing up local OpenTofu tracking state files..." + backup_tofu_state + + echo "--> Deleting OpenTofu tracking state files..." + rm -rf "${REPO_ROOT}"/bootstrap/cluster/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/cluster/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/cluster/.terraform/ + rm -rf "${REPO_ROOT}"/bootstrap/platform/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/platform/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/platform/.terraform/ + rm -rf "${REPO_ROOT}"/bootstrap/apps/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/apps/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/apps/.terraform/ + rm -rf "${REPO_ROOT}"/bootstrap/edge/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/edge/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/edge/.terraform/ + rm -f "${REPO_ROOT}/.lab/pimox-workers.tsv" "${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" + + echo "Destruction complete. Retained data under /data/openebs/local was left intact." +} + +ensure_sops_age_tools() { + local missing_packages=() + + if ! command -v age-keygen >/dev/null 2>&1; then + missing_packages+=(age) + fi + + if ((${#missing_packages[@]} > 0)); then + echo "Installing missing secret-management tools: ${missing_packages[*]}" + sudo apt-get update + sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi + + # age must be present; it's small and apt-provided on Debian. + if ! command -v age-keygen >/dev/null 2>&1; then + echo "age-keygen is still unavailable after package installation." >&2 + exit 1 + fi + + # sops: not packaged on Debian trixie — fall back to the GitHub release binary. + if ! command -v sops >/dev/null 2>&1; then + echo "sops not found in the apt repos; installing from GitHub release..." + local sops_ver sops_url + sops_ver="$(curl -fsSL https://api.github.com/repos/getsops/sops/releases/latest | grep -oE '"tag_name":\s*"v[^"]+"' | head -1 | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+')" + if [[ -z "${sops_ver}" ]]; then + echo "Could not determine the latest SOPS release version." >&2 + exit 1 + fi + sops_url="https://github.com/getsops/sops/releases/download/${sops_ver}/sops-v${sops_ver#v}.linux.amd64" + curl -fsSL -o "/tmp/sops-${sops_ver}" "${sops_url}" && \ + sudo install -m 0755 "/tmp/sops-${sops_ver}" /usr/local/bin/sops && \ + rm -f "/tmp/sops-${sops_ver}" + fi + if ! command -v sops >/dev/null 2>&1; then + echo "sops is still unavailable after package installation." >&2 + exit 1 + fi +} + +sops_age_key_file() { + printf '%s\n' "${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" +} + +sops_age_recipient() { + local key_file="$1" + + awk -F': ' '/^# public key:/ { print $2; exit }' "${key_file}" +} + +# Decrypt a SOPS-encrypted YAML of `KEY: value` pairs into the current env. +# Usage: sops_load_secrets +# Requires the age key (sops_age_key_file) to decrypt; fails loudly if missing. +sops_load_secrets() { + local file="${1:?sops_load_secrets: missing file}" + local key_file + local out + + if [[ ! -f "${file}" ]]; then + echo "Missing encrypted secret file: ${file} (run ./jeannie secrets-init then sops -e)." >&2 + return 1 + fi + if ! sops_available; then + echo "sops not installed. Run ./jeannie secrets-init first." >&2 return 1 fi @@ -227,6 +7732,132 @@ s not installed. Run ./jeannie secrets-init first." >&2 fi } +sops_available() { + command -v sops >/dev/null 2>&1 +} + +secrets_init() { + local key_file + local key_dir + local recipient + local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" + local example_file="${REPO_ROOT}/.sops.yaml.example" + + require_debian_server "secrets-init" + ensure_sops_age_tools + + key_file="$(sops_age_key_file)" + key_dir="$(dirname "${key_file}")" + mkdir -p "${key_dir}" + chmod 700 "${key_dir}" + + if [[ ! -s "${key_file}" ]]; then + echo "Generating age identity at ${key_file}..." + age-keygen -o "${key_file}" + chmod 600 "${key_file}" + else + echo "Using existing age identity at ${key_file}." + chmod 600 "${key_file}" + fi + + recipient="$(sops_age_recipient "${key_file}")" + if [[ -z "${recipient}" ]]; then + echo "Could not read the public recipient from ${key_file}." >&2 + exit 1 + fi + + if [[ -e "${config_file}" ]]; then + if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then + echo "${config_file} still contains the placeholder age recipient." >&2 + exit 1 + fi + echo "SOPS config already exists: ${config_file}" + else + if [[ ! -s "${example_file}" ]]; then + echo "Missing ${example_file}" >&2 + exit 1 + fi + sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}" + echo "Wrote ${config_file} with Debian host age recipient ${recipient}." + fi + + echo "Private age key: ${key_file}" + echo "Public age recipient: ${recipient}" + echo "Review and commit ${config_file}; never commit ${key_file}." +} + +secrets_check_file() { + local file="$1" + local key_file="$2" + + case "${file}" in + *.json) + grep -q '"sops"' "${file}" || { + echo "${file} does not look SOPS-encrypted." >&2 + return 1 + } + ;; + *) + grep -q '^sops:' "${file}" || { + echo "${file} does not look SOPS-encrypted." >&2 + return 1 + } + ;; + esac + + if [[ -s "${key_file}" ]]; then + SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null + fi +} + +secrets_check() { + local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" + local key_file + local recipient="" + local failures=0 + local found_files=0 + local file + + key_file="$(sops_age_key_file)" + + if [[ ! -s "${config_file}" ]]; then + echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2 + failures=$((failures + 1)) + elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then + echo "${config_file} still contains the placeholder age recipient." >&2 + failures=$((failures + 1)) + fi + + if [[ -s "${key_file}" ]]; then + recipient="$(sops_age_recipient "${key_file}")" + if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then + echo "${config_file} does not include this host's age recipient ${recipient}." >&2 + failures=$((failures + 1)) + fi + else + echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting." + fi + + if command -v git >/dev/null 2>&1; then + while IFS= read -r file; do + [[ -n "${file}" ]] || continue + found_files=$((found_files + 1)) + secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1)) + done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json') + fi + + if ((failures > 0)); then + echo "Secret checks failed with ${failures} issue(s)." >&2 + exit 1 + fi + + if ((found_files == 0)); then + echo "SOPS config checks passed. No encrypted secret files are committed yet." + else + echo "SOPS config checks passed for ${found_files} encrypted secret file(s)." + fi +} + tailnet_policy_check() { "${REPO_ROOT}/scripts/validate-tailnet-policy" } @@ -276,7 +7907,17 @@ ai_check() { if backstage_brain_enabled; then if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}" - if python3 "${REPO_ROOT}/scripts/ai-model-check.py" "${endpoint}" "${model}" + if python3 - "${endpoint}" "${model}" <<'PY' +import json +import sys +import urllib.request + +endpoint, model = sys.argv[1:3] +with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response: + tags = json.loads(response.read().decode("utf-8")) +models = {item.get("name", "") for item in tags.get("models", [])} +raise SystemExit(0 if model in models or f"{model}:latest" in models else 1) +PY then printf '%-28s ok - %s\n' "ollama model" "${model}" else @@ -383,3 +8024,666 @@ Useful practice trigger: kubectl -n security-lab exec deploy/juice-shop -- sh -c 'id; uname -a' EOF } + +security_attack_path() { + require_debian_server "security-attack-path" + "${REPO_ROOT}/scripts/security-attack-path" "${@:2}" +} + +prompt_injection_lab() { + "${REPO_ROOT}/scripts/prompt-injection-lab" "${@:2}" +} + +incident_commander() { + "${REPO_ROOT}/scripts/incident-commander" "${@:2}" +} + +safety_case() { + "${REPO_ROOT}/scripts/safety-case" "${@:2}" +} + +agent_sandbox() { + "${REPO_ROOT}/scripts/agent-sandbox" "${@:2}" +} + +ai_scheduler() { + "${REPO_ROOT}/scripts/ai-scheduler" "${@:2}" +} + +ai_memory() { + "${REPO_ROOT}/scripts/ai-memory" "${@:2}" +} + +red_blue_loop() { + "${REPO_ROOT}/scripts/red-blue-loop" "${@:2}" +} + +promote() { + "${REPO_ROOT}/scripts/promote" "${@:2}" +} + +model_observe() { + "${REPO_ROOT}/scripts/model-observe" "${@:2}" +} + +heal() { + require_debian_server "heal" + "${REPO_ROOT}/scripts/heal" "${@:2}" +} + +validate_homelab() { + "${REPO_ROOT}/scripts/validate-homelab" +} + +access_audit() { + require_debian_server "access-audit" + "${REPO_ROOT}/scripts/access-audit" +} + +drill_restore() { + require_debian_server "drill-restore" + "${REPO_ROOT}/scripts/restore-drill" all +} + +drill_pihole_restore() { + "${REPO_ROOT}/scripts/restore-drill" pihole +} + +kubeconfig_readonly() { + require_debian_server "kubeconfig-readonly" + "${REPO_ROOT}/scripts/kubeconfig-readonly" +} + +capacity_report() { + require_debian_server "capacity" + "${REPO_ROOT}/scripts/capacity-report" "${@:2}" +} + +recover_plan() { + require_debian_server "recover-plan" + "${REPO_ROOT}/scripts/recover-plan" +} + +gitops_status() { + require_debian_server "gitops-status" + "${REPO_ROOT}/scripts/gitops-status" +} + +cert_check() { + "${REPO_ROOT}/scripts/cert-check" +} + +release_snapshot() { + require_debian_server "release-snapshot" + "${REPO_ROOT}/scripts/release-snapshot" +} + +backup_status() { + require_debian_server "backup-status" + "${REPO_ROOT}/scripts/backup-status" +} + +synthetic_checks() { + require_debian_server "synthetic-checks" + "${REPO_ROOT}/scripts/synthetic-checks" +} + +resource_budget() { + require_debian_server "resource-budget" + "${REPO_ROOT}/scripts/resource-budget" "${@:2}" +} + +capacity_advisor() { + require_debian_server "capacity-advisor" + "${REPO_ROOT}/scripts/capacity-advisor" "${@:2}" +} + +capacity_limits() { + require_debian_server "capacity-limits" + "${REPO_ROOT}/scripts/capacity-limits" "${@:2}" +} + +control_plane() { + require_debian_server "control-plane" + "${REPO_ROOT}/scripts/control-plane" "${@:2}" +} + +artifact_cache() { + require_debian_server "artifact-cache" + "${REPO_ROOT}/scripts/artifact-cache" "${@:2}" +} + +blockchain_devnet() { + require_debian_server "blockchain-devnet" + "${REPO_ROOT}/scripts/blockchain-devnet" "${@:2}" +} + +blockchain_test() { + "${REPO_ROOT}/scripts/blockchain-test" "${@:2}" +} + +blockchain_wallet() { + "${REPO_ROOT}/scripts/blockchain-wallet" "${@:2}" +} + +golden_ledger() { + "${REPO_ROOT}/scripts/golden-ledger" "${@:2}" +} + +route_inventory() { + require_debian_server "route-inventory" + "${REPO_ROOT}/scripts/route-inventory" +} + +explain() { + if [ "${2:-}" != "output" ]; then + require_debian_server "explain" + fi + "${REPO_ROOT}/scripts/explain" "${@:2}" +} + +recover_power() { + require_debian_server "recover-power" + record_change_journal "recover-power" "$@" + "${REPO_ROOT}/scripts/recover-power" "${@:2}" +} + +scorecard() { + require_debian_server "scorecard" + "${REPO_ROOT}/scripts/scorecard" +} + +grafana_dashboards() { + require_debian_server "grafana-dashboards" + "${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}" +} + +workers_manage() { + require_debian_server "workers" + "${REPO_ROOT}/scripts/workers" "${@:2}" +} + +record_change_journal() { + local command_name="$1" + shift || true + + "${REPO_ROOT}/scripts/change-journal" append "${command_name}" "$*" || true +} + +change_journal() { + "${REPO_ROOT}/scripts/change-journal" "${@:2}" +} + +homelab_map() { + "${REPO_ROOT}/scripts/homelab-map" "${@:2}" +} + +print_usage() { + cat <<'EOF' +Usage: ./jeannie [args] + +Base, Inventory, And Planning + help | -h | --help Show this grouped help. + validate Run repo validation checks. + inventory-check Validate the canonical homelab inventory. + preflight Run non-mutating infrastructure preflight checks. + doctor-preapply Run deeper pre-apply safety checks. + plan [all|provisioning|cluster|platform|apps|edge] + Run OpenTofu plans without applying changes. + +Build And Bootstrap + up Deploy the full homelab pipeline. + deploy-gitea Deploy the Debian-hosted Gitea container. + deploy-heimdall Deploy the Debian-hosted Heimdall dashboard. + deploy-matrix Deploy the Matrix homeserver (Synapse + Element + Postgres). + deploy-omniroute Deploy the OmniRoute LLM routing gateway. + deploy-hermes Deploy the Hermes agent (git-install + venv + gateway service). + bootstrap-gitea-repo Ensure the Gitea repo and SSH key wiring exist. + rpi-services Deploy Pi-hole, Unbound, and Uptime Kuma on RPi4. + ollama-setup Install/configure Ollama on the Debian host. + artifact-cache {status|up|down|instructions} + Manage optional Debian artifact caches. + blockchain-devnet {status|up|down|logs|rpc} + Manage the local Ethereum Anvil devnet. + blockchain-test [forge args...] Run Foundry tests for labs/blockchain. + blockchain-wallet {instructions|new|address|sign-message} + Practice dev wallet and signing workflows. + golden-ledger {show|check} Show or validate Pimox golden image versions. + +Cluster Lifecycle + rebuild-cluster Recreate the cluster through the guarded path. + stop-cluster Stop Kubernetes and worker VMs without destroy. + start-cluster Start Kubernetes and desired worker VMs. + workers + Manage Pimox/Kubernetes workers. + move-prometheus-stack-workers Move monitoring workloads to worker nodes. + doctor-versions [--all|--details|--verbose|--json] + Check Kubernetes/container tooling versions. + +Operator View And Reports + status [--all|--details|--verbose|--json] Problems-first cascade from host to public URLs. + status --heal-plan Build a guarded self-heal plan from status. + status --heal Dry-run auto-eligible self-heal actions. + scorecard Compact pass/warn/fail operator scorecard. + capacity Compact capacity and placement report. + capacity-advisor Decide whether to add VMs or fix placement/resources. + capacity-limits [namespace] Recommend requests/limits YAML snippets. + recover-plan Print disaster recovery order and prerequisites. + recover-power [--dry-run] Run or preview post-outage recovery. + heal {plan|apply} Plan/apply guarded self-healing actions. + incident {list|replay|triage} Classify incident text and suggest read-only next steps. + safety-case [--since REF] [PATH...] Generate risk, blast-radius, test, and rollback case. + agent-sandbox {check|policy} Classify proposed commands against agent policy. + impact [--since REF] [PATH...] Explain affected lab areas before changes. + review-last-change [REF] Summarize changed files, impact, validation. + map [--dot] Print the homelab dependency map. + change-journal {list|path} Show risky-command journal entries. + release-snapshot Write a pre-change release snapshot. + +Observability And GitOps + grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards. + gitops-status Print focused Argo CD health and sync status. + platform Deploy platform stack only. + edge Deploy/check the OCI edge stack only. + promote {plan|validate|rollback APP} Run canary promotion gates and rollback plan. + cert-check Check public DNS, TLS, and edge URL health. + backup-status Check backup and restore-drill freshness. + synthetic-checks Run end-to-end service probes. + resource-budget Report Kubernetes resource request/limit gaps. + route-inventory Report routes and Uptime Kuma coverage. + control-plane {status|taint|untaint|pods} + Keep normal pods off the Debian control plane. + explain |output Explain warnings/errors and fix commands. + +Recovery And State + backup-gitea Back up the Debian-hosted Gitea data. + drill-restore Run all restore drills. + drill-gitea-restore Run the Gitea restore drill. + drill-pihole-restore Run the Pi-hole restore drill. + state-backup Back up local OpenTofu state files. + +Focused Doctors + doctor-edge [--all|--details|--verbose|--json] + Diagnose edge routing and public exposure. + doctor-gitea [--all|--details|--verbose|--json] + Diagnose Gitea local/public access. + doctor-rpi [--all|--details|--verbose|--json] + Diagnose RPi services and DNS. + doctor-cluster [--all|--details|--verbose|--json] + Diagnose Kubernetes cluster health. + +Access, Secrets, And Policy + access-audit Audit SSH, Gitea, Kubernetes, and Tailscale access. + kubeconfig-readonly Create/check read-only kubeconfig material. + secrets-init Initialize SOPS/age secret tooling. + secrets-check Validate repo-managed secret prerequisites. + tailnet-policy-check Validate Tailscale ACL policy as code. + fix-debian-docker-root Repair Debian Docker root placement. + +Apps And Services + apps Deploy application stack only. + website-translation-model Prepare website translation model support. + website-ollama-listen Configure website Ollama access. + install-gitea-runner [TOKEN] Install the Gitea Actions runner. + openwrt Deploy/check OpenWrt lab config. + +Security Learning + security-scan Run the security scan bundle. + security-prepare Prepare defensive security tools. + security-zap Run OWASP ZAP checks. + security-k8s Run Kubernetes security checks. + security-host Run host security checks. + security-trivy Run Trivy scans. + security-secrets Run secret leak checks. + security-nuclei Run Nuclei web checks. + security-web Run web security checks. + security-logs Review security-relevant logs. + security-runtime Check runtime security sensors. + security-attack-path Print prioritized attack-path findings. + prompt-injection-lab {list|run|show} Run local prompt-injection defense drills. + red-blue {plan|run|ledger} Run or plan safe red-team/blue-team loop. + +AI And Indexing + ai-index Build the local homelab RAG index. + ai-check Query/check the local AI index. + ask [--citations] QUESTION... Find relevant docs/runbooks/commands. + ai-evals {list|run|show} Run deterministic Jeannie/RAG eval cases. + ai-scheduler {simulate|recommend NAME} Simulate AI workload placement policy. + ai-memory {check} Check local RAG index provenance freshness. + model-observe {prompts|run|report} Track model behavior over stable prompts. + +Destructive + nuke Guarded cluster state destruction path. +EOF +} + +if [[ "${JEANNIE_LIBRARY_MODE:-false}" == "true" ]]; then + # shellcheck disable=SC2317 + return 0 2>/dev/null || exit 0 +fi + +case "${1:-}" in + __up-task) + up_task "${2:-}" + ;; + "" | help | -h | --help) + print_usage + ;; + up) + record_change_journal "up" "$@" + up + ;; + plan) + plan_homelab "${2:-all}" + ;; + rebuild-cluster) + record_change_journal "rebuild-cluster" "$@" + rebuild_cluster + ;; + stop-cluster) + record_change_journal "stop-cluster" "$@" + stop_cluster + ;; + start-cluster) + record_change_journal "start-cluster" "$@" + start_cluster + ;; + status) + status_report "${@:2}" + ;; + capacity) + capacity_report "$@" + ;; + capacity-advisor) + capacity_advisor "$@" + ;; + capacity-limits) + capacity_limits "$@" + ;; + recover-plan) + recover_plan + ;; + gitops-status) + gitops_status + ;; + platform) + record_change_journal "platform" "$@" + require_debian_server "platform" + jeannie_log_start "platform" + jeannie_step_plan 1 + run_step "Platform OpenTofu apply" platform_apply + echo "Log: ${JEANNIE_LOG_FILE}" + ;; + edge) + record_change_journal "edge" "$@" + edge_apply + ;; + promote) + promote "$@" + ;; + cert-check) + cert_check + ;; + release-snapshot) + record_change_journal "release-snapshot" "$@" + release_snapshot + ;; + backup-status) + backup_status + ;; + synthetic-checks) + synthetic_checks + ;; + resource-budget) + resource_budget "$@" + ;; + control-plane) + control_plane "$@" + ;; + artifact-cache) + artifact_cache "$@" + ;; + blockchain-devnet) + blockchain_devnet "$@" + ;; + blockchain-test) + blockchain_test "$@" + ;; + blockchain-wallet) + blockchain_wallet "$@" + ;; + golden-ledger) + golden_ledger "$@" + ;; + route-inventory) + route_inventory + ;; + explain) + explain "$@" + ;; + recover-power) + recover_power "$@" + ;; + heal) + heal "$@" + ;; + incident) + incident_commander "$@" + ;; + safety-case) + safety_case "$@" + ;; + agent-sandbox) + agent_sandbox "$@" + ;; + scorecard) + scorecard + ;; + grafana-dashboards) + grafana_dashboards "$@" + ;; + workers) + workers_manage "$@" + ;; + change-journal) + change_journal "$@" + ;; + map) + homelab_map "$@" + ;; + validate) + validate_homelab + ;; + access-audit) + access_audit + ;; + kubeconfig-readonly) + kubeconfig_readonly + ;; + apps) + record_change_journal "apps" "$@" + require_debian_server "apps" + jeannie_log_start "apps" + jeannie_step_plan 1 + run_step "Applications" apps + echo "Log: ${JEANNIE_LOG_FILE}" + ;; + website-translation-model) + website_translation_model + ;; + website-ollama-listen) + website_ollama_listen + ;; + ollama-setup) + ollama_setup + ;; + deploy-gitea) + record_change_journal "deploy-gitea" "$@" + deploy_gitea + ;; + deploy-heimdall) + record_change_journal "deploy-heimdall" "$@" + deploy_heimdall + ;; + deploy-matrix) + record_change_journal "deploy-matrix" "$@" + deploy_matrix + ;; + deploy-omniroute) + record_change_journal "deploy-omniroute" "$@" + deploy_omniroute + ;; + deploy-hermes) + record_change_journal "deploy-hermes" "$@" + deploy_hermes + ;; + rpi-services) + record_change_journal "rpi-services" "$@" + deploy_rpi_services + ;; + bootstrap-gitea-repo) + bootstrap_gitea_repo + ;; + backup-gitea) + backup_gitea + ;; + drill-restore) + drill_restore + ;; + drill-gitea-restore) + drill_gitea_restore + ;; + drill-pihole-restore) + drill_pihole_restore + ;; + install-gitea-runner) + install_gitea_runner "${2:-}" + ;; + move-prometheus-stack-workers) + move_prometheus_stack_workers + ;; + doctor-versions) + doctor_versions "${@:2}" + ;; + doctor-edge) + doctor_edge "${@:2}" + ;; + doctor-gitea) + doctor_gitea "${@:2}" + ;; + doctor-rpi) + doctor_rpi "${@:2}" + ;; + doctor-cluster) + doctor_cluster "${@:2}" + ;; + preflight) + homelab_preflight + ;; + doctor-preapply) + doctor_preapply + ;; + inventory-check) + inventory_check + ;; + state-backup) + backup_tofu_state + ;; + fix-debian-docker-root) + fix_debian_docker_root + ;; + secrets-init) + secrets_init + ;; + secrets-check) + secrets_check + ;; + tailnet-policy-check) + tailnet_policy_check + ;; + ai-index) + ai_index + ;; + ai-check) + ai_check + ;; + ask) + ask_homelab "$@" + ;; + ai-evals) + ai_evals "$@" + ;; + ai-scheduler) + ai_scheduler "$@" + ;; + ai-memory) + ai_memory "$@" + ;; + model-observe) + model_observe "$@" + ;; + impact) + impact "$@" + ;; + review-last-change) + review_last_change "$@" + ;; + security-scan) + security_scan + ;; + security-prepare) + security_prepare + ;; + security-zap) + security_zap + ;; + security-k8s) + security_k8s + ;; + security-host) + security_host + ;; + security-trivy) + security_trivy + ;; + security-secrets) + security_secrets + ;; + security-nuclei) + security_nuclei + ;; + security-web) + security_web + ;; + security-logs) + security_logs + ;; + security-runtime) + security_runtime + ;; + security-attack-path) + security_attack_path "$@" + ;; + prompt-injection-lab) + prompt_injection_lab "$@" + ;; + red-blue) + red_blue_loop "$@" + ;; + openwrt) + openwrt + ;; + nuke) + record_change_journal "nuke" "$@" + require_debian_server "nuke" + jeannie_log_start "nuke" + jeannie_step_plan 1 + run_step "Nuke homelab cluster state" nuke + echo "Log: ${JEANNIE_LOG_FILE}" + ;; + *) + printf 'Unknown command: %s\n\n' "$1" >&2 + print_usage >&2 + exit 1 + ;; +esac