#!/usr/bin/env bash set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BUILDX_CONFIG="/tmp/buildx-config.toml" HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}" COSIGN_KEY_PREFIX="${COSIGN_KEY_PREFIX:-${HOMELAB_STATE_DIR}/cosign/cosign}" COSIGN_KEY_PATH="${COSIGN_KEY_PATH:-${COSIGN_KEY_PREFIX}.key}" COSIGN_PUBLIC_KEY_PATH="${COSIGN_PUBLIC_KEY_PATH:-${COSIGN_KEY_PREFIX}.pub}" COSIGN_PASSWORD_FILE="${COSIGN_PASSWORD_FILE:-${HOMELAB_STATE_DIR}/cosign/cosign.password}" COSIGN_VERSION="${COSIGN_VERSION:-2.6.3}" COSIGN_BIN="${COSIGN_BIN:-}" HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-homelab-cosign-public-key}" HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}" JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}" JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}" JEANNIE_STEP_INDEX=0 JEANNIE_STEP_TOTAL=0 trap 'rm -f "${BUILDX_CONFIG}"' EXIT load_homelab_inventory_defaults() { local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" local rendered_defaults if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then return 0 fi rendered_defaults="$(python3 - "${inventory_file}" <<'PY' import re import shlex import sys inventory_file = sys.argv[1] mapping = { "domain.base": "LAB_DOMAIN", "domain.public_url": "LAB_PUBLIC_URL", "domain.gitea_url": "LAB_GITEA_ROOT_URL", "network.lan_cidr": "LAB_LAN_CIDR", "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX", "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP", "hosts.debian.user": "LAB_DEBIAN_USER", "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP", "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP", "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT", "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH", "hosts.rpi4.user": "LAB_RPI_USER", "hosts.rpi4.lan_ip": "LAB_RPI_HOST", "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", "hosts.opi5_pimox.user": "LAB_PIMOX_USER", "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST", "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE", "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE", "hosts.oci_edge.user": "LAB_EDGE_USER", "hosts.oci_edge.public_ip": "LAB_EDGE_HOST", "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR", "services.gitea.http_port": "LAB_GITEA_HTTP_PORT", "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", "services.gitea.root_url": "LAB_GITEA_ROOT_URL", "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS", "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR", "services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE", "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT", "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT", "ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER", "ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED", "ai_gateway.url": "LAB_AI_GATEWAY_URL", "ai_gateway.model": "LAB_AI_GATEWAY_MODEL", "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS", "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR", "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", } def flatten(document, prefix=""): values = {} if isinstance(document, dict): for key, value in document.items(): path = f"{prefix}.{key}" if prefix else str(key) if isinstance(value, dict): values.update(flatten(value, path)) elif isinstance(value, list): values[path] = ",".join(str(item) for item in value) elif value is not None: values[path] = str(value) return values def parse_simple_inventory(path): stack = [] values = {} pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") with open(path, encoding="utf-8") as handle: for raw_line in handle: if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): continue match = pattern.match(raw_line.rstrip("\n")) if not match: continue indent = len(match.group(1)) key = match.group(2) value = (match.group(3) or "").strip() while stack and stack[-1][0] >= indent: stack.pop() current_path = ".".join([item[1] for item in stack] + [key]) if value == "": stack.append((indent, key)) continue if " #" in value: value = value.split(" #", 1)[0].strip() values[current_path] = value.strip("\"'") return values try: import yaml except ImportError: values = parse_simple_inventory(inventory_file) else: with open(inventory_file, encoding="utf-8") as handle: values = flatten(yaml.safe_load(handle) or {}) for path, env_name in mapping.items(): value = values.get(path) if value: print(f": ${{{env_name}:={shlex.quote(value)}}}") debian_ip = values.get("hosts.debian.lan_ip") gitea_http = values.get("services.gitea.http_port") if debian_ip: print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}") if debian_ip and gitea_http: print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}") gitea_ts = values.get("hosts.debian.tailscale_ip") if gitea_ts: print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}") PY )" if [[ -n "${rendered_defaults}" ]]; then eval "${rendered_defaults}" fi } load_homelab_inventory_defaults export_if_unset() { local name="$1" local value="$2" if [[ -z "${value}" ]]; then return 0 fi if [[ -z "${!name:-}" ]]; then printf -v "${name}" '%s' "${value}" fi # shellcheck disable=SC2163 export "${name?}" } export_homelab_inventory_tf_vars() { export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}" export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}" export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}" export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}" export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}" export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}" export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}" export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}" export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}" export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}" export_if_unset TF_VAR_pimox_template_build_host "${LAB_DEBIAN_LAN_IP:-}" export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}" export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}" export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}" export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}" export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}" export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}" export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" } export_homelab_inventory_tf_vars KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}" require_debian_server() { local command_name="$1" local os_id="" if [[ "$(uname -s)" != "Linux" ]]; then echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2 exit 1 fi if [[ -r /etc/os-release ]]; then os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)" fi if [[ "${os_id}" != "debian" ]]; then echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2 exit 1 fi } tofu_state_has_resource() { local stack="$1" local resource_address="$2" tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1 } helm_release_secret_exists() { local namespace="$1" local release_name="$2" local secret_name secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \ -l "owner=helm,name=${release_name}" \ -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" [[ -n "${secret_name}" ]] } kubernetes_resource_exists() { local namespace="$1" local resource_kind="$2" local resource_name="$3" if [[ -n "${namespace}" ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 return $? fi kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 } adopt_tofu_helm_release() { local stack="$1" local resource_address="$2" local namespace="$3" local release_name="$4" if tofu_state_has_resource "${stack}" "${resource_address}"; then return 0 fi if ! helm_release_secret_exists "${namespace}" "${release_name}"; then return 0 fi echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..." tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}" } adopt_tofu_kubernetes_resource() { local stack="$1" local resource_address="$2" local namespace="$3" local resource_kind="$4" local resource_name="$5" local import_id="$6" if tofu_state_has_resource "${stack}" "${resource_address}"; then return 0 fi if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then return 0 fi echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..." tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" } adopt_tofu_kubernetes_manifest() { local stack="$1" local resource_address="$2" local namespace="$3" local kubectl_kind="$4" local api_version="$5" local manifest_kind="$6" local resource_name="$7" local import_id if tofu_state_has_resource "${stack}" "${resource_address}"; then return 0 fi if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then return 0 fi import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}" echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..." tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" } adopt_platform_existing_resources() { local stack="bootstrap/platform" adopt_tofu_helm_release "${stack}" "helm_release.calico_crds" "tigera-operator" "calico-crds" adopt_tofu_helm_release "${stack}" "helm_release.calico" "tigera-operator" "calico" adopt_tofu_helm_release "${stack}" "helm_release.openebs" "openebs" "openebs" adopt_tofu_helm_release "${stack}" "helm_release.metallb[\"enabled\"]" "metallb-system" "metallb" adopt_tofu_helm_release "${stack}" "helm_release.traefik[\"enabled\"]" "traefik" "traefik" adopt_tofu_helm_release "${stack}" "helm_release.argocd" "argocd" "argocd" adopt_tofu_helm_release "${stack}" "helm_release.kyverno" "kyverno" "kyverno" adopt_tofu_helm_release "${stack}" "helm_release.kyverno_policies" "kyverno" "kyverno-policies" adopt_tofu_helm_release "${stack}" "helm_release.loki" "monitoring" "loki" adopt_tofu_helm_release "${stack}" "helm_release.promtail" "monitoring" "promtail" adopt_tofu_helm_release "${stack}" "helm_release.prometheus_stack" "monitoring" "prometheus-stack" adopt_tofu_kubernetes_resource \ "${stack}" \ "kubernetes_storage_class_v1.openebs_hostpath_retain" \ "" \ "storageclass" \ "openebs-hostpath-retain" \ "openebs-hostpath-retain" adopt_tofu_kubernetes_resource \ "${stack}" \ "kubernetes_namespace_v1.monitoring" \ "" \ "namespace" \ "monitoring" \ "monitoring" } adopt_apps_existing_resources() { local stack="bootstrap/apps" local namespace="${TF_VAR_argocd_namespace:-argocd}" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["container-registry"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "container-registry" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["website-production"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "website-production" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["demos-static"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "demos-static" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["heimdall"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "heimdall" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["n8n"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "n8n" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["supply-chain-policy"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "supply-chain-policy" } ensure_homelab_node_labels() { local control_plane_node="${LAB_CONTROL_PLANE_NODE_NAME:-debian}" local raspberry_node="${LAB_RASPBERRY_NODE_NAME:-raspberry}" local prometheus_selector="homelab.dev/node-role=app,homelab.dev/storage=ssd" local node local target_nodes echo "Applying homelab labels to existing Kubernetes nodes..." while IFS= read -r node; do [[ -n "${node}" ]] || continue if [[ "${node}" == "${control_plane_node}" ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=control-plane \ homelab.dev/storage=local \ homelab.dev/workload-class=control-plane \ --overwrite continue fi kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ node-role.kubernetes.io/worker=worker \ --overwrite if [[ "${node}" == pimox-worker-* ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=app \ homelab.dev/storage=ssd \ homelab.dev/workload-class=platform \ --overwrite elif [[ "${node}" == "${raspberry_node}" ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=edge-app \ homelab.dev/storage=local \ homelab.dev/workload-class=edge \ --overwrite fi done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)" if [[ -z "${target_nodes}" ]]; then echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2 exit 1 fi } delete_prometheus_stack_storage() { local namespace="${1:-monitoring}" local pattern='(^|-)prometheus-stack-(prometheus|alertmanager|grafana)(-|$)|^prometheus-prometheus-stack|^alertmanager-prometheus-stack|^storage-prometheus-stack-grafana' local pvc_names local pv_names pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null | awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')" pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \ -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null | awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')" if [[ -n "${pvc_names}" ]]; then echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..." printf '%s\n' "${pvc_names}" | xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s fi if [[ -n "${pv_names}" ]]; then echo "Deleting old prometheus-stack retained PV objects..." printf '%s\n' "${pv_names}" | xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false fi } run_tofu_stack() { local stack="$1" local auto_approve="${LAB_AUTO_APPROVE:-true}" local -a apply_args=() if truthy "${auto_approve}"; then apply_args+=("-auto-approve") elif ! disabled_value "${auto_approve}"; then echo "LAB_AUTO_APPROVE must be true or false." >&2 exit 1 fi if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") fi tofu -chdir="${REPO_ROOT}/${stack}" init if [[ "${stack}" == "bootstrap/platform" ]]; then adopt_platform_existing_resources fi if [[ "${stack}" == "bootstrap/apps" ]]; then adopt_apps_existing_resources fi tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}" } run_tofu_plan_stack() { local stack="$1" local -a plan_args=() if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") fi tofu -chdir="${REPO_ROOT}/${stack}" init tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}" } backup_tofu_state() { local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}" local timestamp local archive local -a paths=() local path timestamp="$(date +%Y%m%d-%H%M%S)" archive="${backup_dir}/tofu-state-${timestamp}.tgz" mkdir -p "${backup_dir}" for path in \ bootstrap/provisioning/terraform.tfstate \ bootstrap/provisioning/terraform.tfstate.backup \ bootstrap/cluster/terraform.tfstate \ bootstrap/cluster/terraform.tfstate.backup \ bootstrap/platform/terraform.tfstate \ bootstrap/platform/terraform.tfstate.backup \ bootstrap/apps/terraform.tfstate \ bootstrap/apps/terraform.tfstate.backup \ bootstrap/edge/terraform.tfstate \ bootstrap/edge/terraform.tfstate.backup \ .lab/cluster-workers.auto.tfvars.json \ .lab/pimox-workers.tsv \ .lab/manual-workers.tsv; do if [[ -e "${REPO_ROOT}/${path}" ]]; then paths+=("${path}") fi done if ((${#paths[@]} == 0)); then echo "No local OpenTofu state files found to back up." return 0 fi tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}" chmod 0600 "${archive}" echo "Backed up local OpenTofu state to ${archive}." } move_prometheus_stack_workers() { local stack="bootstrap/platform" local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}" local auto_approve="${LAB_AUTO_APPROVE:-true}" local -a approve_args=() require_debian_server "move-prometheus-stack-workers" if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then cat >&2 <<'EOF' move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data. Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended. EOF exit 1 fi if truthy "${auto_approve}"; then approve_args+=("-auto-approve") elif ! disabled_value "${auto_approve}"; then echo "LAB_AUTO_APPROVE must be true or false." >&2 exit 1 fi export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" export KUBECONFIG="${TF_VAR_kubeconfig_path}" echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted." ensure_homelab_node_labels tofu -chdir="${REPO_ROOT}/${stack}" init adopt_platform_existing_resources tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}" delete_prometheus_stack_storage "${namespace}" tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide } doctor_versions_report() { local api_version local api_minor local containerd_major_count local kubelet_minor_count local node_rows local status=0 require_debian_server "doctor-versions" ensure_python3 api_version="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json | python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' )" api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" node_rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' )" if [[ -z "${node_rows}" ]]; then echo "No Kubernetes nodes found." >&2 return 1 fi printf 'API server: %s\n\n' "${api_version}" printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME" printf '%s\n' "${node_rows}" | awk -F '\t' '{ version = $2 gsub(/^v/, "", version) split(version, parts, ".") printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3 }' kubelet_minor_count="$( printf '%s\n' "${node_rows}" | awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | sort -u | wc -l | tr -d ' ' )" containerd_major_count="$( printf '%s\n' "${node_rows}" | awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | sort -u | wc -l | tr -d ' ' )" if [[ "${kubelet_minor_count}" != "1" ]]; then echo echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2 status=1 fi if printf '%s\n' "${node_rows}" | awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then echo echo "At least one kubelet does not match the API server minor (${api_minor})." >&2 status=1 fi if [[ "${containerd_major_count}" -gt 1 ]]; then echo echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2 status=1 fi if [[ "${status}" -eq 0 ]]; then echo echo "Node Kubernetes and containerd versions are aligned." fi return "${status}" } doctor_versions() { doctor_versions_report exit "$?" } truthy() { case "${1,,}" in 1 | true | yes | on) return 0 ;; *) return 1 ;; esac } disabled_value() { case "${1,,}" in 0 | false | no | off | disabled) return 0 ;; *) return 1 ;; esac } jeannie_log_start() { local command_name="$1" local timestamp if [[ -n "${JEANNIE_LOG_FILE}" ]]; then return 0 fi timestamp="$(date +%Y%m%d-%H%M%S)" mkdir -p "${JEANNIE_LOG_DIR}" JEANNIE_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.log" : >"${JEANNIE_LOG_FILE}" printf 'Log: %s\n' "${JEANNIE_LOG_FILE}" } jeannie_step_plan() { JEANNIE_STEP_INDEX=0 JEANNIE_STEP_TOTAL="$1" } jeannie_verbose_enabled() { truthy "${JEANNIE_VERBOSE:-false}" } redact_sensitive_output() { sed -E \ -e 's/(GITEA_BOOTSTRAP_PASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(PIHOLE_WEBPASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(COSIGN_PASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(GITEA_RUNNER_REGISTRATION_TOKEN=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(--password[=[:space:]]+)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(--token[=[:space:]]+)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \ -e 's/(password[=:][[:space:]]*)[^[:space:]]+/\1[REDACTED]/Ig' \ -e 's/(token[=:][[:space:]]*)[^[:space:]]+/\1[REDACTED]/Ig' \ -e 's#(https?://[^:/[:space:]]+):[^@/[:space:]]+@#\1:[REDACTED]@#g' } run_step() { local description="$1" local step_log local status shift JEANNIE_STEP_INDEX=$((JEANNIE_STEP_INDEX + 1)) if ((JEANNIE_STEP_TOTAL > 0)); then printf '[%d/%d] %-42s ' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}" else printf '[%d] %-46s ' "${JEANNIE_STEP_INDEX}" "${description}" fi step_log="$(mktemp)" { printf '\n== [%d/%d] %s ==\n' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}" printf 'Started: %s\n' "$(date -Is)" } >>"${JEANNIE_LOG_FILE}" if jeannie_verbose_enabled; then set +e "$@" 2>&1 | redact_sensitive_output | tee "${step_log}" status=${PIPESTATUS[0]} set -e else set +e "$@" 2>&1 | redact_sensitive_output >"${step_log}" status=${PIPESTATUS[0]} set -e fi cat "${step_log}" >>"${JEANNIE_LOG_FILE}" printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}" if ((status == 0)); then printf 'ok\n' rm -f "${step_log}" return 0 fi printf 'failed\n\n' printf 'Step failed: %s\n' "${description}" >&2 printf 'Exit status: %s\n' "${status}" >&2 printf 'Full step output:\n' >&2 sed 's/^/ /' "${step_log}" >&2 printf '\nFull log: %s\n' "${JEANNIE_LOG_FILE}" >&2 rm -f "${step_log}" return "${status}" } worker_index_is_skipped() { local index="$1" local skip_indexes="$2" local skip_index skip_indexes="${skip_indexes//,/ }" for skip_index in ${skip_indexes}; do [[ -z "${skip_index}" ]] && continue if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2 exit 1 fi if ((skip_index == index)); then return 0 fi done return 1 } ensure_python3() { if command -v python3 >/dev/null 2>&1; then return 0 fi sudo apt-get update sudo apt-get install -y --no-install-recommends python3 } detect_route_interface() { local target="$1" ip route get "${target}" 2>/dev/null | awk ' { for (i = 1; i <= NF; i++) { if ($i == "dev") { print $(i + 1) exit } } } ' } pimox_ssh() { local host="$1" local user="$2" local key_path="$3" shift 3 ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@" } pimox_guest_ipv4() { local guest_json local host="$1" local user="$2" local key_path="$3" local vmid="$4" local ip_prefix="$5" local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)" if [[ -z "${guest_json}" ]]; then return 1 fi GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY' import json import os import sys prefix = sys.argv[1] try: interfaces = json.loads(os.environ.get("GUEST_JSON", "")) except Exception: sys.exit(1) for iface in interfaces or []: for address in iface.get("ip-addresses") or []: if address.get("ip-address-type") != "ipv4": continue ip = address.get("ip-address", "") if not ip or ip.startswith(("127.", "169.254.")): continue if prefix and not ip.startswith(prefix): continue print(ip) sys.exit(0) sys.exit(1) PY } pimox_worker_vm_debug() { local host="$1" local user="$2" local key_path="$3" local vmid="$4" local qm_bin="$5" pimox_ssh "${host}" "${user}" "${key_path}" "set +e echo 'Pimox VM ${vmid} status:' sudo '${qm_bin}' status '${vmid}' echo 'Pimox VM ${vmid} config summary:' sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:' sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" >&2 || true } wait_for_pimox_guest_ssh() { local host="$1" local user="$2" local key_path="$3" local vmid="$4" local guest_user="$5" local guest_key_path="$6" local ip_prefix="$7" local timeout_seconds="$8" local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" local deadline local elapsed local guest_ip local ip_filter_description local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" local last_guest_ip="" local last_known_hosts_ip="" local last_ssh_output="" local next_log local ssh_deadline=0 local ssh_output local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}" ip_filter_description="matching prefix ${ip_prefix}" if [[ -z "${ip_prefix}" ]]; then ip_filter_description="that is not loopback or link-local" fi if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2 return 1 fi mkdir -p "$(dirname "${known_hosts_file}")" touch "${known_hosts_file}" chmod 0600 "${known_hosts_file}" deadline=$((SECONDS + timeout_seconds)) next_log="${SECONDS}" while ((SECONDS < deadline)); do guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)" if [[ -n "${guest_ip}" ]]; then if ((ssh_deadline == 0)); then ssh_deadline=$((SECONDS + ssh_timeout_seconds)) elif ((SECONDS >= ssh_deadline)); then break fi last_guest_ip="${guest_ip}" if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true last_known_hosts_ip="${guest_ip}" fi if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then printf '%s\n' "${guest_ip}" return 0 fi last_ssh_output="${ssh_output}" fi if ((SECONDS >= next_log)); then elapsed=$((timeout_seconds - (deadline - SECONDS))) if [[ -n "${last_guest_ip}" ]]; then echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2 if [[ -n "${last_ssh_output}" ]]; then echo "Last SSH failure: ${last_ssh_output}" >&2 fi else echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2 fi next_log=$((SECONDS + 60)) fi sleep 10 done if [[ -n "${last_guest_ip}" ]]; then echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2 if [[ -n "${last_ssh_output}" ]]; then echo "Last SSH failure: ${last_ssh_output}" >&2 fi else echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2 fi pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" return 1 } pimox_generated_mac() { local vmid="$1" printf '02:68:10:%02x:%02x:%02x\n' \ $(((vmid >> 16) & 255)) \ $(((vmid >> 8) & 255)) \ $((vmid & 255)) } cpuset_cpu_count() { local cpuset="$1" local count=0 local part local start local end local -a parts IFS=',' read -r -a parts <<<"${cpuset}" for part in "${parts[@]}"; do if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then start="${BASH_REMATCH[1]}" end="${BASH_REMATCH[2]}" if ((end < start)); then return 1 fi count=$((count + end - start + 1)) elif [[ "${part}" =~ ^[0-9]+$ ]]; then count=$((count + 1)) else return 1 fi done printf '%s\n' "${count}" } pimox_worker_cpu_affinity() { local index="$1" local affinities="$2" local worker_cores="$3" local affinity local affinity_index=1 local cpu_count for affinity in ${affinities}; do if ((affinity_index == index)); then if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 exit 1 fi if ((cpu_count != worker_cores)); then echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2 exit 1 fi printf '%s\n' "${affinity}" return 0 fi affinity_index=$((affinity_index + 1)) done echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2 exit 1 } ensure_pimox_worker_node() { local index="$1" local spec_file="$2" local pimox_host="$3" local pimox_user="$4" local pimox_key="$5" local template_vmid="$6" local bridge="$7" local worker_base_vmid="$8" local worker_name_prefix="$9" local worker_node_prefix="${10}" local worker_key_prefix="${11}" local worker_cores="${12}" local worker_memory="${13}" local worker_user="${14}" local worker_key_path="${15}" local ip_prefix="${16}" local timeout_seconds="${17}" local qm_bin="${18}" local worker_storage="${19}" local worker_replace_existing="${20}" local worker_cpu_affinity="${21}" local padded local vmid local worker_key local worker_name local node_name local mac local guest_ip printf -v padded '%02d' "${index}" vmid=$((worker_base_vmid + index - 1)) worker_key="${worker_key_prefix}${padded}" worker_name="${worker_name_prefix}-${padded}" node_name="${worker_node_prefix}-${padded}" mac="$(pimox_generated_mac "${vmid}")" if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2 exit 1 fi if ! truthy "${worker_replace_existing}" && ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}." worker_replace_existing=true fi if truthy "${worker_replace_existing}"; then echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..." pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true elapsed=0 while [ \"\$elapsed\" -lt 300 ]; do if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then break fi sleep 5 elapsed=\$((elapsed + 5)) done sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" else pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in *'Unknown option: affinity'*) echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' ;; *) printf '%s\n' \"\$affinity_output\" >&2 exit 1 ;; esac } fi if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi" fi fi if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu if ! ip link show '${bridge}' >/dev/null 2>&1; then echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2 exit 1 fi pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then pvesm_cmd=/usr/sbin/pvesm fi if [ -z \"\$pvesm_cmd\" ]; then echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2 exit 1 fi if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2 exit 1 fi sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' set '${vmid}' --agent enabled=1 sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in *'Unknown option: affinity'*) echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' ;; *) printf '%s\n' \"\$affinity_output\" >&2 exit 1 ;; esac } fi sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}' sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0' sudo '${qm_bin}' set '${vmid}' --onboot 1 sudo '${qm_bin}' start '${vmid}'" fi if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2 exit 1 fi printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" } write_cluster_worker_var_file() { local var_file="$1" local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}' local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}" local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}" shift LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \ LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \ LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \ LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \ LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \ LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \ LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \ LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \ python3 - "${var_file}" "$@" <<'PY' import json import os from pathlib import Path import sys var_file = sys.argv[1] spec_files = sys.argv[2:] nodes = {} node_labels = {} try: raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"]) manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"]) pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"]) except json.JSONDecodeError as exc: raise SystemExit(f"Invalid node label JSON: {exc}") from exc if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}: nodes["raspberrypi"] = { "host": os.environ["LAB_RASPBERRY_HOST"], "user": os.environ["LAB_RASPBERRY_USER"], "node_name": os.environ["LAB_RASPBERRY_NODE_NAME"], "ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"], } node_labels["raspberrypi"] = raspberry_labels for spec_file in spec_files: path = Path(spec_file) if not path.exists() or path.stat().st_size == 0: continue labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels with path.open(encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): line = line.rstrip("\n") if not line or line.startswith("#"): continue try: key, host, user, node_name, ssh_key_path = line.split("\t") except ValueError as exc: raise SystemExit( f"{spec_file}:{line_number}: expected tab-separated " "key, host, user, node_name, ssh_key_path" ) from exc nodes[key] = { "host": host, "user": user, "node_name": node_name, "ssh_key_path": ssh_key_path, } node_labels[key] = labels with open(var_file, "w", encoding="utf-8") as handle: json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2) handle.write("\n") PY } prepare_cluster_worker_var_file() { local include_raspberry_default="$1" local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv" local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}" mkdir -p "${REPO_ROOT}/.lab" write_cluster_worker_var_file "${var_file}" "${manual_spec_file}" "${pimox_spec_file}" export LAB_CLUSTER_VAR_FILE="${var_file}" } preflight_check() { local description="$1" local output shift printf '%s %s... ' '-->' "${description}" if output="$("$@" 2>&1)"; then printf 'ok\n' return 0 fi printf 'failed\n' if [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed 's/^/ /' >&2 fi return 1 } preflight_warn() { local description="$1" local output shift printf '%s %s... ' '-->' "${description}" if output="$("$@" 2>&1)"; then printf 'ok\n' return 0 fi printf 'warning\n' if [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed 's/^/ /' >&2 fi return 0 } check_gitea_reachable() { local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" curl -fsS --max-time 10 "${gitea_url}" >/dev/null } check_debian_docker_root() { local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" local actual_root if ! command -v docker >/dev/null 2>&1; then echo "docker command was not found on Debian host" >&2 return 1 fi actual_root="$(sudo docker info --format '{{.DockerRootDir}}')" if [[ "${actual_root}" != "${expected_root}" ]]; then echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2 return 1 fi } fix_debian_docker_root() { local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" local daemon_file="/etc/docker/daemon.json" local current_root="" require_debian_server "fix-debian-docker-root" if ! command -v docker >/dev/null 2>&1; then echo "docker command was not found on Debian host" >&2 exit 1 fi current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" if [[ "${current_root}" == "${expected_root}" ]]; then echo "Debian Docker root is already ${expected_root}." return 0 fi echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..." sudo mkdir -p "${expected_root}" /etc/docker echo "Stopping Docker services..." sudo systemctl stop docker 2>/dev/null || true sudo systemctl stop containerd 2>/dev/null || true if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then echo "Copying existing Docker data to ${expected_root}..." sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/" fi sudo python3 - "${daemon_file}" "${expected_root}" <<'PY' import json import os import sys path, expected_root = sys.argv[1:3] document = {} if os.path.exists(path) and os.path.getsize(path) > 0: with open(path, encoding="utf-8") as handle: document = json.load(handle) document["data-root"] = expected_root with open(path, "w", encoding="utf-8") as handle: json.dump(document, handle, indent=2, sort_keys=True) handle.write("\n") PY echo "Starting Docker services..." sudo systemctl start containerd sudo systemctl start docker check_debian_docker_root echo "Debian Docker root is now ${expected_root}." } check_debian_tailscale_ip() { local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}" if [[ -z "${expected_ip}" ]]; then return 0 fi if ! command -v tailscale >/dev/null 2>&1; then echo "tailscale command was not found on Debian host" >&2 return 1 fi if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then echo "Debian Tailscale IP does not include ${expected_ip}" >&2 tailscale ip -4 >&2 || true return 1 fi } check_edge_ssh() { local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" local edge_user="${LAB_EDGE_USER:-ubuntu}" ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true" } check_rpi_ssh() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true" } check_rpi_docker_root_state() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" if [ \"\$actual_root\" = '${nvme_root}' ]; then mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; } exit 0 fi if [ \"\$actual_root\" = '${fallback_root}' ]; then echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2 exit 0 fi echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2 exit 1" } check_rpi_tailscale_ip() { local expected_ip="${LAB_RPI_TAILSCALE_IP:-}" local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" if [[ -z "${expected_ip}" ]]; then return 0 fi ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'" } check_pimox_storage() { local mode="${LAB_PIMOX_PIPELINE:-true}" local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" if disabled_value "${mode}"; then return 0 fi pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then pvesm_cmd=/usr/sbin/pvesm fi if [ -z \"\$pvesm_cmd\" ]; then echo 'pvesm was not found' >&2 exit 1 fi sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'" } check_local_disk_free_gib() { local path="$1" local min_gib="$2" local available_kib local min_kib available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')" min_kib=$((min_gib * 1024 * 1024)) if ((available_kib < min_kib)); then echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2 return 1 fi } check_systemd_active() { local unit="$1" systemctl is-active --quiet "${unit}" } check_rpi_docker_writable() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" sudo mkdir -p \"\$root/.homelab-check\" sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\" sudo rm -f \"\$root/.homelab-check/write-test\"" } check_edge_disk_free() { local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" local edge_user="${LAB_EDGE_USER:-ubuntu}" local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}" ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\" min_kib=$((min_gib * 1024 * 1024)) if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2 exit 1 fi" } check_pihole_dns_query() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}" local output if command -v dig >/dev/null 2>&1; then output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || { printf '%s\n' "${output}" >&2 echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2 return 1 } if [[ -z "${output}" ]]; then echo "Pi-hole returned no A records for ${query_name}" >&2 return 1 fi return 0 fi if command -v nslookup >/dev/null 2>&1; then nslookup "${query_name}" "${rpi_host}" >/dev/null return $? fi echo "dig or nslookup is required to test Pi-hole DNS directly." >&2 return 1 } homelab_preflight() { local phase="${1:-full}" local failures=0 require_debian_server "preflight" if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}." return 0 fi case "${phase}" in early | full) ;; *) echo "Unsupported preflight phase '${phase}'." >&2 exit 1 ;; esac echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..." preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1)) preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1)) preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1)) if [[ "${phase}" == "full" ]]; then preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1)) preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1)) fi preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:-132.145.170.74}" check_edge_ssh || failures=$((failures + 1)) if ((failures > 0)); then echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2 exit 1 fi echo "${phase^} preflight checks passed." } doctor_preapply() { local failures=0 require_debian_server "doctor-preapply" echo "Running pre-apply readiness checks..." preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1)) preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1)) preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1)) preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1)) preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1)) preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1)) preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1)) if ((failures > 0)); then echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2 exit 1 fi echo "Pre-apply doctor checks passed." } inventory_check() { "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" } run_pimox_pipeline() { local mode="${LAB_PIMOX_PIPELINE:-true}" local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}" local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}" local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-4096}}" local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" local provisioning_interface local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}" local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}" local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}" local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}" local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" local index local readiness_output local readiness_status local worker_cpu_affinity if disabled_value "${mode}"; then return 0 fi if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then mode="true" fi if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 exit 1 fi if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2 exit 1 fi if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2 exit 1 fi if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then echo "Pimox template and worker CPU and memory values must be greater than zero." >&2 exit 1 fi if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2 exit 1 fi if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2 exit 1 fi if [[ "${worker_storage}" == "local" ]]; then echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2 exit 1 fi set +e readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then echo 'qm was not found in PATH and ${qm_bin} is not executable' exit 1 fi if ! ip link show '${bridge}' >/dev/null 2>&1; then echo 'bridge ${bridge} was not found' exit 1 fi if ! sudo -n true >/dev/null 2>&1; then echo 'passwordless sudo is not available for ${pimox_user}' exit 1 fi pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then pvesm_cmd=/usr/sbin/pvesm fi if [ -z \"\$pvesm_cmd\" ]; then echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' exit 1 fi if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then echo 'Pimox worker storage ${worker_storage} was not found or is not active' exit 1 fi" 2>&1)" readiness_status=$? set -e if ((readiness_status != 0)); then if [[ "${mode}" == "auto" ]]; then echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready." return 0 fi echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2 exit 1 fi ensure_python3 provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}" if [[ -z "${provisioning_interface}" ]]; then echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2 exit 1 fi export TF_VAR_provisioning_interface="${provisioning_interface}" export TF_VAR_pimox_host="${pimox_host}" export TF_VAR_pimox_user="${pimox_user}" export TF_VAR_pimox_ssh_key_path="${pimox_key}" export TF_VAR_pimox_qm_bin="${qm_bin}" export TF_VAR_pimox_template_bridge="${bridge}" export TF_VAR_pimox_template_vmid="${template_vmid}" export TF_VAR_pimox_template_name="${template_name}" export TF_VAR_pimox_template_cores="${template_cores}" export TF_VAR_pimox_template_memory="${template_memory}" export TF_VAR_pimox_template_replace_existing="${template_replace_existing}" export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}" export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}" export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}" export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}" export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}" echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..." run_tofu_stack "bootstrap/provisioning" if ((worker_count == 0)); then return 0 fi if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2 exit 1 fi pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1" echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage." mkdir -p "${REPO_ROOT}/.lab" : >"${spec_file}" for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." continue fi worker_cpu_affinity="" if [[ -n "${worker_cpu_affinities}" ]]; then worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" fi ensure_pimox_worker_node \ "${index}" \ "${spec_file}" \ "${pimox_host}" \ "${pimox_user}" \ "${pimox_key}" \ "${template_vmid}" \ "${bridge}" \ "${worker_base_vmid}" \ "${worker_name_prefix}" \ "${worker_node_prefix}" \ "${worker_key_prefix}" \ "${worker_cores}" \ "${worker_memory}" \ "${worker_user}" \ "${worker_key_path}" \ "${ip_prefix}" \ "${timeout_seconds}" \ "${qm_bin}" \ "${worker_storage}" \ "${worker_replace_existing}" \ "${worker_cpu_affinity}" done write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}" export LAB_CLUSTER_VAR_FILE="${var_file}" } run_openwrt_pipeline() { local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}" local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local vmid="${LAB_OPENWRT_VMID:-9100}" local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}" local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}" local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}" local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}" local cores="${LAB_OPENWRT_CORES:-2}" local memory="${LAB_OPENWRT_MEMORY:-512}" local version="${LAB_OPENWRT_VERSION:-24.10.6}" local image_url="${LAB_OPENWRT_IMAGE_URL:-}" local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}" local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}" local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}" local start_vm="${LAB_OPENWRT_START:-false}" local onboot="${LAB_OPENWRT_ONBOOT:-false}" local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}" local root_key_b64="" local lan_dhcp_ignore="1" local start_vm_flag="false" local onboot_flag="0" if disabled_value "${mode}"; then return 0 fi if ! truthy "${mode}"; then echo "LAB_OPENWRT_VM must be true or false." >&2 exit 1 fi if [[ -z "${image_url}" ]]; then image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz" fi if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2 exit 1 fi for value_name in storage wan_bridge lan_bridge vm_name; do local value="${!value_name}" if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2 exit 1 fi done if [[ "${storage}" == "local" ]]; then echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2 exit 1 fi if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2 exit 1 fi if truthy "${lan_dhcp_enabled}"; then lan_dhcp_ignore="0" fi if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then echo "LAB_OPENWRT_START must be true or false." >&2 exit 1 fi if truthy "${start_vm}"; then start_vm_flag="true" fi if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then echo "LAB_OPENWRT_ONBOOT must be true or false." >&2 exit 1 fi if truthy "${onboot}"; then onboot_flag="1" fi if [[ -r "${root_key_path}" ]]; then root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')" fi echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..." pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <&2 exit 1 fi pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)" if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then pvesm_cmd=/usr/sbin/pvesm fi if [ -z "\$pvesm_cmd" ]; then echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2 exit 1 fi if ! sudo -n true >/dev/null 2>&1; then echo "passwordless sudo is required for OpenWrt VM automation" >&2 exit 1 fi if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2 exit 1 fi if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2 exit 1 fi if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then echo "Pimox storage \$storage was not found." >&2 exit 1 fi if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2 exit 1 fi sudo "\$qm_cmd" set "\$vmid" \\ --net0 "virtio,bridge=\$wan_bridge" \\ --net1 "virtio,bridge=\$lan_bridge" \\ --cores "\$cores" \\ --memory "\$memory" \\ --onboot "\$onboot" if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then sudo "\$qm_cmd" start "\$vmid" fi exit 0 fi for required_cmd in curl gzip losetup mount umount awk sed; do if ! command -v "\$required_cmd" >/dev/null 2>&1; then echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2 exit 1 fi done tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)" mnt_dir="\$tmp_dir/root" loopdev="" cleanup() { if mountpoint -q "\$mnt_dir" 2>/dev/null; then sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true fi if [ -n "\$loopdev" ]; then sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true fi rm -rf "\$tmp_dir" } trap cleanup EXIT mkdir -p "\$mnt_dir" curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz" gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img" loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")" root_part="\${loopdev}p2" if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then root_part="\${loopdev}p2" fi if [ ! -b "\$root_part" ]; then echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2 exit 1 fi sudo mount "\$root_part" "\$mnt_dir" sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh" cat >"\$tmp_dir/network" <"\$tmp_dir/dhcp" <"\$tmp_dir/firewall" <<'FIREWALL' config defaults option input 'REJECT' option output 'ACCEPT' option forward 'REJECT' option synflood_protect '1' config zone option name 'lan' list network 'lan' option input 'ACCEPT' option output 'ACCEPT' option forward 'ACCEPT' config zone option name 'wan' list network 'wan' option input 'REJECT' option output 'ACCEPT' option forward 'REJECT' option masq '1' option mtu_fix '1' config forwarding option src 'lan' option dest 'wan' config rule option name 'Allow-DHCP-Renew' option src 'wan' option proto 'udp' option dest_port '68' option target 'ACCEPT' option family 'ipv4' config rule option name 'Allow-Ping' option src 'wan' option proto 'icmp' option icmp_type 'echo-request' option family 'ipv4' option target 'ACCEPT' FIREWALL cat >"\$tmp_dir/system" <"\$tmp_dir/authorized_keys" sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys" sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" fi sync sudo umount "\$mnt_dir" sudo losetup -d "\$loopdev" loopdev="" sudo "\$qm_cmd" create "\$vmid" \\ --name "\$vm_name" \\ --bios ovmf \\ --cores "\$cores" \\ --memory "\$memory" \\ --net0 "virtio,bridge=\$wan_bridge" \\ --net1 "virtio,bridge=\$lan_bridge" \\ --numa 0 \\ --ostype l26 \\ --scsihw virtio-scsi-pci \\ --sockets 1 \\ --vga virtio \\ --onboot "\$onboot" sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0" sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')" if [ -z "\$disk_volume" ]; then echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2 exit 1 fi sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume" sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0" if [ "\$start_vm" = "true" ]; then sudo "\$qm_cmd" start "\$vmid" fi EOF } openwrt() { require_debian_server "openwrt" LAB_OPENWRT_VM=true run_openwrt_pipeline } cleanup_calico_links() { ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true sudo ip link delete vxlan.calico 2>/dev/null || true sudo ip link delete tunl0 2>/dev/null || true sudo ip link delete cni0 2>/dev/null || true sudo ip link delete kube-ipvs0 2>/dev/null || true ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true } cleanup_iptables() { sudo iptables -F || true sudo iptables -X || true sudo iptables -t nat -F || true sudo iptables -t nat -X || true sudo iptables -t mangle -F || true sudo iptables -t mangle -X || true sudo iptables -t raw -F || true sudo iptables -t raw -X || true if command -v ipvsadm >/dev/null 2>&1; then sudo ipvsadm --clear || true fi } cleanup_calico_runtime_files() { local path for path in /run/calico /var/run/calico; do if sudo test -e "${path}"; then sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true sudo rmdir "${path}" 2>/dev/null || true fi done } restore_node_dns() { sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then sudo rm -f /etc/resolv.conf sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf fi sudo systemctl restart systemd-resolved 2>/dev/null || true } cleanup_mounts() { if command -v findmnt >/dev/null 2>&1; then local mount_root while IFS= read -r mountpoint; do sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true done < <( for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true done | sort -ru ) fi while IFS= read -r mountpoint; do sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true } cleanup_node() { sudo kubeadm reset --force || true sudo systemctl stop kubelet 2>/dev/null || true sudo systemctl stop containerd 2>/dev/null || true sudo killall containerd-shim-runc-v2 2>/dev/null || true cleanup_mounts sudo rm -rf \ /etc/kubernetes/ \ /var/lib/etcd/ \ /var/lib/kubelet/ \ /var/lib/cni/ \ /etc/cni/net.d \ /run/flannel \ /var/lib/calico \ /var/log/calico \ /var/lib/containerd/* \ /run/containerd/* \ /etc/containerd/certs.d \ /etc/containerd/config.toml cleanup_calico_runtime_files sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam cleanup_iptables cleanup_calico_links restore_node_dns sudo mkdir -p /etc/containerd/certs.d sudo systemctl reset-failed kubelet containerd 2>/dev/null || true sudo systemctl start containerd 2>/dev/null || true } image_ref_tag() { local image_ref="$1" local tag tag="${image_ref##*:}" if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then echo "Image reference ${image_ref} must include an immutable tag." >&2 exit 1 fi printf '%s\n' "${tag}" } website_image_tag() { local source_hash="$1" printf 'src-%s\n' "${source_hash:0:12}" } apps_registry_endpoint() { if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then printf '%s\n' "${TF_VAR_registry_endpoint}" return 0 fi demos_registry_endpoint } demos_registry_endpoint() { local image image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")" if [[ -z "${image}" || "${image}" != */* ]]; then echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2 exit 1 fi printf '%s\n' "${image%%/*}" } website_source_hash() { ( cd "${REPO_ROOT}" find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' ) } demos_source_hash() { ( cd "${REPO_ROOT}" find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' ) } registry_image_exists() { local registry_endpoint="$1" local repository="$2" local tag="$3" local accept_header if ! command -v curl >/dev/null 2>&1; then return 1 fi accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" curl -fsS \ -H "Accept: ${accept_header}" \ "http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null } image_state_value() { local state_file="$1" local key="$2" awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true } website_image_is_current() { local state_file="$1" local source_hash="$2" local platforms="$3" local image_ref="$4" local registry_endpoint="$5" local image_tag local saved_hash local saved_platforms local saved_image [[ -f "${state_file}" ]] || return 1 saved_hash="$(image_state_value "${state_file}" source_hash)" saved_platforms="$(image_state_value "${state_file}" platforms)" saved_image="$(image_state_value "${state_file}" image)" [[ "${saved_hash}" == "${source_hash}" ]] || return 1 [[ "${saved_platforms}" == "${platforms}" ]] || return 1 [[ "${saved_image}" == "${image_ref}" ]] || return 1 image_tag="$(image_ref_tag "${image_ref}")" registry_image_exists "${registry_endpoint}" php-website "${image_tag}" } ensure_website_image_tag_not_reused() { local state_file="$1" local source_hash="$2" local image_ref="$3" local saved_hash local saved_image [[ -f "${state_file}" ]] || return 0 saved_hash="$(image_state_value "${state_file}" source_hash)" saved_image="$(image_state_value "${state_file}" image)" if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2 echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2 exit 1 fi } demos_image_is_current() { local state_file="$1" local source_hash="$2" local platforms="$3" local image_ref="$4" local registry_endpoint="$5" local saved_hash local saved_platforms local saved_image [[ -f "${state_file}" ]] || return 1 saved_hash="$(image_state_value "${state_file}" source_hash)" saved_platforms="$(image_state_value "${state_file}" platforms)" saved_image="$(image_state_value "${state_file}" image)" [[ "${saved_hash}" == "${source_hash}" ]] || return 1 [[ "${saved_platforms}" == "${platforms}" ]] || return 1 [[ "${saved_image}" == "${image_ref}" ]] || return 1 registry_image_exists "${registry_endpoint}" demos-static latest } write_website_image_state() { local state_file="$1" local source_hash="$2" local platforms="$3" local image_ref="$4" mkdir -p "$(dirname "${state_file}")" { printf 'source_hash=%s\n' "${source_hash}" printf 'platforms=%s\n' "${platforms}" printf 'image=%s\n' "${image_ref}" } > "${state_file}" } write_demos_image_state() { local state_file="$1" local source_hash="$2" local platforms="$3" local image_ref="$4" mkdir -p "$(dirname "${state_file}")" { printf 'source_hash=%s\n' "${source_hash}" printf 'platforms=%s\n' "${platforms}" printf 'image=%s\n' "${image_ref}" } > "${state_file}" } path_available_mb() { local path="$1" while [[ ! -e "${path}" && "${path}" != "/" ]]; do path="$(dirname "${path}")" done df -Pm "${path}" | awk 'NR == 2 {print $4}' } docker_root_dir() { docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n' } prune_unused_docker_build_data() { docker buildx rm lab-builder 2>/dev/null || true docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true docker builder prune -af 2>/dev/null || true docker system prune -af 2>/dev/null || true } ensure_docker_build_space() { local docker_root local free_mb local min_free_mb min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}" docker_root="$(docker_root_dir)" free_mb="$(path_available_mb "${docker_root}")" if (( free_mb >= min_free_mb )); then return 0 fi echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..." prune_unused_docker_build_data free_mb="$(path_available_mb "${docker_root}")" if (( free_mb < min_free_mb )); then echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2 echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2 echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2 exit 1 fi } prepare_buildx_builder() { local registry_endpoint="$1" docker run --rm --privileged multiarch/qemu-user-static --reset -p yes cat < "${BUILDX_CONFIG}" [registry."${registry_endpoint}"] http = true insecure = true [registry."127.0.0.1:30500"] http = true insecure = true [registry."localhost:30500"] http = true insecure = true EOF docker buildx rm lab-builder 2>/dev/null || true docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use docker buildx inspect --bootstrap } ensure_cosign_available() { local arch local cosign_url local target_dir local target_path local tmp_path if [[ -n "${COSIGN_BIN}" ]]; then if [[ -x "${COSIGN_BIN}" ]]; then return 0 fi echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2 exit 1 fi if command -v cosign >/dev/null 2>&1; then COSIGN_BIN="$(command -v cosign)" return 0 fi case "$(uname -m)" in x86_64 | amd64) arch="amd64" ;; aarch64 | arm64) arch="arm64" ;; *) echo "Unsupported Cosign install architecture: $(uname -m)" >&2 exit 1 ;; esac target_dir="${HOMELAB_STATE_DIR}/bin" target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}" if [[ -x "${target_path}" ]]; then COSIGN_BIN="${target_path}" return 0 fi if ! command -v curl >/dev/null 2>&1; then echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2 exit 1 fi mkdir -p "${target_dir}" tmp_path="${target_path}.tmp" cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..." curl -fsSL "${cosign_url}" -o "${tmp_path}" chmod 0755 "${tmp_path}" mv "${tmp_path}" "${target_path}" COSIGN_BIN="${target_path}" } ensure_cosign_password() { if [[ -n "${COSIGN_PASSWORD:-}" ]]; then return 0 fi if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" export COSIGN_PASSWORD return 0 fi mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")" if command -v openssl >/dev/null 2>&1; then openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}" elif command -v python3 >/dev/null 2>&1; then python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}" import secrets print(secrets.token_hex(32)) PY else echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2 exit 1 fi chmod 0600 "${COSIGN_PASSWORD_FILE}" COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" export COSIGN_PASSWORD } run_cosign() { ensure_cosign_available ensure_cosign_password COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@" } ensure_cosign_keypair() { ensure_cosign_available ensure_cosign_password if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then return 0 fi if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2 exit 1 fi mkdir -p "$(dirname "${COSIGN_KEY_PATH}")" echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..." COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}" chmod 0600 "${COSIGN_KEY_PATH}" chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}" } publish_cosign_public_key_config_map() { kubectl --kubeconfig "${KUBECONFIG}" create namespace kyverno --dry-run=client -o yaml | kubectl --kubeconfig "${KUBECONFIG}" apply -f - kubectl --kubeconfig "${KUBECONFIG}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \ --from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \ --dry-run=client -o yaml | kubectl --kubeconfig "${KUBECONFIG}" apply -f - } cosign_registry_flags() { local registry_endpoint="$1" local image_ref="$2" if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then printf '%s\n' "--allow-http-registry" fi } image_supply_chain_metadata_exists() { local image_ref="$1" local registry_endpoint="$2" local digest_ref local -a registry_flags=() if ! digest_ref="$(image_digest_ref "${image_ref}")"; then return 1 fi mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") run_cosign verify \ --key "${COSIGN_PUBLIC_KEY_PATH}" \ --insecure-ignore-tlog=true \ "${registry_flags[@]}" \ "${digest_ref}" >/dev/null 2>&1 && run_cosign verify-attestation \ --key "${COSIGN_PUBLIC_KEY_PATH}" \ --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ --insecure-ignore-tlog=true \ "${registry_flags[@]}" \ "${digest_ref}" >/dev/null 2>&1 } write_image_sbom_predicate() { local image_ref="$1" local output_file="$2" local source_hash="$3" docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}" python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY' from datetime import datetime, timezone import json import re import sys path = sys.argv[1] image_ref = sys.argv[2] source_hash = sys.argv[3] try: with open(path, encoding="utf-8") as handle: document = json.load(handle) except json.JSONDecodeError: document = None if not isinstance(document, dict): safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-") document = { "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", "name": f"homelab image SBOM for {image_ref}", "documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}", "creationInfo": { "created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "creators": ["Tool: jeannie"], }, "packages": [ { "name": image_ref, "SPDXID": "SPDXRef-Image", "downloadLocation": "NOASSERTION", "filesAnalyzed": False, "licenseConcluded": "NOASSERTION", "licenseDeclared": "NOASSERTION", "copyrightText": "NOASSERTION", } ], "relationships": [ { "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-Image", } ], } with open(path, "w", encoding="utf-8") as handle: json.dump(document, handle, sort_keys=True) handle.write("\n") if not isinstance(document, dict): raise SystemExit("image SBOM predicate is not a JSON object") if document.get("SPDXID") != "SPDXRef-DOCUMENT": raise SystemExit("image SBOM predicate is not an SPDX document") if not document.get("packages"): raise SystemExit("image SBOM predicate does not include packages") PY } image_digest_ref() { local image_ref="$1" local digest local repository if [[ "${image_ref}" == *@sha256:* ]]; then printf '%s\n' "${image_ref}" return 0 fi digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')" if [[ -z "${digest}" || "${digest}" == "" ]]; then echo "Unable to resolve image digest for ${image_ref}." >&2 return 1 fi if [[ "${image_ref##*/}" == *:* ]]; then repository="${image_ref%:*}" else repository="${image_ref}" fi printf '%s@%s\n' "${repository}" "${digest}" } publish_image_supply_chain_metadata() { local image_ref="$1" local registry_endpoint="$2" local source_hash="$3" local digest_ref local sbom_file local -a registry_flags=() if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM." return 0 fi digest_ref="$(image_digest_ref "${image_ref}")" mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") sbom_file="$(mktemp)" write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}" echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..." run_cosign sign \ --yes \ --key "${COSIGN_KEY_PATH}" \ --tlog-upload=false \ -a "homelab.dev/source-hash=${source_hash}" \ "${registry_flags[@]}" \ "${digest_ref}" run_cosign attest \ --yes \ --key "${COSIGN_KEY_PATH}" \ --predicate "${sbom_file}" \ --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ --tlog-upload=false \ "${registry_flags[@]}" \ "${digest_ref}" rm -f "${sbom_file}" if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then echo "Cosign verification failed after signing ${digest_ref}." >&2 exit 1 fi } dump_argocd_debug() { local app="$1" kubectl --kubeconfig "${KUBECONFIG}" -n argocd get application "${app}" -o yaml || true kubectl --kubeconfig "${KUBECONFIG}" -n argocd describe application "${app}" || true kubectl --kubeconfig "${KUBECONFIG}" -n argocd get pods -o wide || true kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs deployment/argocd-repo-server --tail=120 || true kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true } dump_namespace_debug() { local namespace="$1" kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get all -o wide || true kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pvc -o wide || true kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe pods || true kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true } wait_for_namespace() { local namespace="$1" local app="$2" local timeout_seconds="$3" local elapsed=0 until kubectl --kubeconfig "${KUBECONFIG}" get namespace "${namespace}" >/dev/null 2>&1; do if ((elapsed >= timeout_seconds)); then echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2 dump_argocd_debug "${app}" exit 1 fi sleep 5 elapsed=$((elapsed + 5)) done } wait_for_namespaced_resource() { local namespace="$1" local kind="$2" local name="$3" local app="$4" local timeout_seconds="$5" local elapsed=0 until kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do if ((elapsed >= timeout_seconds)); then echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2 dump_argocd_debug "${app}" kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true exit 1 fi sleep 5 elapsed=$((elapsed + 5)) done } wait_for_cluster_resource() { local kind="$1" local name="$2" local app="$3" local timeout_seconds="$4" local elapsed=0 until kubectl --kubeconfig "${KUBECONFIG}" get "${kind}/${name}" >/dev/null 2>&1; do if ((elapsed >= timeout_seconds)); then echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2 dump_argocd_debug "${app}" exit 1 fi sleep 5 elapsed=$((elapsed + 5)) done } wait_for_deployment_ready() { local namespace="$1" local deployment="$2" local app="$3" local timeout_seconds="$4" local desired_replicas local ready_replicas local elapsed=0 desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" desired_replicas="${desired_replicas:-1}" until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \ (( ${ready_replicas:-0} >= desired_replicas )); do if ((elapsed >= timeout_seconds)); then echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2 dump_argocd_debug "${app}" dump_namespace_debug "${namespace}" exit 1 fi sleep 5 elapsed=$((elapsed + 5)) done } deploy_gitea() { local mode="${LAB_GITEA_DEPLOY:-true}" local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}" local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}" local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}" local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}" local http_port="${LAB_GITEA_HTTP_PORT:-3000}" local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:-lab2025.duckdns.org}}" local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml" local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}" require_debian_server "deploy-gitea" if disabled_value "${mode}"; then install_gitea_backup_timer return 0 fi if [[ ! -s "${compose_file}" ]]; then echo "Missing ${compose_file}" >&2 exit 1 fi echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..." ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea" scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml" ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu install_dir='${install_dir}' install_docker='${install_docker}' install_missing_packages() { missing_packages='' for package in \"\$@\"; do if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then missing_packages=\"\$missing_packages \$package\" fi done if [ -n \"\$missing_packages\" ]; then sudo apt-get update sudo apt-get install -y --no-install-recommends \$missing_packages fi } install_missing_packages ca-certificates curl iptables if ! command -v docker >/dev/null 2>&1; then if [ \"\$install_docker\" = 'true' ]; then curl -fsSL https://get.docker.com | sudo sh else echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2 exit 1 fi fi if ! sudo docker compose version >/dev/null 2>&1; then install_missing_packages docker-compose-plugin fi repair_docker_iptables() { if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then return 0 fi echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...' sudo systemctl restart docker sleep 3 if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then return 0 fi echo 'Docker NAT chain is still missing after restarting Docker.' >&2 sudo iptables -t nat -S >&2 || true sudo systemctl status docker --no-pager -l >&2 || true exit 1 } repair_docker_iptables sudo mkdir -p \"\$install_dir/data\" sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\" sudo chown -R 1000:1000 \"\$install_dir/data\" sudo tee \"\$install_dir/.env\" >/dev/null <&2 exit 1 fi for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do value="${!value_name}" if [[ "${value}" == *"'"* ]]; then echo "${value_name} cannot contain a single quote." >&2 exit 1 fi done echo "Deploying RPi services on ${rpi_user}@${rpi_host}..." ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services" scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ "${source_dir}/docker-compose.yml" \ "${source_dir}/adlists.txt" \ "${source_dir}/local-dns-records.txt" \ "${source_dir}/cname-records.txt" \ "${source_dir}/static-dhcp-hosts.txt" \ "${source_dir}/uptime-kuma-monitors.json" \ "${source_dir}/seed-uptime-kuma.js" \ "${source_dir}/bootstrap.sh" \ "${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \ LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \ LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \ LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \ LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \ LAB_RPI_INSTALL_DOCKER='${install_docker}' \ PIHOLE_WEBPASSWORD='${pihole_webpassword}' \ bash /tmp/homelab-rpi-services/bootstrap.sh" } gitea_bootstrap_password() { if command -v openssl >/dev/null 2>&1; then openssl rand -hex 32 return 0 fi python3 - <<'PY' import secrets print(secrets.token_hex(32)) PY } gitea_api_base_url() { local gitea_host="$1" local http_port="$2" local candidate local api_base_override="${LAB_GITEA_API_BASE_URL:-}" if [[ -n "${api_base_override}" ]]; then printf '%s\n' "${api_base_override%/}" return 0 fi for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do if curl -fsS "${candidate}/version" >/dev/null 2>&1; then printf '%s\n' "${candidate}" return 0 fi done echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2 exit 1 } gitea_repo_exists() { local api_base="$1" local auth_user="$2" local auth_password="$3" local owner="$4" local repo_name="$5" local status status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")" case "${status}" in 200) return 0 ;; 404) return 1 ;; 401 | 403) echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2 exit 1 ;; *) echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2 exit 1 ;; esac } gitea_branch_exists() { local api_base="$1" local auth_user="$2" local auth_password="$3" local owner="$4" local repo_name="$5" local branch="$6" local status status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")" case "${status}" in 200) return 0 ;; 404) return 1 ;; 401 | 403) echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2 exit 1 ;; *) echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2 exit 1 ;; esac } create_gitea_repo() { local api_base="$1" local auth_user="$2" local auth_password="$3" local repo_name="$4" local default_branch="$5" local payload payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY' import json import sys repo_name, default_branch = sys.argv[1:3] print(json.dumps({ "name": repo_name, "private": False, "auto_init": False, "default_branch": default_branch, "description": "Homelab infrastructure configuration", })) PY )" curl -fsS \ -u "${auth_user}:${auth_password}" \ -H "Content-Type: application/json" \ -X POST \ -d "${payload}" \ "${api_base}/user/repos" >/dev/null } gitea_public_key_registered() { local api_base="$1" local auth_user="$2" local auth_password="$3" local owner="$4" local repo_name="$5" local public_key_path="$6" local repo_keys local user_keys user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")" repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")" GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ GITEA_USER_KEYS="${user_keys}" \ GITEA_REPO_KEYS="${repo_keys}" \ python3 - <<'PY' import json import os import sys public_key = os.environ["GITEA_PUBLIC_KEY"].strip() for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"): for key in json.loads(os.environ[env_name]) or []: if key.get("key", "").strip() == public_key: sys.exit(0) sys.exit(1) PY } create_gitea_repo_deploy_key() { local api_base="$1" local auth_user="$2" local auth_password="$3" local owner="$4" local repo_name="$5" local title="$6" local public_key_path="$7" local read_only="$8" local payload payload="$( GITEA_DEPLOY_KEY_TITLE="${title}" \ GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \ python3 - <<'PY' import json import os print(json.dumps({ "title": os.environ["GITEA_DEPLOY_KEY_TITLE"], "key": os.environ["GITEA_PUBLIC_KEY"].strip(), "read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true", })) PY )" curl -fsS \ -u "${auth_user}:${auth_password}" \ -H "Content-Type: application/json" \ -X POST \ -d "${payload}" \ "${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null } ensure_gitea_repo_ssh_access() { local api_base="$1" local auth_user="$2" local auth_password="$3" local owner="$4" local repo_name="$5" local ssh_host="$6" local ssh_port="$7" local key_path="$8" local key_title="$9" local key_read_only="${10}" local key_dir local known_hosts local public_key_path local read_only_json="false" local ssh_repo_url if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2 exit 1 fi key_dir="$(dirname "${key_path}")" public_key_path="${key_path}.pub" mkdir -p "${key_dir}" chmod 0700 "${key_dir}" if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then ssh-keygen -y -f "${key_path}" >"${public_key_path}" elif [[ ! -s "${key_path}" ]]; then echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2 exit 1 fi chmod 0600 "${key_path}" chmod 0644 "${public_key_path}" if truthy "${key_read_only}"; then read_only_json="true" fi if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then echo "Gitea already has Debian host SSH key ${public_key_path}." else create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}" echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}." fi known_hosts="${HOME}/.ssh/known_hosts" touch "${known_hosts}" chmod 0644 "${known_hosts}" if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null fi ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git" git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null || git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}" git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null echo "Gitea SSH remote: ${ssh_repo_url}" } bootstrap_gitea_repo() { local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}" local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" local gitea_user="${LAB_GITEA_USER:-jv}" local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" local http_port="${LAB_GITEA_HTTP_PORT:-3000}" local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" local root_url="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}" local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}" local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}" local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}" local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}" local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}" local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}" local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}" local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}" local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}" local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}" local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}" local api_base local public_repo_url local direct_repo_url local push_url local askpass local credentials_dir local remote_status local worktree_status require_debian_server "bootstrap-gitea-repo" if disabled_value "${mode}"; then return 0 fi ensure_python3 for value_name in repo_owner repo_name default_branch bootstrap_user; do local value="${!value_name}" if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then echo "${value_name} contains unsupported characters." >&2 exit 1 fi done if [[ "${bootstrap_email}" == *"'"* ]]; then echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2 exit 1 fi if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then echo "LAB_GITEA_SSH_PORT must be numeric." >&2 exit 1 fi if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then # shellcheck disable=SC1090 source "${credentials_file}" bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}" bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}" bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}" fi if [[ -z "${bootstrap_password}" ]]; then bootstrap_password="$(gitea_bootstrap_password)" credentials_dir="$(dirname "${credentials_file}")" mkdir -p "${credentials_dir}" chmod 0700 "${credentials_dir}" { printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}" printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}" printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}" } > "${credentials_file}" chmod 0600 "${credentials_file}" echo "Generated Gitea bootstrap credentials at ${credentials_file}." fi for value_name in repo_owner repo_name default_branch bootstrap_user; do local value="${!value_name}" if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then echo "${value_name} contains unsupported characters." >&2 exit 1 fi done for value_name in bootstrap_email bootstrap_password; do local value="${!value_name}" if [[ "${value}" == *"'"* ]]; then echo "${value_name} cannot contain a single quote." >&2 exit 1 fi done echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..." # shellcheck disable=SC2087 ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" </dev/null 2>&1; then echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2 exit 1 fi for attempt in \$(seq 1 60); do if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 || curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then break fi if [ "\${attempt}" = "60" ]; then echo "Timed out waiting for Gitea API inside \${container_name}." >&2 exit 1 fi sleep 2 done if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \ --username "\${bootstrap_user}" \ --password "\${bootstrap_password}" \ --email "\${bootstrap_email}" \ --admin \ --must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \ --username "\${bootstrap_user}" \ --password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || { cat /tmp/homelab-gitea-user-password.log >&2 exit 1 } else cat /tmp/homelab-gitea-user-create.log >&2 exit 1 fi fi EOF api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")" if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then echo "Gitea repository ${repo_owner}/${repo_name} already exists." else if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2 exit 1 fi create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}" echo "Created Gitea repository ${repo_owner}/${repo_name}." fi public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git" if [[ "${api_base}" == */git/api/v1 ]]; then direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git" else direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git" fi push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}" git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null || git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}" if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged." else worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)" if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2 echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2 exit 1 fi askpass="$(mktemp)" trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT cat > "${askpass}" </dev/null </dev/null 2>&1 || true" >/dev/null 2>&1 || true } trap cleanup EXIT ssh_gitea "set -eu sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories' sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}' sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}' sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}' sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ "\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}" sudo mkdir -p "\${GITEA_BACKUP_DIR}" sudo chown jv:jv "\${GITEA_BACKUP_DIR}" sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}" sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete echo "Created \${backup_archive}" BACKUP_SCRIPT_EOT sudo chmod 0755 "${backup_script}" sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT' [Unit] Description=Back up external Homelab Gitea to Debian host storage After=network-online.target Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/homelab-gitea-backup.sh SERVICE_EOT sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT' [Unit] Description=Run daily Homelab Gitea backups [Timer] OnCalendar=*-*-* 02:35:00 RandomizedDelaySec=20m Persistent=true [Install] WantedBy=timers.target TIMER_EOT sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT' #!/usr/bin/env bash set -euo pipefail GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}" GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}" if ! command -v python3 >/dev/null 2>&1; then echo "python3 is required for Gitea restore drills." >&2 exit 1 fi latest_archive="$( { find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } | sort -nr | awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }' )" if [[ -z "${latest_archive}" ]]; then echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}." exit 0 fi timestamp="$(date -u +%Y%m%dT%H%M%SZ)" tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")" tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")" report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt" cleanup() { rm -rf "${tmp_dir}" rm -f "${tmp_report}" } trap cleanup EXIT python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY' import os import sys import zipfile archive_path, extract_dir, report_path = sys.argv[1:4] with zipfile.ZipFile(archive_path) as archive: bad_member = archive.testzip() if bad_member: raise SystemExit(f"ZIP integrity check failed at {bad_member}") members = archive.infolist() if not members: raise SystemExit("ZIP archive is empty") extract_root = os.path.abspath(extract_dir) for member in members: target = os.path.abspath(os.path.join(extract_root, member.filename)) if target != extract_root and not target.startswith(extract_root + os.sep): raise SystemExit(f"Unsafe archive path: {member.filename}") archive.extractall(extract_root) file_count = 0 total_bytes = 0 for root, _, files in os.walk(extract_dir): for name in files: file_count += 1 total_bytes += os.path.getsize(os.path.join(root, name)) if file_count == 0: raise SystemExit("Archive extracted no files") with open(report_path, "w", encoding="utf-8") as handle: handle.write("Gitea restore drill report\n") handle.write(f"archive={archive_path}\n") handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n") handle.write(f"extracted_files={file_count}\n") handle.write(f"extracted_bytes={total_bytes}\n") handle.write("result=ok\n") PY sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}" sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}" sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete echo "Created ${report_path}" RESTORE_DRILL_SCRIPT_EOT sudo chmod 0755 "${restore_drill_script}" sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT' [Unit] Description=Run a non-destructive Gitea backup restore drill After=network-online.target homelab-gitea-backup.service Wants=network-online.target [Service] Type=oneshot ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh RESTORE_DRILL_SERVICE_EOT sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT' [Unit] Description=Run monthly Homelab Gitea restore drills [Timer] OnCalendar=monthly RandomizedDelaySec=2h Persistent=true [Install] WantedBy=timers.target RESTORE_DRILL_TIMER_EOT sudo systemctl daemon-reload sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null } backup_gitea() { require_debian_server "backup-gitea" install_gitea_backup_timer sudo /usr/local/sbin/homelab-gitea-backup.sh } drill_gitea_restore() { require_debian_server "drill-gitea-restore" install_gitea_backup_timer sudo /usr/local/sbin/homelab-gitea-restore-drill.sh } install_gitea_runner() { local runner_arch local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}" local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}" local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}" local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}" local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}" local runner_user="${GITEA_RUNNER_USER:-jv}" local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}" local missing_packages=() require_debian_server "install-gitea-runner" case "$(dpkg --print-architecture)" in amd64) runner_arch="linux-amd64" ;; arm64) runner_arch="linux-arm64" ;; *) echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2 exit 1 ;; esac for package in ca-certificates curl git nodejs python3; do if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then missing_packages+=("$package") fi done if [[ ${#missing_packages[@]} -gt 0 ]]; then sudo apt-get update sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" fi sudo curl -fsSL \ -o /usr/local/bin/act_runner \ "https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}" sudo chmod 0755 /usr/local/bin/act_runner sudo chown root:root /usr/local/bin/act_runner sudo -u "${runner_user}" mkdir -p "${runner_home}" if [[ ! -f "${runner_home}/.runner" ]]; then if [[ -z "${runner_token}" ]]; then echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2 exit 1 fi sudo -u "${runner_user}" env \ HOME="/home/${runner_user}" \ GITEA_RUNNER_HOME="${runner_home}" \ GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \ GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \ GITEA_RUNNER_NAME="${runner_name}" \ GITEA_RUNNER_LABELS="${runner_labels}" \ bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"' else echo "Existing runner registration found at ${runner_home}/.runner; keeping it." fi sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null </dev/null sudo systemctl status homelab-gitea-runner.service --no-pager -l } recreate_pods_for_selector() { local namespace="$1" local selector="$2" local app="$3" if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2 dump_argocd_debug "${app}" dump_namespace_debug "${namespace}" exit 1 fi } refresh_argocd_application() { local app="$1" kubectl --kubeconfig "${KUBECONFIG}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null } website_translation_model() { local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}" local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}" require_debian_server "website-translation-model" if ! command -v ollama >/dev/null 2>&1; then echo "ollama is not installed or not in PATH on this Debian server." >&2 exit 1 fi if [[ ! -r "${modelfile}" ]]; then echo "Ollama Modelfile is not readable: ${modelfile}" >&2 exit 1 fi echo "Creating or updating Ollama model ${model_name} from ${modelfile}..." ollama create "${model_name}" -f "${modelfile}" if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then echo "Ollama model ${model_name} was not found after creation." >&2 exit 1 fi echo "Ollama model ${model_name} is ready." } website_ollama_listen() { local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}" local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}" local dropin_dir="/etc/systemd/system/ollama.service.d" local dropin_file="${dropin_dir}/homelab.conf" local waited=0 require_debian_server "website-ollama-listen" bind_address="${bind_address#http://}" bind_address="${bind_address#https://}" if ! systemctl cat ollama.service >/dev/null 2>&1; then echo "ollama.service was not found on this Debian server." >&2 exit 1 fi echo "Configuring ollama.service to listen on ${bind_address}..." sudo mkdir -p "${dropin_dir}" sudo mkdir -p "${models_dir}" if id ollama >/dev/null 2>&1; then sudo chown ollama:ollama "${models_dir}" fi printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" | sudo tee "${dropin_file}" >/dev/null sudo systemctl daemon-reload sudo systemctl restart ollama.service sudo systemctl is-active --quiet ollama.service echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..." until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do waited=$((waited + 2)) if ((waited >= 60)); then echo "Ollama service did not expose the local API after ${waited}s." >&2 echo "Recent service logs:" >&2 sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true exit 1 fi sleep 2 done echo "Ollama API is reachable on the Debian host." } ollama_linux_arch() { case "$(uname -m)" in x86_64 | amd64) printf 'amd64\n' ;; aarch64 | arm64) printf 'arm64\n' ;; *) echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2 return 1 ;; esac } ollama_write_systemd_service() { sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF' [Unit] Description=Ollama Service After=network-online.target [Service] ExecStart=/usr/bin/ollama serve User=ollama Group=ollama Restart=always RestartSec=3 Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" [Install] WantedBy=default.target EOF } ollama_install_from_tgz() { local tarball="$1" if [[ ! -r "${tarball}" ]]; then echo "Ollama tarball is not readable: ${tarball}" >&2 exit 1 fi echo "Installing Ollama from local tarball ${tarball}..." sudo tar -C /usr -xzf "${tarball}" if ! id ollama >/dev/null 2>&1; then sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama fi if getent group render >/dev/null 2>&1; then sudo usermod -aG render ollama fi if getent group video >/dev/null 2>&1; then sudo usermod -aG video ollama fi ollama_write_systemd_service } ollama_setup() { local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}" local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}" local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}" local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}" local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}" local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}" local arch local tmp_installer local curl_ip_args=() require_debian_server "ollama-setup" if ! command -v curl >/dev/null 2>&1; then echo "curl is required to install Ollama." >&2 exit 1 fi if [[ -n "${curl_ip_version}" ]]; then curl_ip_args+=("${curl_ip_version}") fi if ! command -v ollama >/dev/null 2>&1; then if [[ -n "${local_tarball}" ]]; then ollama_install_from_tgz "${local_tarball}" else echo "Installing Ollama from ${installer_url}..." tmp_installer="$(mktemp)" if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then rm -f "${tmp_installer}" arch="$(ollama_linux_arch || true)" cat >&2 < 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then echo "Ollama model ${model} is already present." else OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}" fi echo "Ollama homelab setup is ready." } apps() { local buildx_builder_ready=false local demos_image_built=false local demos_image_ref local demos_image_state_file local demos_platforms local demos_registry_endpoint local demos_source_hash local registry_endpoint local website_image_built=false local website_image_ref local website_image_state_file local website_platforms local website_source_hash require_debian_server "apps" registry_endpoint="$(apps_registry_endpoint)" demos_registry_endpoint="$(demos_registry_endpoint)" demos_image_ref="${registry_endpoint}/demos-static:latest" demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state" demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" demos_source_hash="$(demos_source_hash)" website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state" website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" website_source_hash="$(website_source_hash)" website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}" export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}" export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}" export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" export KUBECONFIG="${TF_VAR_kubeconfig_path}" ensure_cosign_keypair publish_cosign_public_key_config_map if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2 exit 1 fi if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2 exit 1 fi if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2 exit 1 fi if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2 exit 1 fi ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}" echo "Deploying homelab applications..." run_tofu_stack "bootstrap/apps" refresh_argocd_application container-registry refresh_argocd_application supply-chain-policy wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300 wait_for_namespace container-registry container-registry 300 wait_for_namespaced_resource container-registry deployment local-registry container-registry 300 wait_for_deployment_ready container-registry local-registry container-registry 300 if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" && image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build." else echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..." ensure_docker_build_space if [[ "${buildx_builder_ready}" != "true" ]]; then prepare_buildx_builder "${registry_endpoint}" buildx_builder_ready=true fi docker buildx build \ --network host \ --platform "${website_platforms}" \ --provenance=false \ --sbom=true \ --label "dev.homelab.website.source-hash=${website_source_hash}" \ -t "${website_image_ref}" \ -f "${REPO_ROOT}/apps/website/Dockerfile" \ "${REPO_ROOT}/apps/website/" \ --push website_image_built=true fi publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}" if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" && image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build." else echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..." ensure_docker_build_space if [[ "${buildx_builder_ready}" != "true" ]]; then prepare_buildx_builder "${registry_endpoint}" buildx_builder_ready=true fi docker buildx build \ --network host \ --platform "${demos_platforms}" \ --provenance=false \ --sbom=true \ --label "dev.homelab.demos.source-hash=${demos_source_hash}" \ -t "${demos_image_ref}" \ -f "${REPO_ROOT}/apps/demos-static/Dockerfile" \ "${REPO_ROOT}/apps/demos-static/" \ --push demos_image_built=true fi publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}" refresh_argocd_application website-production wait_for_namespace website-production website-production 300 wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300 if [[ "${website_image_built}" == "true" ]]; then recreate_pods_for_selector website-production app=php-website website-production else echo "Skipping website pod restart because the image did not change." fi wait_for_deployment_ready website-production php-website-deployment website-production 300 if [[ "${website_image_built}" == "true" ]]; then write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" fi refresh_argocd_application demos-static wait_for_namespace demos-static demos-static 300 wait_for_namespaced_resource demos-static deployment demos-static demos-static 300 if [[ "${demos_image_built}" == "true" ]]; then recreate_pods_for_selector demos-static app=demos-static demos-static else echo "Skipping demos pod restart because the image did not change." fi wait_for_deployment_ready demos-static demos-static demos-static 300 if [[ "${demos_image_built}" == "true" ]]; then write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" fi refresh_argocd_application heimdall wait_for_namespace heimdall heimdall 300 wait_for_namespaced_resource heimdall deployment heimdall heimdall 300 wait_for_deployment_ready heimdall heimdall heimdall 300 refresh_argocd_application n8n wait_for_namespace n8n n8n 300 wait_for_namespaced_resource n8n deployment n8n n8n 300 wait_for_deployment_ready n8n n8n n8n 300 echo "Application deployment successfully completed." } ensure_cluster_worker_var_file() { if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then prepare_cluster_worker_var_file true fi } up() { require_debian_server "up" echo "Deploying the homelab infrastructure..." jeannie_log_start "up" jeannie_step_plan 15 run_step "Early preflight" homelab_preflight early run_step "Gitea deploy" deploy_gitea run_step "Gitea repo bootstrap" bootstrap_gitea_repo run_step "RPi services" deploy_rpi_services run_step "Full preflight" homelab_preflight full run_step "Pre-apply doctor" doctor_preapply run_step "Pimox provisioning and workers" run_pimox_pipeline run_step "OpenWrt VM" run_openwrt_pipeline run_step "Worker var file" ensure_cluster_worker_var_file run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" run_step "Version report" doctor_versions_report run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" run_step "Applications" apps run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" echo "Deployment successfully completed." echo "Log: ${JEANNIE_LOG_FILE}" } tofu_stack_from_plan_target() { case "$1" in provisioning) printf 'bootstrap/provisioning\n' ;; cluster) printf 'bootstrap/cluster\n' ;; platform) printf 'bootstrap/platform\n' ;; apps) printf 'bootstrap/apps\n' ;; edge) printf 'bootstrap/edge\n' ;; *) echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2 return 1 ;; esac } plan_homelab() { local target="${1:-all}" local stack local -a stacks=() require_debian_server "plan" if [[ "${target}" == "all" ]]; then if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then prepare_cluster_worker_var_file true fi stacks=( "bootstrap/provisioning" "bootstrap/cluster" "bootstrap/platform" "bootstrap/apps" "bootstrap/edge" ) else stack="$(tofu_stack_from_plan_target "${target}")" if [[ "${stack}" == "bootstrap/cluster" && -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then prepare_cluster_worker_var_file true fi stacks=("${stack}") fi for stack in "${stacks[@]}"; do echo echo "Planning ${stack}..." run_tofu_plan_stack "${stack}" done } rebuild_cluster() { require_debian_server "rebuild-cluster" export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}" export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-true}" export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}" export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}" export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}" export TF_VAR_force_worker_rejoin="${TF_VAR_force_worker_rejoin:-true}" echo "Rebuilding the Kubernetes cluster without touching external Gitea..." jeannie_log_start "rebuild-cluster" jeannie_step_plan 10 run_step "Preflight" homelab_preflight run_step "Nuke existing cluster state" nuke_for_rebuild run_step "Pimox provisioning and workers" run_pimox_pipeline run_step "OpenWrt VM" run_openwrt_pipeline run_step "Worker var file" ensure_cluster_worker_var_file run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" run_step "Version report" doctor_versions_report run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" run_step "Applications" apps run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" echo "Cluster rebuild successfully completed." echo "Log: ${JEANNIE_LOG_FILE}" } nuke_for_rebuild() { LAB_NUKE_SKIP_CONFIRM=true nuke } cluster_worker_targets() { local worker_ssh_targets="${WORKER_SSH_TARGETS-}" local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" CLUSTER_WORKER_TARGETS=() read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}" if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then while IFS= read -r target; do [[ -n "${target}" ]] || continue if [[ ! " ${CLUSTER_WORKER_TARGETS[*]} " =~ [[:space:]]${target}[[:space:]] ]]; then CLUSTER_WORKER_TARGETS+=("${target}") fi done < <(python3 - "${var_file}" "${worker_key_prefix}" <<'PY' import json import sys var_file, worker_key_prefix = sys.argv[1:3] with open(var_file, encoding="utf-8") as handle: document = json.load(handle) for key, node in sorted((document.get("worker_nodes") or {}).items()): if key.startswith(worker_key_prefix): continue host = node.get("host") user = node.get("user") if host and user: print(f"{user}@{host}") PY ) fi } stop_local_kubernetes_services() { echo "--> Stopping local Kubernetes services on the Debian control plane..." sudo systemctl stop kubelet 2>/dev/null || true if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true fi if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then sudo systemctl stop containerd 2>/dev/null || true sudo systemctl reset-failed containerd 2>/dev/null || true fi sudo systemctl reset-failed kubelet 2>/dev/null || true } start_local_kubernetes_services() { echo "--> Starting local Kubernetes services on the Debian control plane..." sudo systemctl start containerd sudo systemctl start docker 2>/dev/null || true sudo systemctl start kubelet } stop_remote_kubernetes_services() { local target="$1" echo "--> Stopping Kubernetes services on remote worker ${target}..." ssh -o ConnectTimeout=5 "${target}" \ "sudo systemctl stop kubelet 2>/dev/null || true; if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true; fi; if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi; sudo systemctl reset-failed kubelet 2>/dev/null || true" } start_remote_kubernetes_services() { local target="$1" echo "--> Starting Kubernetes services on remote worker ${target}..." ssh -o ConnectTimeout=5 "${target}" \ "sudo systemctl start containerd && sudo systemctl start kubelet" } pimox_worker_count_default() { local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" local max_count=0 local count local state_count if [[ -s "${spec_file}" ]]; then count="$(awk -F'\t' -v key_prefix="${worker_key_prefix}" -v node_prefix="${worker_node_prefix}-" ' $1 ~ "^" key_prefix "[0-9]+$" { index = substr($1, length(key_prefix) + 1) + 0 if (index > max) max = index } $4 ~ "^" node_prefix "[0-9]+$" { index = substr($4, length(node_prefix) + 1) + 0 if (index > max) max = index } END { print max + 0 } ' "${spec_file}")" if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then max_count="${count}" fi fi if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then count="$(python3 - "${var_file}" "${worker_key_prefix}" "${worker_node_prefix}" <<'PY' import json import re import sys var_file, worker_key_prefix, worker_node_prefix = sys.argv[1:4] with open(var_file, encoding="utf-8") as handle: document = json.load(handle) nodes = document.get("worker_nodes") or {} highest = 0 key_pattern = re.compile(rf"^{re.escape(worker_key_prefix)}(\d+)$") node_pattern = re.compile(rf"^{re.escape(worker_node_prefix)}-(\d+)$") for key, node in nodes.items(): for candidate in (key, str(node.get("node_name", ""))): match = key_pattern.match(candidate) or node_pattern.match(candidate) if match: highest = max(highest, int(match.group(1))) print(highest) PY )" if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then max_count="${count}" fi fi state_count="$(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" state show null_resource.worker_nodes_required 2>/dev/null | awk -F'"' '/"worker_count"[[:space:]]*=/ { print $4; found = 1 } END { exit found ? 0 : 1 }' || true)" if [[ "${state_count}" =~ ^[0-9]+$ && "${state_count}" -gt "${max_count}" ]]; then max_count="${state_count}" fi count="${LAB_PIMOX_DEFAULT_WORKER_COUNT:-1}" if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then max_count="${count}" fi printf '%s\n' "${max_count}" } stop_pimox_worker_vms() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}" local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}" local index local vmid if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 fi if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2 exit 1 fi if ((worker_count == 0)); then return 0 fi echo "--> Stopping Pimox worker VMs on ${pimox_host}..." for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." continue fi vmid=$((worker_base_vmid + index - 1)) pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then echo 'Pimox worker VM ${vmid} does not exist; skipping.' exit 0 fi if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then echo 'Pimox worker VM ${vmid} is already stopped.' exit 0 fi echo 'Gracefully shutting down Pimox worker VM ${vmid}...' if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then exit 0 fi if [ '${force_stop}' = 'true' ]; then echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.' sudo '${qm_bin}' stop '${vmid}' else echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2 exit 1 fi" done } destroy_pimox_worker_vms() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid if ! truthy "${LAB_NUKE_DESTROY_PIMOX_WORKERS:-true}"; then echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}." return 0 fi if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 fi if ((worker_count == 0)); then return 0 fi echo "--> Destroying Pimox worker VMs on ${pimox_host}..." for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." continue fi vmid=$((worker_base_vmid + index - 1)) pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then echo 'Pimox worker VM ${vmid} does not exist; skipping.' exit 0 fi if sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'; then echo 'Pimox VM ${vmid} is a template; refusing to destroy it as a worker.' >&2 exit 1 fi sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" done } start_pimox_worker_vms() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 exit 1 fi if ((worker_count == 0)); then return 0 fi echo "--> Starting Pimox worker VMs on ${pimox_host}..." for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." continue fi vmid=$((worker_base_vmid + index - 1)) pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then echo 'Pimox worker VM ${vmid} does not exist; skipping.' exit 0 fi if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then echo 'Pimox worker VM ${vmid} is already running.' exit 0 fi echo 'Starting Pimox worker VM ${vmid}...' sudo '${qm_bin}' start '${vmid}'" done } stop_cluster() { local target declare -a CLUSTER_WORKER_TARGETS=() require_debian_server "stop-cluster" cluster_worker_targets echo "Stopping Kubernetes cluster runtime without destroying state..." for target in "${CLUSTER_WORKER_TARGETS[@]}"; do stop_remote_kubernetes_services "${target}" done stop_pimox_worker_vms stop_local_kubernetes_services echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact." } start_cluster() { local target declare -a CLUSTER_WORKER_TARGETS=() require_debian_server "start-cluster" cluster_worker_targets echo "Starting Kubernetes cluster runtime without rebuilding state..." start_local_kubernetes_services start_pimox_worker_vms for target in "${CLUSTER_WORKER_TARGETS[@]}"; do start_remote_kubernetes_services "${target}" done echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness." } kubernetes_api_reachable() { kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1 } cluster_control_plane_tracked() { tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane" } wait_for_kubernetes_api() { local timeout_seconds="${1:-180}" local elapsed=0 until kubernetes_api_reachable; do if ((elapsed >= timeout_seconds)); then echo "Kubernetes API did not become reachable after ${timeout_seconds}s." >&2 return 1 fi sleep 5 elapsed=$((elapsed + 5)) done } ensure_existing_cluster_started_for_up() { if ! cluster_control_plane_tracked; then echo "No tracked kubeadm control plane found; bootstrap/cluster will create one." return 0 fi if kubernetes_api_reachable; then echo "Existing Kubernetes control plane is reachable." return 0 fi echo "Existing Kubernetes control plane is tracked but API is down; starting cluster runtime..." start_cluster wait_for_kubernetes_api "${LAB_CLUSTER_START_WAIT_SECONDS:-180}" echo "Existing Kubernetes control plane is reachable after start." } status_section() { printf '\n== %s ==\n' "$1" } status_run() { local description="$1" shift printf '\n-- %s\n' "${description}" if ! "$@"; then printf 'status check failed: %s\n' "${description}" >&2 fi } status_http() { local name="$1" local url="$2" if command -v curl >/dev/null 2>&1; then printf '%-28s ' "${name}" curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code} %{url_effective}\n' "${url}" || printf 'unreachable %s\n' "${url}" else printf '%-28s curl not installed\n' "${name}" fi } status_http_ok() { local url="$1" local status if ! command -v curl >/dev/null 2>&1; then echo "curl not installed" >&2 return 1 fi status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { echo "unreachable ${url}" >&2 return 1 } case "${status}" in 2* | 3*) return 0 ;; *) echo "HTTP ${status} from ${url}" >&2 return 1 ;; esac } status_http_reachable() { local url="$1" local status if ! command -v curl >/dev/null 2>&1; then echo "curl not installed" >&2 return 1 fi status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { echo "unreachable ${url}" >&2 return 1 } case "${status}" in 2* | 3* | 4*) return 0 ;; *) echo "HTTP ${status} from ${url}" >&2 return 1 ;; esac } status_gitea_public_ok() { local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" status_http_reachable "${root_url}" } status_cascade_check() { local description="$1" local output shift printf '%-36s ' "${description}" if output="$("$@" 2>&1)"; then printf 'ok\n' if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' fi return 0 fi printf 'fail\n' if [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /' fi return 1 } status_cascade_warn() { local description="$1" local output shift printf '%-36s ' "${description}" if output="$("$@" 2>&1)"; then printf 'ok\n' if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' fi return 0 fi printf 'warn\n' if [[ -n "${output}" ]]; then printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' fi return 0 } status_container_running() { local container="$1" if ! command -v docker >/dev/null 2>&1; then echo "docker not installed" >&2 return 1 fi if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then echo "container ${container} is not running" >&2 return 1 fi } status_kubernetes_api_ok() { if ! command -v kubectl >/dev/null 2>&1; then echo "kubectl not installed" >&2 return 1 fi if [[ ! -s "${KUBECONFIG_PATH}" ]]; then echo "kubeconfig missing: ${KUBECONFIG_PATH}" >&2 return 1 fi kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null } status_kubernetes_nodes_ready() { local nodes local not_ready nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes --no-headers 2>/dev/null)" || return 1 if [[ -z "${nodes}" ]]; then echo "no Kubernetes nodes found" >&2 return 1 fi not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")" if [[ -n "${not_ready}" ]]; then echo "not ready: ${not_ready}" >&2 return 1 fi } status_kubernetes_deployment_ready() { local namespace="$1" local deployment="$2" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null } status_no_problem_pods() { local rows rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }' )" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" >&2 return 1 fi } status_recent_deployments() { local rows rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get deployments -A \ -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.readyReplicas,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,AGE:.metadata.creationTimestamp' \ --no-headers 2>/dev/null | awk '$3 != $4 || $5 != $4 || $6 != $4 { print }' )" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" >&2 return 1 fi } status_pod_restart_pressure() { local threshold="${LAB_STATUS_RESTART_THRESHOLD:-3}" local rows rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | awk -v threshold="${threshold}" '$5 + 0 >= threshold { print $1 "/" $2 ": restarts=" $5 " age=" $6 }' )" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" >&2 return 1 fi } status_node_pressure() { local rows rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}{"="}{.status}{" "}{end}{"\n"}{end}' 2>/dev/null | awk '/MemoryPressure=True|DiskPressure=True|PIDPressure=True|NetworkUnavailable=True/ { print }' )" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" >&2 return 1 fi } status_disk_pressure() { local threshold="${LAB_STATUS_DISK_USE_THRESHOLD:-85}" local rows rows="$(df -P / /data 2>/dev/null | awk -v threshold="${threshold}" 'NR > 1 { gsub(/%/, "", $5); if ($5 + 0 >= threshold) print $6 ": " $5 "% used" }')" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" >&2 return 1 fi } status_traefik_error_rates() { local since="${LAB_STATUS_LOG_SINCE:-30m}" local sample local count_502 local count_404 sample="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik logs -l app.kubernetes.io/name=traefik --since="${since}" --tail=2000 2>/dev/null || true)" if [[ -z "${sample}" ]]; then echo "no recent Traefik logs available" >&2 return 1 fi count_502="$(grep -Eac '(^|[^0-9])50[234]([^0-9]|$)|level=error' <<<"${sample}" || true)" count_404="$(grep -Eac '(^|[^0-9])404([^0-9]|$)' <<<"${sample}" || true)" if ((count_502 > 0)); then printf '5xx/error lines in last %s: %s\n' "${since}" "${count_502}" >&2 return 1 fi printf '404 lines in last %s: %s\n' "${since}" "${count_404}" >&2 } status_gitea_recent_errors() { local since="${LAB_STATUS_LOG_SINCE:-30m}" local container="${GITEA_CONTAINER_NAME:-homelab-gitea}" local sample local matches if ! command -v docker >/dev/null 2>&1; then echo "docker not installed" >&2 return 1 fi if ! sudo docker inspect "${container}" >/dev/null 2>&1; then echo "container ${container} not found" >&2 return 1 fi sample="$(sudo docker logs --since "${since}" "${container}" 2>&1 || true)" matches="$(grep -Eai 'error|panic|fatal|authentication failed|denied| 50[0-9] ' <<<"${sample}" | tail -10 || true)" if [[ -n "${matches}" ]]; then printf '%s\n' "${matches}" >&2 return 1 fi } status_registry_ok() { status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/" } status_rpi_uptime_kuma_ok() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \ "curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null" } status_pimox_workers_running() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid local failures=0 if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then echo "invalid worker count ${worker_count}" >&2 return 1 fi for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then continue fi vmid=$((worker_base_vmid + index - 1)) if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then echo "VM ${vmid} is not running" >&2 failures=$((failures + 1)) fi done ((failures == 0)) } status_cascade_report() { local failures=0 status_section "Status Cascade" status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1)) status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1)) status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1)) status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1)) status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1)) status_section "Git And Local Services" status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1)) status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1)) status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/" status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/" status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/" status_section "RPi Bootstrap And DNS" status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1)) status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1)) status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1)) status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok status_section "Pimox And Cluster" status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1)) status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1)) status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1)) status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1)) status_cascade_warn "No problem pods" status_no_problem_pods status_section "What Broke Signals" status_cascade_warn "Backup freshness" backup_status status_cascade_warn "Recent deployments healthy" status_recent_deployments status_cascade_warn "Pod restart pressure" status_pod_restart_pressure status_cascade_warn "Node pressure" status_node_pressure status_cascade_warn "Disk usage" status_disk_pressure status_cascade_warn "Traefik 5xx/404 signals" status_traefik_error_rates status_cascade_warn "Gitea recent errors" status_gitea_recent_errors status_section "Platform And Apps" status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1)) status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1)) status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1)) status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server status_section "Edge And Public URLs" status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1)) status_cascade_check "Traefik LoadBalancer HTTP" status_http_reachable "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" || failures=$((failures + 1)) status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" || failures=$((failures + 1)) status_cascade_check "Gitea public route" status_gitea_public_ok || failures=$((failures + 1)) if ((failures > 0)); then printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2 return 1 fi printf '\nStatus cascade passed.\n' } backstage_brain_enabled() { local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}" local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}" [[ "${provider,,}" == "ollama" ]] && truthy "${enabled}" } backstage_brain_note() { local topic="$1" local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}" local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" local context local retrieved_context="" if ! backstage_brain_enabled; then return 0 fi if ! command -v python3 >/dev/null 2>&1; then return 0 fi context="$(cat)" if [[ -s "${index_dir}/index.json" ]]; then retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)" fi if [[ -n "${retrieved_context}" ]]; then context="${context} Retrieved homelab knowledge: ${retrieved_context}" fi BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true import json import os import sys import urllib.error import urllib.request endpoint, model, timeout, topic = sys.argv[1:5] context = os.environ.get("BACKSTAGE_CONTEXT", "").strip() try: timeout_seconds = int(timeout) except ValueError: timeout_seconds = 20 prompt = f"""You are the backstage helper for a personal homelab script named jeannie. Be concise and operational. Do not invent facts. Use only the supplied context. Return at most 5 bullets. Focus on likely cause, next command, and risk. Topic: {topic} Context: {context} """ payload = { "model": model, "prompt": prompt, "stream": False, "options": { "temperature": 0.1, "num_predict": 220, }, } try: tags_request = urllib.request.Request(f"{endpoint.rstrip('/')}/api/tags") with urllib.request.urlopen(tags_request, timeout=3) as response: tags = json.loads(response.read().decode("utf-8")) available = {item.get("name", "") for item in tags.get("models", [])} if model not in available and f"{model}:latest" not in available: print(f"\n== Backstage Notes ==\nOllama is reachable, but model '{model}' is not pulled. Run: ollama pull {model}") raise SystemExit(0) request = urllib.request.Request( f"{endpoint.rstrip('/')}/api/generate", data=json.dumps(payload).encode("utf-8"), headers={"Content-Type": "application/json"}, method="POST", ) with urllib.request.urlopen(request, timeout=timeout_seconds) as response: result = json.loads(response.read().decode("utf-8")) except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError): raise SystemExit(0) note = (result.get("response") or "").strip() if note: print("\n== Backstage Notes ==") print(note) PY } status_systemd_units() { local unit for unit in "$@"; do if systemctl list-unit-files --no-legend "${unit}.service" 2>/dev/null | grep -q .; then printf '%-28s %s\n' "${unit}" "$(systemctl is-active "${unit}" 2>/dev/null || true)" else printf '%-28s not-installed\n' "${unit}" fi done } status_compose_stack() { local name="$1" local directory="$2" printf '\n-- Docker Compose: %s (%s)\n' "${name}" "${directory}" if [[ ! -d "${directory}" ]]; then printf 'missing directory\n' return 0 fi if ! command -v docker >/dev/null 2>&1; then printf 'docker not installed\n' return 0 fi if ! sudo docker compose version >/dev/null 2>&1; then printf 'docker compose unavailable\n' return 0 fi (cd "${directory}" && sudo docker compose ps) || true } status_kubernetes() { status_section "Kubernetes" if ! command -v kubectl >/dev/null 2>&1; then printf 'kubectl not installed\n' return 0 fi if [[ ! -s "${KUBECONFIG_PATH}" ]]; then printf 'kubeconfig missing: %s\n' "${KUBECONFIG_PATH}" return 0 fi if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1; then printf 'API server not reachable through %s\n' "${KUBECONFIG_PATH}" return 0 fi status_run "nodes" kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide status_kubernetes_problem_pods status_run "traefik services" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik get svc,pods -o wide status_run "website pods" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n website-production get pods -o wide } status_kubernetes_problem_pods() { local rows printf '\n-- pods not Running/Completed\n' rows="$( kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers -o wide | awk '$4 != "Running" && $4 != "Completed" { print }' )" if [[ -n "${rows}" ]]; then printf '%s\n' "${rows}" else printf 'none\n' fi } status_pimox_workers() { local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}" local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local index local vmid status_section "Pimox Workers" if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}" return 0 fi if ((worker_count == 0)); then printf 'worker count is 0\n' return 0 fi for ((index = 1; index <= worker_count; index++)); do if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then printf 'worker index %s skipped\n' "${index}" continue fi vmid=$((worker_base_vmid + index - 1)) printf 'VM %-6s ' "${vmid}" pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}'" 2>/dev/null || printf 'unreachable or missing\n' done } status_rpi_services() { local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}" local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}" local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" status_section "RPi Services" ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " set +e dns_query() { server=\"\$1\" port=\"\$2\" name=\"\$3\" python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY' import random import socket import struct import sys server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.') query_id = random.randrange(0, 65536) packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0) for label in name.split('.'): packet += bytes([len(label)]) + label.encode('ascii') packet += b'\x00' + struct.pack('!HH', 1, 1) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.settimeout(5) sock.sendto(packet, (server, port)) data, _ = sock.recvfrom(512) if len(data) < 12: raise SystemExit('short DNS response') response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12]) rcode = flags & 0x000f if response_id != query_id: raise SystemExit('mismatched DNS response id') if rcode != 0: raise SystemExit(f'DNS rcode {rcode}') if an < 1: raise SystemExit('DNS response had no answers') print(f'ok answers={an}') PY } check() { label=\"\$1\" shift printf '%-34s ' \"\$label\" output=\"\$("\$@" 2>&1)\" status=\$? if [ \"\$status\" -eq 0 ]; then printf 'ok' if [ -n \"\$output\" ]; then printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\" fi printf '\n' else printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\" fi return 0 } docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\" printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\" if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then if mountpoint -q '${docker_nvme_root}'; then printf '%-34s ok - nvme mount active\n' 'Docker root class' else printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class' fi elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then printf '%-34s ok - fallback root active\n' 'Docker root class' else printf '%-34s warn - unexpected docker root\n' 'Docker root class' fi if [ -d '${install_dir}' ]; then cd '${install_dir}' sudo docker compose ps || true unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\" check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com if [ -n \"\$unbound_ip\" ]; then check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com else printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query' fi upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\" if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config' else printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config' fi check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\" else echo 'RPi services install directory is missing: ${install_dir}' fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" } status_report() { local cascade_status=0 require_debian_server "status" status_cascade_report || cascade_status=$? if ! truthy "${LAB_STATUS_DETAILS:-false}"; then return "${cascade_status}" fi status_section "Host" printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)" printf 'date: %s\n' "$(date -Is)" printf 'uptime: %s\n' "$(uptime -p 2>/dev/null || uptime)" status_run "memory" free -h status_run "disk" df -h / /data /data/openebs/local status_section "Systemd" status_systemd_units ssh docker containerd kubelet tailscaled homelab-gitea-runner status_section "Docker" if command -v docker >/dev/null 2>&1; then status_run "docker containers" sudo docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' else printf 'docker not installed\n' fi status_compose_stack "gitea" "/data/homelab-gitea" status_compose_stack "arr-stack" "${REPO_ROOT}/infra/arr-stack" status_kubernetes status_pimox_workers status_rpi_services status_section "Tailscale" if command -v tailscale >/dev/null 2>&1; then status_run "tailscale ip" tailscale ip -4 status_run "tailscale peers" tailscale status else printf 'tailscale not installed\n' fi status_section "HTTP" status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" status_http "arr radarr" "http://127.0.0.1:7878/" status_http "arr sonarr" "http://127.0.0.1:8989/" status_http "arr prowlarr" "http://127.0.0.1:9696/" return "${cascade_status}" } doctor_edge() { local edge_host="${LAB_EDGE_HOST:-132.145.170.74}" local edge_user="${LAB_EDGE_USER:-ubuntu}" local traefik_ip="${LAB_TRAEFIK_LB_IP:-192.168.100.240}" local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}" local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" require_debian_server "doctor-edge" status_section "Doctor Edge" status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}" printf '\n-- edge backend reachability\n' if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " set +e printf '%-28s ' 'traefik backend' curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable printf '%-28s ' 'gitea backend' curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable printf '\n-- tailscale\n' tailscale status || true "; then printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}" fi cat <<'EOF' Next steps: - If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster. - If only Gitea is unreachable, run ./jeannie doctor-gitea. - If both are unreachable from OCI, check Tailscale routes and bootstrap/edge. - Runbook: docs/runbooks/edge-failures.md EOF backstage_brain_note "doctor-edge" </dev/null 2>&1; then sudo crictl ps -a | head -50 || true else printf 'crictl not installed\n' fi cat <<'EOF' Next steps: - If API server is unreachable after a stop, run ./jeannie start-cluster. - If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node. - If Pimox workers are stopped, start them with ./jeannie start-cluster. - If kubelet versions drift, run ./jeannie doctor-versions. - Runbook: docs/runbooks/cluster-stop-start-failures.md EOF backstage_brain_note "doctor-cluster" <&2 <<'EOF' nuke destroys Kubernetes state and Pimox worker VMs. Rerun with LAB_CONFIRM_NUKE=homelab when that destructive action is intended. EOF exit 1 fi echo "Brutally nuking the homelab infrastructure..." cluster_worker_targets echo "--> Terminating local OpenTofu tasks..." killall tofu terraform 2>/dev/null || true echo "--> Eviscerating local Kubernetes components..." cleanup_node sudo rm -f "${KUBECONFIG_PATH}" for target in "${CLUSTER_WORKER_TARGETS[@]}"; do echo "--> Eviscerating remote Kubernetes components (${target})..." if ! ssh -o ConnectTimeout=5 "${target}" "bash -s" <<'EOF' set -euo pipefail cleanup_calico_links() { ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true sudo ip link delete vxlan.calico 2>/dev/null || true sudo ip link delete tunl0 2>/dev/null || true sudo ip link delete cni0 2>/dev/null || true sudo ip link delete kube-ipvs0 2>/dev/null || true ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true } cleanup_iptables() { sudo iptables -F || true sudo iptables -X || true sudo iptables -t nat -F || true sudo iptables -t nat -X || true sudo iptables -t mangle -F || true sudo iptables -t mangle -X || true sudo iptables -t raw -F || true sudo iptables -t raw -X || true if command -v ipvsadm >/dev/null 2>&1; then sudo ipvsadm --clear || true fi } cleanup_calico_runtime_files() { local path for path in /run/calico /var/run/calico; do if sudo test -e "${path}"; then sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true sudo rmdir "${path}" 2>/dev/null || true fi done } restore_node_dns() { sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then sudo rm -f /etc/resolv.conf sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf fi sudo systemctl restart systemd-resolved 2>/dev/null || true } cleanup_mounts() { if command -v findmnt >/dev/null 2>&1; then local mount_root while IFS= read -r mountpoint; do sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true done < <( for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true done | sort -ru ) fi while IFS= read -r mountpoint; do sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true } sudo kubeadm reset --force || true sudo systemctl stop kubelet 2>/dev/null || true sudo systemctl stop containerd 2>/dev/null || true sudo killall containerd-shim-runc-v2 2>/dev/null || true cleanup_mounts sudo rm -rf \ /etc/kubernetes/ \ /var/lib/etcd/ \ /var/lib/kubelet/ \ /var/lib/cni/ \ /etc/cni/net.d \ /run/flannel \ /var/lib/calico \ /var/log/calico \ /var/lib/containerd/* \ /run/containerd/* \ /etc/containerd/certs.d \ /etc/containerd/config.toml cleanup_calico_runtime_files sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam cleanup_iptables cleanup_calico_links restore_node_dns sudo mkdir -p /etc/containerd/certs.d sudo systemctl reset-failed kubelet containerd 2>/dev/null || true sudo systemctl start containerd 2>/dev/null || true EOF then echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 exit 1 fi done destroy_pimox_worker_vms docker buildx rm lab-builder 2>/dev/null || true docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true rm -f "${BUILDX_CONFIG}" || true echo "--> Backing up local OpenTofu tracking state files..." backup_tofu_state echo "--> Deleting OpenTofu tracking state files..." rm -rf "${REPO_ROOT}"/bootstrap/cluster/terraform.tfstate* rm -f "${REPO_ROOT}"/bootstrap/cluster/.terraform.tfstate.lock.info rm -rf "${REPO_ROOT}"/bootstrap/cluster/.terraform/ rm -rf "${REPO_ROOT}"/bootstrap/platform/terraform.tfstate* rm -f "${REPO_ROOT}"/bootstrap/platform/.terraform.tfstate.lock.info rm -rf "${REPO_ROOT}"/bootstrap/platform/.terraform/ rm -rf "${REPO_ROOT}"/bootstrap/apps/terraform.tfstate* rm -f "${REPO_ROOT}"/bootstrap/apps/.terraform.tfstate.lock.info rm -rf "${REPO_ROOT}"/bootstrap/apps/.terraform/ rm -rf "${REPO_ROOT}"/bootstrap/edge/terraform.tfstate* rm -f "${REPO_ROOT}"/bootstrap/edge/.terraform.tfstate.lock.info rm -rf "${REPO_ROOT}"/bootstrap/edge/.terraform/ rm -f "${REPO_ROOT}/.lab/pimox-workers.tsv" "${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" echo "Destruction complete. Retained data under /data/openebs/local was left intact." } ensure_sops_age_tools() { local missing_packages=() if ! command -v age-keygen >/dev/null 2>&1; then missing_packages+=(age) fi if ! command -v sops >/dev/null 2>&1; then missing_packages+=(sops) fi if ((${#missing_packages[@]} > 0)); then echo "Installing missing secret-management tools: ${missing_packages[*]}" sudo apt-get update sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" fi if ! command -v age-keygen >/dev/null 2>&1; then echo "age-keygen is still unavailable after package installation." >&2 exit 1 fi if ! command -v sops >/dev/null 2>&1; then echo "sops is still unavailable after package installation." >&2 exit 1 fi } sops_age_key_file() { printf '%s\n' "${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" } sops_age_recipient() { local key_file="$1" awk -F': ' '/^# public key:/ { print $2; exit }' "${key_file}" } secrets_init() { local key_file local key_dir local recipient local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" local example_file="${REPO_ROOT}/.sops.yaml.example" require_debian_server "secrets-init" ensure_sops_age_tools key_file="$(sops_age_key_file)" key_dir="$(dirname "${key_file}")" mkdir -p "${key_dir}" chmod 700 "${key_dir}" if [[ ! -s "${key_file}" ]]; then echo "Generating age identity at ${key_file}..." age-keygen -o "${key_file}" chmod 600 "${key_file}" else echo "Using existing age identity at ${key_file}." chmod 600 "${key_file}" fi recipient="$(sops_age_recipient "${key_file}")" if [[ -z "${recipient}" ]]; then echo "Could not read the public recipient from ${key_file}." >&2 exit 1 fi if [[ -e "${config_file}" ]]; then if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then echo "${config_file} still contains the placeholder age recipient." >&2 exit 1 fi echo "SOPS config already exists: ${config_file}" else if [[ ! -s "${example_file}" ]]; then echo "Missing ${example_file}" >&2 exit 1 fi sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}" echo "Wrote ${config_file} with Debian host age recipient ${recipient}." fi echo "Private age key: ${key_file}" echo "Public age recipient: ${recipient}" echo "Review and commit ${config_file}; never commit ${key_file}." } secrets_check_file() { local file="$1" local key_file="$2" case "${file}" in *.json) grep -q '"sops"' "${file}" || { echo "${file} does not look SOPS-encrypted." >&2 return 1 } ;; *) grep -q '^sops:' "${file}" || { echo "${file} does not look SOPS-encrypted." >&2 return 1 } ;; esac if [[ -s "${key_file}" ]]; then SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null fi } secrets_check() { local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" local key_file local recipient="" local failures=0 local found_files=0 local file key_file="$(sops_age_key_file)" if [[ ! -s "${config_file}" ]]; then echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2 failures=$((failures + 1)) elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then echo "${config_file} still contains the placeholder age recipient." >&2 failures=$((failures + 1)) fi if [[ -s "${key_file}" ]]; then recipient="$(sops_age_recipient "${key_file}")" if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then echo "${config_file} does not include this host's age recipient ${recipient}." >&2 failures=$((failures + 1)) fi else echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting." fi if command -v git >/dev/null 2>&1; then while IFS= read -r file; do [[ -n "${file}" ]] || continue found_files=$((found_files + 1)) secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1)) done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json') fi if ((failures > 0)); then echo "Secret checks failed with ${failures} issue(s)." >&2 exit 1 fi if ((found_files == 0)); then echo "SOPS config checks passed. No encrypted secret files are committed yet." else echo "SOPS config checks passed for ${found_files} encrypted secret file(s)." fi } tailnet_policy_check() { "${REPO_ROOT}/scripts/validate-tailnet-policy" } ai_index() { local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" require_debian_server "ai-index" ensure_python3 if [[ "${index_dir}" == /data/* ]]; then sudo mkdir -p "${index_dir}" sudo chown "${USER}:$(id -gn)" "${index_dir}" fi "${REPO_ROOT}/scripts/build-homelab-ai-index" --index-dir "${index_dir}" } ai_check() { local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" local failures=0 require_debian_server "ai-check" status_section "AI Knowledge" if command -v python3 >/dev/null 2>&1; then printf '%-28s ok\n' "python3" else printf '%-28s missing\n' "python3" failures=$((failures + 1)) fi if [[ -s "${index_dir}/index.json" ]]; then printf '%-28s %s\n' "knowledge index" "${index_dir}/index.json" else printf '%-28s missing - run ./jeannie ai-index\n' "knowledge index" failures=$((failures + 1)) fi if "${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 2 "gitea edge traefik" >/dev/null 2>&1; then printf '%-28s ok\n' "retrieval smoke test" else printf '%-28s failed\n' "retrieval smoke test" failures=$((failures + 1)) fi if backstage_brain_enabled; then if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}" if python3 - "${endpoint}" "${model}" <<'PY' import json import sys import urllib.request endpoint, model = sys.argv[1:3] with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response: tags = json.loads(response.read().decode("utf-8")) models = {item.get("name", "") for item in tags.get("models", [])} raise SystemExit(0 if model in models or f"{model}:latest" in models else 1) PY then printf '%-28s ok - %s\n' "ollama model" "${model}" else printf '%-28s missing - run: ollama pull %s\n' "ollama model" "${model}" failures=$((failures + 1)) fi else printf '%-28s unreachable - %s\n' "ollama endpoint" "${endpoint}" failures=$((failures + 1)) fi else printf '%-28s disabled\n' "backstage helper" fi if ((failures > 0)); then echo "AI check failed with ${failures} issue(s)." >&2 exit 1 fi echo "AI check passed." } security_scan() { require_debian_server "security-scan" "${REPO_ROOT}/scripts/security-scan" all } security_prepare() { require_debian_server "security-prepare" "${REPO_ROOT}/scripts/security-scan" prepare } security_zap() { require_debian_server "security-zap" "${REPO_ROOT}/scripts/security-scan" zap } security_k8s() { require_debian_server "security-k8s" "${REPO_ROOT}/scripts/security-scan" kube-bench } security_host() { require_debian_server "security-host" "${REPO_ROOT}/scripts/security-scan" host } security_trivy() { require_debian_server "security-trivy" "${REPO_ROOT}/scripts/security-scan" trivy } security_secrets() { require_debian_server "security-secrets" "${REPO_ROOT}/scripts/security-scan" secrets } security_nuclei() { require_debian_server "security-nuclei" "${REPO_ROOT}/scripts/security-scan" nuclei } security_web() { require_debian_server "security-web" "${REPO_ROOT}/scripts/security-scan" web } security_logs() { require_debian_server "security-logs" "${REPO_ROOT}/scripts/security-logs" } security_runtime() { require_debian_server "security-runtime" if ! command -v kubectl >/dev/null 2>&1; then echo "kubectl is required for security-runtime." >&2 exit 1 fi kubectl --kubeconfig "${KUBECONFIG}" -n kube-system rollout status daemonset/tetragon --timeout=30s kubectl --kubeconfig "${KUBECONFIG}" -n kube-system get pods -l app.kubernetes.io/name=tetragon -o wide cat <<'EOF' Watch recent Tetragon events: kubectl -n kube-system logs -l app.kubernetes.io/name=tetragon -c export-stdout --tail=100 -f Useful practice trigger: kubectl -n security-lab exec deploy/juice-shop -- sh -c 'id; uname -a' EOF } security_attack_path() { require_debian_server "security-attack-path" "${REPO_ROOT}/scripts/security-attack-path" "${@:2}" } validate_homelab() { "${REPO_ROOT}/scripts/validate-homelab" } access_audit() { require_debian_server "access-audit" "${REPO_ROOT}/scripts/access-audit" } drill_restore() { require_debian_server "drill-restore" "${REPO_ROOT}/scripts/restore-drill" all } drill_pihole_restore() { "${REPO_ROOT}/scripts/restore-drill" pihole } kubeconfig_readonly() { require_debian_server "kubeconfig-readonly" "${REPO_ROOT}/scripts/kubeconfig-readonly" } capacity_report() { require_debian_server "capacity" "${REPO_ROOT}/scripts/capacity-report" "${@:2}" } recover_plan() { require_debian_server "recover-plan" "${REPO_ROOT}/scripts/recover-plan" } gitops_status() { require_debian_server "gitops-status" "${REPO_ROOT}/scripts/gitops-status" } cert_check() { "${REPO_ROOT}/scripts/cert-check" } release_snapshot() { require_debian_server "release-snapshot" "${REPO_ROOT}/scripts/release-snapshot" } backup_status() { require_debian_server "backup-status" "${REPO_ROOT}/scripts/backup-status" } synthetic_checks() { require_debian_server "synthetic-checks" "${REPO_ROOT}/scripts/synthetic-checks" } resource_budget() { require_debian_server "resource-budget" "${REPO_ROOT}/scripts/resource-budget" } artifact_cache() { require_debian_server "artifact-cache" "${REPO_ROOT}/scripts/artifact-cache" "${@:2}" } blockchain_devnet() { require_debian_server "blockchain-devnet" "${REPO_ROOT}/scripts/blockchain-devnet" "${@:2}" } blockchain_test() { "${REPO_ROOT}/scripts/blockchain-test" "${@:2}" } blockchain_wallet() { "${REPO_ROOT}/scripts/blockchain-wallet" "${@:2}" } golden_ledger() { "${REPO_ROOT}/scripts/golden-ledger" "${@:2}" } route_inventory() { require_debian_server "route-inventory" "${REPO_ROOT}/scripts/route-inventory" } recover_power() { require_debian_server "recover-power" record_change_journal "recover-power" "$@" "${REPO_ROOT}/scripts/recover-power" "${@:2}" } scorecard() { require_debian_server "scorecard" "${REPO_ROOT}/scripts/scorecard" } grafana_dashboards() { require_debian_server "grafana-dashboards" "${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}" } workers_manage() { require_debian_server "workers" "${REPO_ROOT}/scripts/workers" "${@:2}" } record_change_journal() { local command_name="$1" shift || true "${REPO_ROOT}/scripts/change-journal" append "${command_name}" "$*" || true } change_journal() { "${REPO_ROOT}/scripts/change-journal" "${@:2}" } homelab_map() { "${REPO_ROOT}/scripts/homelab-map" "${@:2}" } print_usage() { cat <<'EOF' Usage: ./jeannie [args] Base, Inventory, And Planning help | -h | --help Show this grouped help. validate Run repo validation checks. inventory-check Validate the canonical homelab inventory. preflight Run non-mutating infrastructure preflight checks. doctor-preapply Run deeper pre-apply safety checks. plan [all|provisioning|cluster|platform|apps|edge] Run OpenTofu plans without applying changes. Build And Bootstrap up Deploy the full homelab pipeline. deploy-gitea Deploy the Debian-hosted Gitea container. bootstrap-gitea-repo Ensure the Gitea repo and SSH key wiring exist. rpi-services Deploy Pi-hole, Unbound, and Uptime Kuma on RPi4. ollama-setup Install/configure Ollama on the Debian host. artifact-cache {status|up|down|instructions} Manage optional Debian artifact caches. blockchain-devnet {status|up|down|logs|rpc} Manage the local Ethereum Anvil devnet. blockchain-test [forge args...] Run Foundry tests for labs/blockchain. blockchain-wallet {instructions|new|address|sign-message} Practice dev wallet and signing workflows. golden-ledger {show|check} Show or validate Pimox golden image versions. Cluster Lifecycle rebuild-cluster Recreate the cluster through the guarded path. stop-cluster Stop Kubernetes and worker VMs without destroy. start-cluster Start Kubernetes and desired worker VMs. workers Manage Pimox/Kubernetes workers. move-prometheus-stack-workers Move monitoring workloads to worker nodes. doctor-versions Check Kubernetes/container tooling versions. Operator View And Reports status Cascading health check from host to public URLs. scorecard Compact pass/warn/fail operator scorecard. capacity Compact capacity and placement report. recover-plan Print disaster recovery order and prerequisites. recover-power [--dry-run] Run or preview post-outage recovery. map [--dot] Print the homelab dependency map. change-journal {list|path} Show risky-command journal entries. release-snapshot Write a pre-change release snapshot. Observability And GitOps grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards. gitops-status Print focused Argo CD health and sync status. cert-check Check public DNS, TLS, and edge URL health. backup-status Check backup and restore-drill freshness. synthetic-checks Run end-to-end service probes. resource-budget Report Kubernetes resource request/limit gaps. route-inventory Report routes and Uptime Kuma coverage. Recovery And State backup-gitea Back up the Debian-hosted Gitea data. drill-restore Run all restore drills. drill-gitea-restore Run the Gitea restore drill. drill-pihole-restore Run the Pi-hole restore drill. state-backup Back up local OpenTofu state files. Focused Doctors doctor-edge Diagnose edge routing and public exposure. doctor-gitea Diagnose Gitea local/public access. doctor-rpi Diagnose RPi services and DNS. doctor-cluster Diagnose Kubernetes cluster health. Access, Secrets, And Policy access-audit Audit SSH, Gitea, Kubernetes, and Tailscale access. kubeconfig-readonly Create/check read-only kubeconfig material. secrets-init Initialize SOPS/age secret tooling. secrets-check Validate repo-managed secret prerequisites. tailnet-policy-check Validate Tailscale ACL policy as code. fix-debian-docker-root Repair Debian Docker root placement. Apps And Services apps Deploy application stack only. website-translation-model Prepare website translation model support. website-ollama-listen Configure website Ollama access. install-gitea-runner [TOKEN] Install the Gitea Actions runner. openwrt Deploy/check OpenWrt lab config. Security Learning security-scan Run the security scan bundle. security-prepare Prepare defensive security tools. security-zap Run OWASP ZAP checks. security-k8s Run Kubernetes security checks. security-host Run host security checks. security-trivy Run Trivy scans. security-secrets Run secret leak checks. security-nuclei Run Nuclei web checks. security-web Run web security checks. security-logs Review security-relevant logs. security-runtime Check runtime security sensors. security-attack-path Print prioritized attack-path findings. AI And Indexing ai-index Build the local homelab RAG index. ai-check Query/check the local AI index. Destructive nuke Guarded cluster state destruction path. EOF } case "${1:-}" in "" | help | -h | --help) print_usage ;; up) record_change_journal "up" "$@" up ;; plan) plan_homelab "${2:-all}" ;; rebuild-cluster) record_change_journal "rebuild-cluster" "$@" rebuild_cluster ;; stop-cluster) record_change_journal "stop-cluster" "$@" stop_cluster ;; start-cluster) record_change_journal "start-cluster" "$@" start_cluster ;; status) status_report ;; capacity) capacity_report "$@" ;; recover-plan) recover_plan ;; gitops-status) gitops_status ;; cert-check) cert_check ;; release-snapshot) record_change_journal "release-snapshot" "$@" release_snapshot ;; backup-status) backup_status ;; synthetic-checks) synthetic_checks ;; resource-budget) resource_budget ;; artifact-cache) artifact_cache "$@" ;; blockchain-devnet) blockchain_devnet "$@" ;; blockchain-test) blockchain_test "$@" ;; blockchain-wallet) blockchain_wallet "$@" ;; golden-ledger) golden_ledger "$@" ;; route-inventory) route_inventory ;; recover-power) recover_power "$@" ;; scorecard) scorecard ;; grafana-dashboards) grafana_dashboards "$@" ;; workers) workers_manage "$@" ;; change-journal) change_journal "$@" ;; map) homelab_map "$@" ;; validate) validate_homelab ;; access-audit) access_audit ;; kubeconfig-readonly) kubeconfig_readonly ;; apps) record_change_journal "apps" "$@" require_debian_server "apps" jeannie_log_start "apps" jeannie_step_plan 1 run_step "Applications" apps echo "Log: ${JEANNIE_LOG_FILE}" ;; website-translation-model) website_translation_model ;; website-ollama-listen) website_ollama_listen ;; ollama-setup) ollama_setup ;; deploy-gitea) record_change_journal "deploy-gitea" "$@" deploy_gitea ;; rpi-services) record_change_journal "rpi-services" "$@" deploy_rpi_services ;; bootstrap-gitea-repo) bootstrap_gitea_repo ;; backup-gitea) backup_gitea ;; drill-restore) drill_restore ;; drill-gitea-restore) drill_gitea_restore ;; drill-pihole-restore) drill_pihole_restore ;; install-gitea-runner) install_gitea_runner "${2:-}" ;; move-prometheus-stack-workers) move_prometheus_stack_workers ;; doctor-versions) doctor_versions ;; doctor-edge) doctor_edge ;; doctor-gitea) doctor_gitea ;; doctor-rpi) doctor_rpi ;; doctor-cluster) doctor_cluster ;; preflight) homelab_preflight ;; doctor-preapply) doctor_preapply ;; inventory-check) inventory_check ;; state-backup) backup_tofu_state ;; fix-debian-docker-root) fix_debian_docker_root ;; secrets-init) secrets_init ;; secrets-check) secrets_check ;; tailnet-policy-check) tailnet_policy_check ;; ai-index) ai_index ;; ai-check) ai_check ;; security-scan) security_scan ;; security-prepare) security_prepare ;; security-zap) security_zap ;; security-k8s) security_k8s ;; security-host) security_host ;; security-trivy) security_trivy ;; security-secrets) security_secrets ;; security-nuclei) security_nuclei ;; security-web) security_web ;; security-logs) security_logs ;; security-runtime) security_runtime ;; security-attack-path) security_attack_path "$@" ;; openwrt) openwrt ;; nuke) record_change_journal "nuke" "$@" require_debian_server "nuke" jeannie_log_start "nuke" jeannie_step_plan 1 run_step "Nuke homelab cluster state" nuke echo "Log: ${JEANNIE_LOG_FILE}" ;; *) printf 'Unknown command: %s\n\n' "$1" >&2 print_usage >&2 exit 1 ;; esac