diff --git a/jeannie b/jeannie index 0333709..54455e9 100755 --- a/jeannie +++ b/jeannie @@ -27,321 +27,12 @@ source "${REPO_ROOT}/scripts/report-ui" trap 'rm -f "${BUILDX_CONFIG}"' EXIT -load_homelab_inventory_defaults() { - local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" - local rendered_defaults - - if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then - return 0 - fi - - rendered_defaults="$(python3 - "${inventory_file}" <<'PY' -import re -import json -import shlex -import sys - -inventory_file = sys.argv[1] -mapping = { - "domain.base": "LAB_DOMAIN", - "domain.public_url": "LAB_PUBLIC_URL", - "domain.gitea_url": "LAB_GITEA_ROOT_URL", - "network.lan_cidr": "LAB_LAN_CIDR", - "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX", - "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP", - "hosts.debian.user": "LAB_DEBIAN_USER", - "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP", - "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP", - "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT", - "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH", - "hosts.rpi4.user": "LAB_RPI_USER", - "hosts.rpi4.lan_ip": "LAB_RPI_HOST", - "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", - "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT", - "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", - "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", - "hosts.opi5_pimox.user": "LAB_PIMOX_USER", - "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST", - "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE", - "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE", - "hosts.oci_edge.user": "LAB_EDGE_USER", - "hosts.oci_edge.public_ip": "LAB_EDGE_HOST", - "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR", - "services.gitea.http_port": "LAB_GITEA_HTTP_PORT", - "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", - "services.gitea.root_url": "LAB_GITEA_ROOT_URL", - "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", - "services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR", - "services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT", - "services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL", - "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", - "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS", - "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR", - "services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE", - "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT", - "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT", - "ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER", - "ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED", - "ai_gateway.url": "LAB_AI_GATEWAY_URL", - "ai_gateway.model": "LAB_AI_GATEWAY_MODEL", - "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS", - "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR", - "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", - "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", -} - -def flatten(document, prefix=""): - values = {} - if isinstance(document, dict): - for key, value in document.items(): - path = f"{prefix}.{key}" if prefix else str(key) - if isinstance(value, dict): - values.update(flatten(value, path)) - elif isinstance(value, list): - values[path] = ",".join(str(item) for item in value) - elif value is not None: - values[path] = str(value) - return values - - -def parse_simple_inventory(path): - stack = [] - values = {} - pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") - - with open(path, encoding="utf-8") as handle: - for raw_line in handle: - if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): - continue - match = pattern.match(raw_line.rstrip("\n")) - if not match: - continue - indent = len(match.group(1)) - key = match.group(2) - value = (match.group(3) or "").strip() - while stack and stack[-1][0] >= indent: - stack.pop() - current_path = ".".join([item[1] for item in stack] + [key]) - if value == "": - stack.append((indent, key)) - continue - if " #" in value: - value = value.split(" #", 1)[0].strip() - values[current_path] = value.strip("\"'") - return values - - -try: - import yaml -except ImportError: - values = parse_simple_inventory(inventory_file) -else: - with open(inventory_file, encoding="utf-8") as handle: - values = flatten(yaml.safe_load(handle) or {}) - -for path, env_name in mapping.items(): - value = values.get(path) - if value: - print(f": ${{{env_name}:={shlex.quote(value)}}}") - -debian_ip = values.get("hosts.debian.lan_ip") -gitea_http = values.get("services.gitea.http_port") -if debian_ip: - print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}") -if debian_ip and gitea_http: - print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}") -gitea_ts = values.get("hosts.debian.tailscale_ip") -if gitea_ts: - print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}") -subdomains = values.get("domain.subdomains") -if subdomains: - if isinstance(subdomains, str): - subdomain_values = [item.strip() for item in subdomains.split(",") if item.strip()] - else: - subdomain_values = list(subdomains) - if subdomain_values: - print(f": ${{LAB_ADDITIONAL_SERVER_NAMES_JSON:={shlex.quote(json.dumps(subdomain_values))}}}") -PY -)" - - if [[ -n "${rendered_defaults}" ]]; then - eval "${rendered_defaults}" - fi -} - load_homelab_inventory_defaults -export_if_unset() { - local name="$1" - local value="$2" - - if [[ -z "${value}" ]]; then - return 0 - fi - if [[ -z "${!name:-}" ]]; then - printf -v "${name}" '%s' "${value}" - fi - # shellcheck disable=SC2163 - export "${name?}" -} - -export_homelab_inventory_tf_vars() { - export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}" - export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}" - export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}" - export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}" - export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}" - export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}" - export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}" - export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}" - export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}" - export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}" - export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}" - export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}" - export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}" - export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}" - export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}" - export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}" - export_if_unset TF_VAR_additional_server_names "${LAB_ADDITIONAL_SERVER_NAMES_JSON:-}" - export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" - export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" - export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" - export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" - export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}" - export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" - export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}" - export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}" - export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}" -} - export_homelab_inventory_tf_vars KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}" -require_debian_server() { - local command_name="$1" - local os_id="" - - if [[ "$(uname -s)" != "Linux" ]]; then - echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2 - exit 1 - fi - - if [[ -r /etc/os-release ]]; then - os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)" - fi - - if [[ "${os_id}" != "debian" ]]; then - echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2 - exit 1 - fi -} - -tofu_state_has_resource() { - local stack="$1" - local resource_address="$2" - - tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1 -} - -helm_release_secret_exists() { - local namespace="$1" - local release_name="$2" - local secret_name - - secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \ - -l "owner=helm,name=${release_name}" \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" - - [[ -n "${secret_name}" ]] -} - -kubernetes_resource_exists() { - local namespace="$1" - local resource_kind="$2" - local resource_name="$3" - - if [[ -n "${namespace}" ]]; then - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 - return $? - fi - - kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 -} - -adopt_tofu_helm_release() { - local stack="$1" - local resource_address="$2" - local namespace="$3" - local release_name="$4" - local has_state=false - local has_release_secret=false - - if tofu_state_has_resource "${stack}" "${resource_address}"; then - has_state=true - fi - if helm_release_secret_exists "${namespace}" "${release_name}"; then - has_release_secret=true - fi - - if [[ "${has_state}" == "true" && "${has_release_secret}" == "false" ]]; then - echo "Removing stale Helm release state for ${namespace}/${release_name} from ${stack} (${resource_address}) because no Helm release secret exists..." - tofu -chdir="${REPO_ROOT}/${stack}" state rm "${resource_address}" - return 0 - fi - - if [[ "${has_state}" == "true" ]]; then - return 0 - fi - if [[ "${has_release_secret}" == "false" ]]; then - return 0 - fi - - echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..." - tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}" -} - -adopt_tofu_kubernetes_resource() { - local stack="$1" - local resource_address="$2" - local namespace="$3" - local resource_kind="$4" - local resource_name="$5" - local import_id="$6" - - if tofu_state_has_resource "${stack}" "${resource_address}"; then - return 0 - fi - if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then - return 0 - fi - - echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..." - tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" -} - -adopt_tofu_kubernetes_manifest() { - local stack="$1" - local resource_address="$2" - local namespace="$3" - local kubectl_kind="$4" - local api_version="$5" - local manifest_kind="$6" - local resource_name="$7" - local import_id - - if tofu_state_has_resource "${stack}" "${resource_address}"; then - return 0 - fi - if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then - return 0 - fi - - import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}" - echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..." - tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" -} - adopt_platform_existing_resources() { local stack="bootstrap/platform" @@ -499,7207 +190,11 @@ delete_prometheus_stack_storage() { run_tofu_stack() { local stack="$1" - local auto_approve="${LAB_AUTO_APPROVE:-true}" - local -a apply_args=() + local auto_approve="${L - if [[ "${stack}" == "bootstrap/cluster" ]]; then - ensure_cluster_worker_var_file - fi - if [[ "${stack}" == "bootstrap/edge" ]]; then - ensure_edge_tailscale_routes - ensure_edge_haproxy_stats_password - fi +... [OUTPUT TRUNCATED - 267,570 chars omitted out of 317,495 total] ... - if truthy "${auto_approve}"; then - apply_args+=("-auto-approve") - elif ! disabled_value "${auto_approve}"; then - echo "LAB_AUTO_APPROVE must be true or false." >&2 - exit 1 - fi - - if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then - apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") - fi - - tofu -chdir="${REPO_ROOT}/${stack}" init - ensure_kubernetes_api_ready_for_tofu_stack "${stack}" - if [[ "${stack}" == "bootstrap/platform" ]]; then - adopt_platform_existing_resources - fi - if [[ "${stack}" == "bootstrap/apps" ]]; then - adopt_apps_existing_resources - fi - ensure_kubernetes_api_ready_for_tofu_stack "${stack}" - tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}" -} - -run_tofu_plan_stack() { - local stack="$1" - local -a plan_args=() - - if [[ "${stack}" == "bootstrap/cluster" ]]; then - ensure_cluster_worker_var_file - fi - - if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then - plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}") - fi - - tofu -chdir="${REPO_ROOT}/${stack}" init - tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}" -} - -backup_tofu_state() { - local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}" - local timestamp - local archive - local -a paths=() - local path - - timestamp="$(date +%Y%m%d-%H%M%S)" - archive="${backup_dir}/tofu-state-${timestamp}.tgz" - mkdir -p "${backup_dir}" - - for path in \ - bootstrap/provisioning/terraform.tfstate \ - bootstrap/provisioning/terraform.tfstate.backup \ - bootstrap/cluster/terraform.tfstate \ - bootstrap/cluster/terraform.tfstate.backup \ - bootstrap/platform/terraform.tfstate \ - bootstrap/platform/terraform.tfstate.backup \ - bootstrap/apps/terraform.tfstate \ - bootstrap/apps/terraform.tfstate.backup \ - bootstrap/edge/terraform.tfstate \ - bootstrap/edge/terraform.tfstate.backup \ - .lab/cluster-workers.auto.tfvars.json \ - .lab/pimox-workers.tsv \ - .lab/manual-workers.tsv; do - if [[ -e "${REPO_ROOT}/${path}" ]]; then - paths+=("${path}") - fi - done - - if ((${#paths[@]} == 0)); then - echo "No local OpenTofu state files found to back up." - return 0 - fi - - tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}" - chmod 0600 "${archive}" - echo "Backed up local OpenTofu state to ${archive}." -} - -move_prometheus_stack_workers() { - local stack="bootstrap/platform" - local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}" - local auto_approve="${LAB_AUTO_APPROVE:-true}" - local -a approve_args=() - - require_debian_server "move-prometheus-stack-workers" - - if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then - cat >&2 <<'EOF' -move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data. -Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended. -EOF - exit 1 - fi - - if truthy "${auto_approve}"; then - approve_args+=("-auto-approve") - elif ! disabled_value "${auto_approve}"; then - echo "LAB_AUTO_APPROVE must be true or false." >&2 - exit 1 - fi - - export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" - export KUBECONFIG="${TF_VAR_kubeconfig_path}" - - echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted." - ensure_homelab_node_labels - tofu -chdir="${REPO_ROOT}/${stack}" init - adopt_platform_existing_resources - tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}" - delete_prometheus_stack_storage "${namespace}" - tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}" - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide -} - -doctor_versions_report() { - local api_version - local api_minor - local containerd_major_count - local kubelet_minor_count - local node_rows - local status=0 - - require_debian_server "doctor-versions" - ensure_python3 - - api_version="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json | - python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' - )" - api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" - node_rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' - )" - - if [[ -z "${node_rows}" ]]; then - echo "No Kubernetes nodes found." >&2 - return 1 - fi - - printf 'API server: %s\n\n' "${api_version}" - printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME" - printf '%s\n' "${node_rows}" | - awk -F '\t' '{ - version = $2 - gsub(/^v/, "", version) - split(version, parts, ".") - printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3 - }' - - kubelet_minor_count="$( - printf '%s\n' "${node_rows}" | - awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | - sort -u | - wc -l | - tr -d ' ' - )" - containerd_major_count="$( - printf '%s\n' "${node_rows}" | - awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | - sort -u | - wc -l | - tr -d ' ' - )" - - if [[ "${kubelet_minor_count}" != "1" ]]; then - echo - echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2 - status=1 - fi - - if printf '%s\n' "${node_rows}" | - awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then - echo - echo "At least one kubelet does not match the API server minor (${api_minor})." >&2 - status=1 - fi - - if [[ "${containerd_major_count}" -gt 1 ]]; then - echo - echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2 - status=1 - fi - - if [[ "${status}" -eq 0 ]]; then - echo - echo "Node Kubernetes and containerd versions are aligned." - fi - - return "${status}" -} - -doctor_versions_compact() { - local api_version - local api_minor - local node_rows - local kubelet_minor_count - local containerd_major_count - local mismatch_count - local failures=0 - - ensure_python3 - report_ui_begin - - if ! status_kubernetes_api_ok; then - report_ui_emit fail "Kubernetes" "API" "not reachable" "kubectl readyz failed" "./jeannie start-cluster" - report_ui_render "Doctor Versions" - report_ui_cleanup - return 1 - fi - - api_version="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json | - python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])' - )" - api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')" - node_rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}' - )" - - if [[ -z "${node_rows}" ]]; then - report_ui_emit fail "Kubernetes" "Nodes" "none found" "kubectl returned no node rows" "./jeannie doctor-cluster" - report_ui_render "Doctor Versions" - report_ui_cleanup - return 1 - fi - - kubelet_minor_count="$( - printf '%s\n' "${node_rows}" | - awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' | - sort -u | - wc -l | - tr -d ' ' - )" - mismatch_count="$( - printf '%s\n' "${node_rows}" | - awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) count++ } END { print count + 0 }' - )" - containerd_major_count="$( - printf '%s\n' "${node_rows}" | - awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' | - sort -u | - wc -l | - tr -d ' ' - )" - - report_ui_emit ok "Kubernetes" "API version" "${api_version}" - if [[ "${kubelet_minor_count}" != "1" || "${mismatch_count}" != "0" ]]; then - report_ui_emit fail "Kubernetes" "Kubelet versions" "${mismatch_count} node(s) differ from ${api_minor}" "unique kubelet minors=${kubelet_minor_count}" "./jeannie doctor-versions --verbose" - failures=$((failures + 1)) - else - report_ui_emit ok "Kubernetes" "Kubelet versions" "aligned to ${api_minor}" - fi - if [[ "${containerd_major_count}" -gt 1 ]]; then - report_ui_emit warn "Runtime" "Containerd versions" "mixed major versions" "unique majors=${containerd_major_count}" "./jeannie doctor-versions --verbose" - else - report_ui_emit ok "Runtime" "Containerd versions" "single major version" - fi - - report_ui_render "Doctor Versions" - report_ui_cleanup - return "${failures}" -} - -doctor_versions() { - require_debian_server "doctor-versions" - if ! report_ui_parse_args "$@"; then - return 1 - fi - if [[ "${REPORT_UI_VERBOSE}" == "true" ]]; then - doctor_versions_report - exit "$?" - fi - doctor_versions_compact - exit "$?" -} - - - - - - - - - - - - -worker_index_is_skipped() { - local index="$1" - local skip_indexes="$2" - local skip_index - - skip_indexes="${skip_indexes//,/ }" - for skip_index in ${skip_indexes}; do - [[ -z "${skip_index}" ]] && continue - if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2 - exit 1 - fi - if ((skip_index == index)); then - return 0 - fi - done - - return 1 -} - -ensure_python3() { - if command -v python3 >/dev/null 2>&1; then - return 0 - fi - - sudo apt-get update - sudo apt-get install -y --no-install-recommends python3 -} - -ensure_go_toolchain() { - require_debian_server "go toolchain" - - if command -v go >/dev/null 2>&1; then - return 0 - fi - - echo "Installing Go toolchain for Jeannie core helpers..." - sudo apt-get update - sudo apt-get install -y --no-install-recommends golang-go - - if ! command -v go >/dev/null 2>&1; then - echo "go is still unavailable after installing golang-go." >&2 - exit 1 - fi -} - -detect_route_interface() { - local target="$1" - - ip route get "${target}" 2>/dev/null | awk ' - { - for (i = 1; i <= NF; i++) { - if ($i == "dev") { - print $(i + 1) - exit - } - } - } - ' -} - -pimox_ssh() { - local host="$1" - local user="$2" - local key_path="$3" - - shift 3 - ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@" -} - -pimox_guest_ipv4() { - local guest_json - local host="$1" - local user="$2" - local key_path="$3" - local vmid="$4" - local ip_prefix="$5" - local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" - - guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)" - if [[ -z "${guest_json}" ]]; then - return 1 - fi - - GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY' -import json -import os -import sys - -prefix = sys.argv[1] -try: - interfaces = json.loads(os.environ.get("GUEST_JSON", "")) -except Exception: - sys.exit(1) - -for iface in interfaces or []: - for address in iface.get("ip-addresses") or []: - if address.get("ip-address-type") != "ipv4": - continue - ip = address.get("ip-address", "") - if not ip or ip.startswith(("127.", "169.254.")): - continue - if prefix and not ip.startswith(prefix): - continue - print(ip) - sys.exit(0) -sys.exit(1) -PY -} - -pimox_worker_vm_debug() { - local host="$1" - local user="$2" - local key_path="$3" - local vmid="$4" - local qm_bin="$5" - local script - local encoded_script - - script="$(cat <<'EOF' -set +e -echo "-- addresses --" -ip -br addr -echo "-- routes --" -ip route -echo "-- units --" -systemctl is-active ssh sshd 2>/dev/null || true -echo "-- listeners --" -ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true -echo "-- sshd config test --" -sudo sshd -t 2>&1 || true -echo "-- recent ssh logs --" -journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true -EOF -)" - encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" - pimox_ssh "${host}" "${user}" "${key_path}" "set +e -echo 'Pimox VM ${vmid} status:' -sudo '${qm_bin}' status '${vmid}' -echo 'Pimox VM ${vmid} config summary:' -sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true -echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:' -sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true -echo 'Pimox VM ${vmid} guest SSH diagnostics:' -sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\" || true" >&2 || true -} - -pimox_worker_guest_agent_recovery_hint() { - local host="$1" - local user="$2" - local vmid="$3" - local qm_bin="$4" - - cat >&2 </dev/null | grep -q '^ssh[.]service'; then - ssh_unit=ssh.service -elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then - ssh_unit=sshd.service -else - echo "Neither ssh.service nor sshd.service exists in the guest" >&2 - exit 1 -fi - -sudo ssh-keygen -A -sudo install -d -m 0755 /run/sshd -sudo mkdir -p /etc/ssh/sshd_config.d -sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' -Port 22 -ListenAddress 0.0.0.0 -PubkeyAuthentication yes -PasswordAuthentication no -KbdInteractiveAuthentication no -PermitRootLogin no -UsePAM yes -SSHD_CONFIG - -sudo sshd -t -sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true -sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true -sudo systemctl enable "$ssh_unit" >/dev/null -sudo systemctl restart "$ssh_unit" -sleep 2 -sudo systemctl is-active "$ssh_unit" -if command -v ss >/dev/null 2>&1; then - ss -ltnp 2>/dev/null || ss -ltn - ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' -fi -EOF -)" - encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" - if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then - echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2 - printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 - return 1 - fi - - printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 - if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then - return 0 - fi - echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2 - if [[ -n "${repair_exitcode:-}" ]]; then - echo "Guest exit code: ${repair_exitcode}" >&2 - fi - return 1 -} - -wait_for_pimox_guest_ssh() { - local host="$1" - local user="$2" - local key_path="$3" - local vmid="$4" - local guest_user="$5" - local guest_key_path="$6" - local ip_prefix="$7" - local timeout_seconds="$8" - local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}" - local deadline - local elapsed - local guest_ip - local ip_filter_description - local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" - local last_guest_ip="" - local last_known_hosts_ip="" - local last_ssh_output="" - local last_host_probe_output="" - local host_probe_failures=0 - local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-false}" - local next_log - local ssh_deadline=0 - local ssh_output - local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}" - local ssh_refused_since=0 - local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}" - local ssh_repair_attempted=false - - ip_filter_description="matching prefix ${ip_prefix}" - if [[ -z "${ip_prefix}" ]]; then - ip_filter_description="that is not loopback or link-local" - fi - if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then - echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2 - return 1 - fi - if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then - echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2 - return 1 - fi - if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then - echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2 - return 1 - fi - mkdir -p "$(dirname "${known_hosts_file}")" - touch "${known_hosts_file}" - chmod 0600 "${known_hosts_file}" - - deadline=$((SECONDS + timeout_seconds)) - next_log="${SECONDS}" - while ((SECONDS < deadline)); do - guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)" - if [[ -n "${guest_ip}" ]]; then - if ((ssh_deadline == 0)); then - ssh_deadline=$((SECONDS + ssh_timeout_seconds)) - elif ((SECONDS >= ssh_deadline)); then - break - fi - last_guest_ip="${guest_ip}" - if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then - ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true - last_known_hosts_ip="${guest_ip}" - fi - if truthy "${require_host_probe}"; then - if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then - host_probe_failures=0 - else - host_probe_failures=$((host_probe_failures + 1)) - last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}" - if ((host_probe_failures >= 3)); then - echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2 - echo "Latest host probe: ${last_host_probe_output}" >&2 - echo "Fast checks:" >&2 - echo " ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2 - echo " arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2 - echo " Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2 - pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" - return 1 - fi - sleep 10 - continue - fi - fi - if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then - printf '%s\n' "${guest_ip}" - return 0 - fi - last_ssh_output="${ssh_output}" - if [[ "${ssh_output}" == *"Connection refused"* ]]; then - if ((ssh_refused_since == 0)); then - ssh_refused_since="${SECONDS}" - fi - if [[ "${ssh_repair_attempted}" == "false" ]]; then - echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2 - pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true - ssh_repair_attempted=true - elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then - echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2 - echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2 - pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" - return 1 - fi - else - ssh_refused_since=0 - fi - fi - - if ((SECONDS >= next_log)); then - elapsed=$((timeout_seconds - (deadline - SECONDS))) - if [[ -n "${last_guest_ip}" ]]; then - echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2 - if [[ -n "${last_ssh_output}" ]]; then - echo "Last SSH failure: ${last_ssh_output}" >&2 - fi - else - echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2 - fi - next_log=$((SECONDS + 60)) - fi - sleep 10 - done - - if [[ -n "${last_guest_ip}" ]]; then - echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2 - if [[ -n "${last_ssh_output}" ]]; then - echo "Last SSH failure: ${last_ssh_output}" >&2 - fi - if [[ -n "${last_host_probe_output}" ]]; then - echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2 - fi - else - echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2 - fi - pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" - - return 1 -} - -pimox_generated_mac() { - local vmid="$1" - - printf '02:68:10:%02x:%02x:%02x\n' \ - $(((vmid >> 16) & 255)) \ - $(((vmid >> 8) & 255)) \ - $((vmid & 255)) -} - -pimox_worker_static_ip() { - local index="$1" - local static_ips="$2" - local current_index=1 - local ip - - static_ips="${static_ips//,/ }" - for ip in ${static_ips}; do - if ((current_index == index)); then - printf '%s\n' "${ip}" - return 0 - fi - current_index=$((current_index + 1)) - done - - return 1 -} - -pimox_worker_net0_config() { - local vmid="$1" - local bridge="$2" - local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-deterministic}" - local mac - - case "${mac_mode}" in - auto) - printf 'virtio,bridge=%s\n' "${bridge}" - ;; - deterministic) - mac="$(pimox_generated_mac "${vmid}")" - printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}" - ;; - *) - echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2 - return 1 - ;; - esac -} - -validate_ipv4_cidr_or_host() { - local value="$1" - - [[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]] -} - -configure_pimox_worker_guest_network() { - local pimox_host="$1" - local pimox_user="$2" - local pimox_key="$3" - local vmid="$4" - local qm_bin="$5" - local worker_ip="$6" - local gateway="$7" - local dns_servers="$8" - local interface_name="$9" - local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}" - local deadline - local script - local encoded_script - local network_output - local network_exitcode - local worker_address - local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}" - local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-false}" - local lan_probe_block="" - - if ! validate_ipv4_cidr_or_host "${worker_ip}"; then - echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2 - return 1 - fi - if [[ "${worker_ip}" == */* && "${worker_ip}" != */24 ]]; then - echo "LAB_PIMOX_WORKER_STATIC_IPS currently supports host IPv4 values or /24 CIDRs, got '${worker_ip}'." >&2 - return 1 - fi - if [[ "${worker_ip}" != */* ]]; then - worker_ip="${worker_ip}/24" - fi - worker_address="${worker_ip%%/*}" - if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then - echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2 - return 1 - fi - if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then - echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2 - return 1 - fi - if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then - echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2 - return 1 - fi - if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then - echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2 - return 1 - fi - if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then - echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2 - return 1 - fi - - if truthy "${require_lan_probe}"; then - lan_probe_block="$(cat </dev/null 2>&1 || true -done -sudo pkill -x dhclient >/dev/null 2>&1 || true -sudo pkill -x dhcpcd >/dev/null 2>&1 || true -sudo systemctl enable networking >/dev/null 2>&1 || true -sudo ifdown --force ${interface_name} 2>/dev/null || true -sudo ip addr flush dev ${interface_name} scope global || true -sudo ip route flush dev ${interface_name} || true -sudo ip link set ${interface_name} up -sudo ip addr add ${worker_ip} dev ${interface_name} -sudo ip route replace default via ${gateway} dev ${interface_name} src ${worker_address} -printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null -sudo ifup --force ${interface_name} 2>/dev/null || true -sudo ssh-keygen -A -sudo install -d -m 0755 /run/sshd -sudo mkdir -p /etc/ssh/sshd_config.d -sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' -Port 22 -ListenAddress 0.0.0.0 -PubkeyAuthentication yes -PasswordAuthentication no -KbdInteractiveAuthentication no -PermitRootLogin no -UsePAM yes -SSHD_CONFIG -sudo sshd -t -sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true -sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true -ssh_unit=ssh.service -if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then - ssh_unit=sshd.service -fi -sudo systemctl enable "\$ssh_unit" >/dev/null -sudo systemctl restart "\$ssh_unit" -ip -br addr show ${interface_name} -ip route -ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/' -${lan_probe_block} -echo 'guest-check: ssh service' -sudo systemctl is-active "\$ssh_unit" -if command -v ss >/dev/null 2>&1; then - ss -ltnp 2>/dev/null || ss -ltn - ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' -fi -EOF -)" - encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" - - deadline=$((SECONDS + timeout_seconds)) - while ((SECONDS < deadline)); do - if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then - printf '%s\n' "${network_output}" | sed 's/^/ guest-network: /' - if network_exitcode="$(pimox_guest_exec_exitcode "${network_output}")" && [[ "${network_exitcode}" == "0" ]]; then - return 0 - fi - echo "Static guest network validation failed inside Pimox VM ${vmid}." >&2 - if [[ -n "${network_exitcode:-}" ]]; then - echo "Guest exit code: ${network_exitcode}" >&2 - fi - return 1 - fi - sleep 5 - done - - echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2 - if [[ -n "${network_output:-}" ]]; then - printf '%s\n' "${network_output}" | sed 's/^/ /' >&2 - fi - pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}" - return 1 -} - -configure_pimox_worker_forwarding() { - local pimox_host="$1" - local pimox_user="$2" - local pimox_key="$3" - local worker_ip="$4" - local enabled="${LAB_PIMOX_WORKER_CONFIGURE_FORWARDING:-true}" - local backup_dir="${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}" - local worker_address - - if disabled_value "${enabled}"; then - return 0 - fi - if ! truthy "${enabled}"; then - echo "LAB_PIMOX_WORKER_CONFIGURE_FORWARDING must be true or false." >&2 - return 1 - fi - if ! validate_ipv4_cidr_or_host "${worker_ip}"; then - echo "Invalid Pimox worker forwarding IP '${worker_ip}'." >&2 - return 1 - fi - worker_address="${worker_ip%%/*}" - - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\" -if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then - iptables_bin=/usr/sbin/iptables -fi -if [ -z \"\$iptables_bin\" ]; then - if command -v apt-get >/dev/null 2>&1; then - echo 'iptables not found on Pimox host; installing iptables for worker forwarding rules.' - sudo apt-get update - sudo apt-get install -y --no-install-recommends iptables - fi -fi -iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\" -if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then - iptables_bin=/usr/sbin/iptables -fi -if [ -z \"\$iptables_bin\" ]; then - echo 'iptables not found on Pimox host and could not be installed; cannot configure worker forwarding rules.' >&2 - exit 1 -fi -iptables_wait_seconds='${LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS:-60}' -case \"\$iptables_wait_seconds\" in - ''|*[!0-9]*) - echo \"LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS must be a non-negative integer, got '\$iptables_wait_seconds'.\" >&2 - exit 1 - ;; -esac -iptables_wait_args= -if \"\$iptables_bin\" -h 2>&1 | grep -q -- ' -w'; then - iptables_wait_args=\"-w \$iptables_wait_seconds\" -else - echo 'Pimox host iptables does not support -w; proceeding without xtables lock wait.' -fi -iptables_cmd() { - sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\" -} -iptables_save_bin=\"\$(command -v iptables-save 2>/dev/null || true)\" -if [ -z \"\$iptables_save_bin\" ] && [ -x /usr/sbin/iptables-save ]; then - iptables_save_bin=/usr/sbin/iptables-save -fi -backup_dir='${backup_dir}' -case \"\$backup_dir\" in - '~'|'~/') backup_dir=\"\$HOME\" ;; - '~/'*) backup_dir=\"\$HOME/\${backup_dir#~/}\" ;; -esac -mkdir -p \"\$backup_dir\" -backup_file=\"\$backup_dir/iptables-before-jeannie-workers-\$(date +%Y%m%d-%H%M%S).rules\" -if [ -n \"\$iptables_save_bin\" ]; then - sudo \"\$iptables_save_bin\" >\"\$backup_file\" - echo \"Saved Pimox iptables backup: \$backup_file\" -fi -sudo mkdir -p /etc/sysctl.d -printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-pimox-worker-forwarding.conf >/dev/null -sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null -chain=FORWARD -if iptables_cmd -S DOCKER-USER >/dev/null 2>&1; then - chain=DOCKER-USER -fi -if ! iptables_cmd -C \"\$chain\" -s '${worker_address}/32' -j ACCEPT 2>/dev/null; then - iptables_cmd -I \"\$chain\" 1 -s '${worker_address}/32' -j ACCEPT -fi -if ! iptables_cmd -C \"\$chain\" -d '${worker_address}/32' -j ACCEPT 2>/dev/null; then - iptables_cmd -I \"\$chain\" 1 -d '${worker_address}/32' -j ACCEPT -fi -echo \"Pimox worker forwarding rules active in \$chain for ${worker_address}\"" -} - -cpuset_cpu_count() { - local cpuset="$1" - local count=0 - local part - local start - local end - local -a parts - - IFS=',' read -r -a parts <<<"${cpuset}" - for part in "${parts[@]}"; do - if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then - start="${BASH_REMATCH[1]}" - end="${BASH_REMATCH[2]}" - if ((end < start)); then - return 1 - fi - count=$((count + end - start + 1)) - elif [[ "${part}" =~ ^[0-9]+$ ]]; then - count=$((count + 1)) - else - return 1 - fi - done - - printf '%s\n' "${count}" -} - -pimox_worker_cpu_affinity() { - local index="$1" - local affinities="$2" - local worker_cores="$3" - local affinity - local affinity_index=1 - local cpu_count - - for affinity in ${affinities}; do - if ((affinity_index == index)); then - if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then - echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 - exit 1 - fi - if ((cpu_count != worker_cores)); then - echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2 - exit 1 - fi - printf '%s\n' "${affinity}" - return 0 - fi - affinity_index=$((affinity_index + 1)) - done - - echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2 - exit 1 -} - -pimox_start_vm_with_cpuset() { - local host="$1" - local user="$2" - local key_path="$3" - local qm_bin="$4" - local vmid="$5" - local cpuset="$6" - - if [[ -z "${cpuset}" ]]; then - pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'" - return - fi - if ! cpuset_cpu_count "${cpuset}" >/dev/null; then - echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2 - exit 1 - fi - - # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated - # command under taskset so PSCI brings secondary guest CPUs online reliably. - pimox_ssh "${host}" "${user}" "${key_path}" "set -eu -vmid='${vmid}' -qm_bin='${qm_bin}' -cpuset='${cpuset}' - -if ! command -v taskset >/dev/null 2>&1; then - echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2 - exit 1 -fi -if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then - echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2 - exit 1 -fi - -start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\") -cleanup() { - rm -f \"\$start_script\" -} -trap cleanup 0 - -sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\" -if ! awk -v vmid=\"\$vmid\" ' - NR == 1 && (\$1 == "/usr/bin/kvm" || \$1 ~ /qemu-system-aarch64$/) { kvm = 1 } - \$1 == "-id" && \$2 == vmid { id = 1 } - END { exit (kvm && id) ? 0 : 1 } -' \"\$start_script\"; then - echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2 - exit 1 -fi -smp_count=\$(awk '\$1 == "-smp" { count++ } END { print count + 0 }' \"\$start_script\") -if [ \"\$smp_count\" -ne 1 ]; then - echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2 - exit 1 -fi - -sudo taskset -c \"\$cpuset\" bash \"\$start_script\"" -} - -pimox_shutdown_vm_gracefully() { - local host="$1" - local user="$2" - local key_path="$3" - local qm_bin="$4" - local vmid="$5" - - pimox_ssh "${host}" "${user}" "${key_path}" "set -eu -sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true -elapsed=0 -while [ \"\$elapsed\" -lt 300 ]; do - if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then - exit 0 - fi - sleep 5 - elapsed=\$((elapsed + 5)) -done -echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2 -exit 1" -} - -pimox_guest_cpu_count() { - local guest_ip="$1" - local guest_user="$2" - local guest_key_path="$3" - local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" - - ssh -i "${guest_key_path}" \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=accept-new \ - -o UserKnownHostsFile="${known_hosts_file}" \ - "${guest_user}@${guest_ip}" \ - 'nproc --all' -} - -ensure_pimox_worker_node() { - local index="$1" - local spec_file="$2" - local pimox_host="$3" - local pimox_user="$4" - local pimox_key="$5" - local template_vmid="$6" - local bridge="$7" - local worker_base_vmid="$8" - local worker_name_prefix="$9" - local worker_node_prefix="${10}" - local worker_key_prefix="${11}" - local worker_cores="${12}" - local worker_memory="${13}" - local worker_user="${14}" - local worker_key_path="${15}" - local ip_prefix="${16}" - local timeout_seconds="${17}" - local qm_bin="${18}" - local worker_storage="${19}" - local worker_replace_existing="${20}" - local worker_cpu_affinity="${21}" - local worker_kvm_cpuset="${22}" - local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" - local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" - local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" - local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" - local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" - local padded - local vmid - local worker_key - local worker_name - local node_name - local net0_config - local guest_ip - local guest_cpu_count - local static_ip - - printf -v padded '%02d' "${index}" - vmid=$((worker_base_vmid + index - 1)) - worker_key="${worker_key_prefix}${padded}" - worker_name="${worker_name_prefix}-${padded}" - node_name="${worker_node_prefix}-${padded}" - if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then - exit 1 - fi - if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then - if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then - echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2 - exit 1 - fi - if ! truthy "${worker_replace_existing}" && - ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then - echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}." - worker_replace_existing=true - fi - if truthy "${worker_replace_existing}"; then - echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..." - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true -elapsed=0 -while [ \"\$elapsed\" -lt 300 ]; do - if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then - break - fi - sleep 5 - elapsed=\$((elapsed + 5)) -done -sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" - else - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -was_running=false -had_legacy_args=false -if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then - was_running=true -fi -if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then - had_legacy_args=true -fi -sudo '${qm_bin}' set '${vmid}' \ - --agent enabled=1 \ - --bios ovmf \ - --boot 'order=scsi0;net0' \ - --cores '${worker_cores}' \ - --memory '${worker_memory}' \ - --net0 '${net0_config}' \ - --numa 0 \ - --ostype l26 \ - --scsihw virtio-scsi-pci \ - --sockets 1 \ - --vga virtio -if [ \"\$had_legacy_args\" = true ]; then - sudo '${qm_bin}' set '${vmid}' --delete args -fi -if [ -n '${worker_cpu_affinity}' ]; then - affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { - case \"\$affinity_output\" in - *'Unknown option: affinity'*) - echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' - ;; - *) - printf '%s\n' \"\$affinity_output\" >&2 - exit 1 - ;; - esac - } -fi -if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then - echo 'Restarting VM ${vmid} to apply removal of legacy qm args.' - sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true - elapsed=0 - while [ \"\$elapsed\" -lt 300 ]; do - if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then - break - fi - sleep 5 - elapsed=\$((elapsed + 5)) - done - if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then - echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2 - exit 1 - fi -fi" - fi - fi - - if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -if ! ip link show '${bridge}' >/dev/null 2>&1; then - echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2 - exit 1 -fi -pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" -if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then - pvesm_cmd=/usr/sbin/pvesm -fi -if [ -z \"\$pvesm_cmd\" ]; then - echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2 - exit 1 -fi -if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then - echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2 - exit 1 -fi -sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' -sudo '${qm_bin}' set '${vmid}' \ - --agent enabled=1 \ - --bios ovmf \ - --boot 'order=scsi0;net0' \ - --cores '${worker_cores}' \ - --memory '${worker_memory}' \ - --net0 '${net0_config}' \ - --numa 0 \ - --ostype l26 \ - --scsihw virtio-scsi-pci \ - --sockets 1 \ - --vga virtio -if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then - sudo '${qm_bin}' set '${vmid}' --delete args -fi -if [ -n '${worker_cpu_affinity}' ]; then - affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { - case \"\$affinity_output\" in - *'Unknown option: affinity'*) - echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' - ;; - *) - printf '%s\n' \"\$affinity_output\" >&2 - exit 1 - ;; - esac -} -fi -sudo '${qm_bin}' set '${vmid}' --onboot 1" - fi - - if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then - pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" - elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then - echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2 - exit 1 - fi - - case "${worker_network_mode}" in - static) - if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then - echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2 - exit 1 - fi - echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..." - configure_pimox_worker_forwarding \ - "${pimox_host}" \ - "${pimox_user}" \ - "${pimox_key}" \ - "${static_ip}" - if ! configure_pimox_worker_guest_network \ - "${pimox_host}" \ - "${pimox_user}" \ - "${pimox_key}" \ - "${vmid}" \ - "${qm_bin}" \ - "${static_ip}" \ - "${worker_gateway}" \ - "${worker_dns_servers}" \ - "${worker_interface}"; then - pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}" - exit 1 - fi - ;; - dhcp) - ;; - *) - echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2 - exit 1 - ;; - esac - - if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then - echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2 - exit 1 - fi - - guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" - if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then - echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2 - pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" - pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" - if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then - echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2 - exit 1 - fi - guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" - fi - if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then - echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2 - echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2 - exit 1 - fi - - printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" -} - -write_cluster_worker_var_file() { - local var_file="$1" - local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}' - local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}" - local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}" - shift - - LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \ - LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \ - LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \ - LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \ - LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \ - LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \ - LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \ - LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \ - python3 - "${var_file}" "$@" <<'PY' -import json -import os -from pathlib import Path -import sys - -var_file = sys.argv[1] -spec_files = sys.argv[2:] -nodes = {} -node_labels = {} - -try: - raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"]) - manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"]) - pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"]) -except json.JSONDecodeError as exc: - raise SystemExit(f"Invalid node label JSON: {exc}") from exc - -if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}: - nodes["raspberrypi"] = { - "host": os.environ["LAB_RASPBERRY_HOST"], - "user": os.environ["LAB_RASPBERRY_USER"], - "node_name": os.environ["LAB_RASPBERRY_NODE_NAME"], - "ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"], - } - node_labels["raspberrypi"] = raspberry_labels - -for spec_file in spec_files: - path = Path(spec_file) - if not path.exists() or path.stat().st_size == 0: - continue - labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels - with path.open(encoding="utf-8") as handle: - for line_number, line in enumerate(handle, start=1): - line = line.rstrip("\n") - if not line or line.startswith("#"): - continue - try: - key, host, user, node_name, ssh_key_path = line.split("\t") - except ValueError as exc: - raise SystemExit( - f"{spec_file}:{line_number}: expected tab-separated " - "key, host, user, node_name, ssh_key_path" - ) from exc - nodes[key] = { - "host": host, - "user": user, - "node_name": node_name, - "ssh_key_path": ssh_key_path, - } - node_labels[key] = labels - -with open(var_file, "w", encoding="utf-8") as handle: - json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2) - handle.write("\n") -PY -} - -prepare_cluster_worker_var_file() { - local include_raspberry_default="$1" - local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv" - local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" - local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - local include_manual="${LAB_INCLUDE_MANUAL_WORKERS:-false}" - local -a spec_files=() - - export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}" - - mkdir -p "${REPO_ROOT}/.lab" - ensure_static_pimox_worker_spec_file "${pimox_spec_file}" - if truthy "${include_manual}"; then - spec_files+=("${manual_spec_file}") - elif ! disabled_value "${include_manual}"; then - echo "LAB_INCLUDE_MANUAL_WORKERS must be true or false." >&2 - return 1 - fi - spec_files+=("${pimox_spec_file}") - write_cluster_worker_var_file "${var_file}" "${spec_files[@]}" - export LAB_CLUSTER_VAR_FILE="${var_file}" -} - -ensure_static_pimox_worker_spec_file() { - local spec_file="$1" - local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" - local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" - local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" - local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" - local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" - local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" - local index - local padded - local worker_key - local node_name - local static_ip - - if [[ -s "${spec_file}" || "${worker_network_mode}" != "static" ]]; then - return 0 - fi - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 - return 1 - fi - if ((worker_count == 0)); then - return 0 - fi - - mkdir -p "$(dirname "${spec_file}")" - : >"${spec_file}" - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - continue - fi - if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then - echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2 - return 1 - fi - printf -v padded '%02d' "${index}" - worker_key="${worker_key_prefix}${padded}" - node_name="${worker_node_prefix}-${padded}" - printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${static_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" - done -} - -cluster_worker_var_file_has_workers() { - local var_file="$1" - - [[ -s "${var_file}" ]] || return 1 - python3 - "${var_file}" <<'PY' -import json -import sys - -with open(sys.argv[1], encoding="utf-8") as handle: - document = json.load(handle) -sys.exit(0 if document.get("worker_nodes") else 1) -PY -} - -preflight_check() { - local description="$1" - local output - shift - - printf '%s %s... ' '-->' "${description}" - if output="$("$@" 2>&1)"; then - printf 'ok\n' - return 0 - fi - - printf 'failed\n' - if [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed 's/^/ /' >&2 - fi - return 1 -} - -preflight_warn() { - local description="$1" - local output - shift - - printf '%s %s... ' '-->' "${description}" - if output="$("$@" 2>&1)"; then - printf 'ok\n' - return 0 - fi - - printf 'warning\n' - if [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed 's/^/ /' >&2 - fi - return 0 -} - -check_gitea_reachable() { - local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" - local output - - if output="$(curl -fsS --max-time 10 "${gitea_url}" 2>&1 >/dev/null)"; then - return 0 - fi - - printf '%s\n' "${output}" >&2 - cat >&2 <<'EOF' -Gitea local HTTP failed. Run ./jeannie doctor-gitea --verbose. -If the local Gitea container is down or unhealthy, run ./jeannie deploy-gitea. -EOF - return 1 -} - -check_debian_docker_root() { - local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" - local actual_root - - if ! command -v docker >/dev/null 2>&1; then - echo "docker command was not found on Debian host" >&2 - return 1 - fi - actual_root="$(sudo docker info --format '{{.DockerRootDir}}')" - if [[ "${actual_root}" != "${expected_root}" ]]; then - echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2 - return 1 - fi -} - -fix_debian_docker_root() { - local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" - local daemon_file="/etc/docker/daemon.json" - local current_root="" - - require_debian_server "fix-debian-docker-root" - - if ! command -v docker >/dev/null 2>&1; then - echo "docker command was not found on Debian host" >&2 - exit 1 - fi - - current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)" - if [[ "${current_root}" == "${expected_root}" ]]; then - echo "Debian Docker root is already ${expected_root}." - return 0 - fi - - echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..." - sudo mkdir -p "${expected_root}" /etc/docker - - echo "Stopping Docker services..." - sudo systemctl stop docker 2>/dev/null || true - sudo systemctl stop containerd 2>/dev/null || true - - if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then - echo "Copying existing Docker data to ${expected_root}..." - sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/" - fi - - sudo python3 - "${daemon_file}" "${expected_root}" <<'PY' -import json -import os -import sys - -path, expected_root = sys.argv[1:3] -document = {} -if os.path.exists(path) and os.path.getsize(path) > 0: - with open(path, encoding="utf-8") as handle: - document = json.load(handle) -document["data-root"] = expected_root -with open(path, "w", encoding="utf-8") as handle: - json.dump(document, handle, indent=2, sort_keys=True) - handle.write("\n") -PY - - echo "Starting Docker services..." - sudo systemctl start containerd - sudo systemctl start docker - - check_debian_docker_root - echo "Debian Docker root is now ${expected_root}." -} - -check_debian_tailscale_ip() { - local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}" - - if [[ -z "${expected_ip}" ]]; then - return 0 - fi - if ! command -v tailscale >/dev/null 2>&1; then - echo "tailscale command was not found on Debian host" >&2 - return 1 - fi - if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then - echo "Debian Tailscale IP does not include ${expected_ip}" >&2 - tailscale ip -4 >&2 || true - return 1 - fi -} - -check_edge_ssh() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true" -} - -check_rpi_ssh() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true" -} - -check_rpi_docker_root_state() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" - local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu -actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" -if [ \"\$actual_root\" = '${nvme_root}' ]; then - mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; } - exit 0 -fi -if [ \"\$actual_root\" = '${fallback_root}' ]; then - echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2 - exit 0 -fi -echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2 -exit 1" -} - -check_rpi_tailscale_ip() { - local expected_ip="${LAB_RPI_TAILSCALE_IP:-}" - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - - if [[ -z "${expected_ip}" ]]; then - return 0 - fi - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'" -} - -check_pimox_storage() { - local mode="${LAB_PIMOX_PIPELINE:-true}" - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" - - if disabled_value "${mode}"; then - return 0 - fi - - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" -if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then - pvesm_cmd=/usr/sbin/pvesm -fi -if [ -z \"\$pvesm_cmd\" ]; then - echo 'pvesm was not found' >&2 - exit 1 -fi -sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'" -} - -check_local_disk_free_gib() { - local path="$1" - local min_gib="$2" - local available_kib - local min_kib - - available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')" - min_kib=$((min_gib * 1024 * 1024)) - if ((available_kib < min_kib)); then - echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2 - return 1 - fi -} - -check_systemd_active() { - local unit="$1" - - systemctl is-active --quiet "${unit}" -} - -check_rpi_docker_writable() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu -root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\" -sudo mkdir -p \"\$root/.homelab-check\" -sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\" -sudo rm -f \"\$root/.homelab-check/write-test\"" -} - -check_edge_disk_free() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}" - - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu -available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\" -min_kib=$((min_gib * 1024 * 1024)) -if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then - echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2 - exit 1 -fi" -} - -check_pihole_dns_query() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}" - local output - - if command -v dig >/dev/null 2>&1; then - output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || { - printf '%s\n' "${output}" >&2 - echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2 - return 1 - } - if [[ -z "${output}" ]]; then - echo "Pi-hole returned no A records for ${query_name}" >&2 - return 1 - fi - return 0 - fi - if command -v nslookup >/dev/null 2>&1; then - nslookup "${query_name}" "${rpi_host}" >/dev/null - return $? - fi - - echo "dig or nslookup is required to test Pi-hole DNS directly." >&2 - return 1 -} - -homelab_preflight() { - local phase="${1:-full}" - local failures=0 - - require_debian_server "preflight" - - if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then - echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}." - return 0 - fi - - case "${phase}" in - early | full) - ;; - *) - echo "Unsupported preflight phase '${phase}'." >&2 - exit 1 - ;; - esac - - echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..." - - preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1)) - preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1)) - preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1)) - if [[ "${phase}" == "full" ]]; then - preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1)) - preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1)) - fi - preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip - preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) - preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" check_edge_ssh || failures=$((failures + 1)) - - if ((failures > 0)); then - echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2 - exit 1 - fi - - echo "${phase^} preflight checks passed." -} - -doctor_preapply() { - local failures=0 - - require_debian_server "doctor-preapply" - - echo "Running pre-apply readiness checks..." - preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1)) - preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1)) - preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1)) - preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1)) - preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1)) - preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1)) - preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1)) - preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1)) - - if ((failures > 0)); then - echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2 - exit 1 - fi - - echo "Pre-apply doctor checks passed." -} - -inventory_check() { - "${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" -} - -run_pimox_pipeline() { - local mode="${LAB_PIMOX_PIPELINE:-true}" - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}" - local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}" - local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" - local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" - local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}" - local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" - local provisioning_interface - local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}" - local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" - local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" - local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}" - local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" - local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}" - local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}" - local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}" - local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" - local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" - local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" - local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" - local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}" - local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}" - local worker_wait_timeout_seconds="${LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS:-600}" - local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" - local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - local index - local readiness_output - local readiness_status - local worker_cpu_affinity - local cpuset_cpu_total - local cpuset_name - local cpuset_value - local required_cores - - if disabled_value "${mode}"; then - return 0 - fi - - if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then - mode="true" - fi - - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 - exit 1 - fi - if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2 - exit 1 - fi - if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2 - exit 1 - fi - if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then - echo "Pimox template and worker CPU and memory values must be greater than zero." >&2 - exit 1 - fi - for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do - cpuset_value="${!cpuset_name}" - if [[ -z "${cpuset_value}" ]]; then - continue - fi - if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then - echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2 - exit 1 - fi - required_cores="${template_cores}" - if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then - required_cores="${worker_cores}" - fi - if ((cpuset_cpu_total < required_cores)); then - echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2 - exit 1 - fi - done - if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then - echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2 - exit 1 - fi - if ! [[ "${worker_wait_timeout_seconds}" =~ ^[0-9]+$ && "${worker_wait_timeout_seconds}" -gt 0 ]]; then - echo "LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS must be a positive integer." >&2 - exit 1 - fi - if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then - echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2 - exit 1 - fi - if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then - echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2 - exit 1 - fi - if [[ "${worker_storage}" == "local" ]]; then - echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2 - exit 1 - fi - - set +e - readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then - echo 'qm was not found in PATH and ${qm_bin} is not executable' - exit 1 -fi -if ! ip link show '${bridge}' >/dev/null 2>&1; then - echo 'bridge ${bridge} was not found' - exit 1 -fi -if ! sudo -n true >/dev/null 2>&1; then - echo 'passwordless sudo is not available for ${pimox_user}' - exit 1 -fi -pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" -if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then - pvesm_cmd=/usr/sbin/pvesm -fi -if [ -z \"\$pvesm_cmd\" ]; then - echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' - exit 1 -fi -if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then - echo 'Pimox worker storage ${worker_storage} was not found or is not active' - exit 1 -fi" 2>&1)" - readiness_status=$? - set -e - if ((readiness_status != 0)); then - if [[ "${mode}" == "auto" ]]; then - echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready." - return 0 - fi - echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2 - exit 1 - fi - - ensure_python3 - provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}" - if [[ -z "${provisioning_interface}" ]]; then - echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2 - exit 1 - fi - - export TF_VAR_provisioning_interface="${provisioning_interface}" - export TF_VAR_pimox_host="${pimox_host}" - export TF_VAR_pimox_user="${pimox_user}" - export TF_VAR_pimox_ssh_key_path="${pimox_key}" - export TF_VAR_pimox_qm_bin="${qm_bin}" - export TF_VAR_pimox_template_bridge="${bridge}" - export TF_VAR_pimox_template_vmid="${template_vmid}" - export TF_VAR_pimox_template_name="${template_name}" - export TF_VAR_pimox_template_cores="${template_cores}" - export TF_VAR_pimox_template_memory="${template_memory}" - export TF_VAR_pimox_template_replace_existing="${template_replace_existing}" - export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}" - export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}" - export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}" - export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}" - export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}" - export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}" - - echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..." - run_tofu_stack "bootstrap/provisioning" - - if ((worker_count == 0)); then - return 0 - fi - - if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then - echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2 - exit 1 - fi - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1" - echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage." - - mkdir -p "${REPO_ROOT}/.lab" - : >"${spec_file}" - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." - continue - fi - - worker_cpu_affinity="" - if [[ -n "${worker_cpu_affinities}" ]]; then - worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" - fi - ensure_pimox_worker_node \ - "${index}" \ - "${spec_file}" \ - "${pimox_host}" \ - "${pimox_user}" \ - "${pimox_key}" \ - "${template_vmid}" \ - "${bridge}" \ - "${worker_base_vmid}" \ - "${worker_name_prefix}" \ - "${worker_node_prefix}" \ - "${worker_key_prefix}" \ - "${worker_cores}" \ - "${worker_memory}" \ - "${worker_user}" \ - "${worker_key_path}" \ - "${ip_prefix}" \ - "${worker_wait_timeout_seconds}" \ - "${qm_bin}" \ - "${worker_storage}" \ - "${worker_replace_existing}" \ - "${worker_cpu_affinity}" \ - "${worker_kvm_cpuset}" - done - - write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}" - export LAB_CLUSTER_VAR_FILE="${var_file}" -} - -run_openwrt_pipeline() { - local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}" - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local vmid="${LAB_OPENWRT_VMID:-9100}" - local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}" - local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}" - local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}" - local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}" - local cores="${LAB_OPENWRT_CORES:-2}" - local memory="${LAB_OPENWRT_MEMORY:-512}" - local version="${LAB_OPENWRT_VERSION:-24.10.6}" - local image_url="${LAB_OPENWRT_IMAGE_URL:-}" - local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}" - local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}" - local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}" - local start_vm="${LAB_OPENWRT_START:-false}" - local onboot="${LAB_OPENWRT_ONBOOT:-false}" - local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}" - local root_key_b64="" - local lan_dhcp_ignore="1" - local start_vm_flag="false" - local onboot_flag="0" - - if disabled_value "${mode}"; then - return 0 - fi - if ! truthy "${mode}"; then - echo "LAB_OPENWRT_VM must be true or false." >&2 - exit 1 - fi - - if [[ -z "${image_url}" ]]; then - image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz" - fi - - if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then - echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2 - exit 1 - fi - for value_name in storage wan_bridge lan_bridge vm_name; do - local value="${!value_name}" - if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then - echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2 - exit 1 - fi - done - if [[ "${storage}" == "local" ]]; then - echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2 - exit 1 - fi - if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then - echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2 - exit 1 - fi - if truthy "${lan_dhcp_enabled}"; then - lan_dhcp_ignore="0" - fi - if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then - echo "LAB_OPENWRT_START must be true or false." >&2 - exit 1 - fi - if truthy "${start_vm}"; then - start_vm_flag="true" - fi - if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then - echo "LAB_OPENWRT_ONBOOT must be true or false." >&2 - exit 1 - fi - if truthy "${onboot}"; then - onboot_flag="1" - fi - if [[ -r "${root_key_path}" ]]; then - root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')" - fi - - echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..." - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <&2 - exit 1 -fi - -pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)" -if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then - pvesm_cmd=/usr/sbin/pvesm -fi -if [ -z "\$pvesm_cmd" ]; then - echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2 - exit 1 -fi - -if ! sudo -n true >/dev/null 2>&1; then - echo "passwordless sudo is required for OpenWrt VM automation" >&2 - exit 1 -fi -if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then - echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2 - exit 1 -fi -if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then - echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2 - exit 1 -fi -if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then - echo "Pimox storage \$storage was not found." >&2 - exit 1 -fi - -if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then - if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then - echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2 - exit 1 - fi - sudo "\$qm_cmd" set "\$vmid" \\ - --net0 "virtio,bridge=\$wan_bridge" \\ - --net1 "virtio,bridge=\$lan_bridge" \\ - --cores "\$cores" \\ - --memory "\$memory" \\ - --onboot "\$onboot" - if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then - sudo "\$qm_cmd" start "\$vmid" - fi - exit 0 -fi - -for required_cmd in curl gzip losetup mount umount awk sed; do - if ! command -v "\$required_cmd" >/dev/null 2>&1; then - echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2 - exit 1 - fi -done - -tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)" -mnt_dir="\$tmp_dir/root" -loopdev="" -cleanup() { - if mountpoint -q "\$mnt_dir" 2>/dev/null; then - sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true - fi - if [ -n "\$loopdev" ]; then - sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true - fi - rm -rf "\$tmp_dir" -} -trap cleanup EXIT - -mkdir -p "\$mnt_dir" -curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz" -gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img" - -loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")" -root_part="\${loopdev}p2" -if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then - root_part="\${loopdev}p2" -fi -if [ ! -b "\$root_part" ]; then - echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2 - exit 1 -fi - -sudo mount "\$root_part" "\$mnt_dir" -sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh" - -cat >"\$tmp_dir/network" <"\$tmp_dir/dhcp" <"\$tmp_dir/firewall" <<'FIREWALL' -config defaults - option input 'REJECT' - option output 'ACCEPT' - option forward 'REJECT' - option synflood_protect '1' - -config zone - option name 'lan' - list network 'lan' - option input 'ACCEPT' - option output 'ACCEPT' - option forward 'ACCEPT' - -config zone - option name 'wan' - list network 'wan' - option input 'REJECT' - option output 'ACCEPT' - option forward 'REJECT' - option masq '1' - option mtu_fix '1' - -config forwarding - option src 'lan' - option dest 'wan' - -config rule - option name 'Allow-DHCP-Renew' - option src 'wan' - option proto 'udp' - option dest_port '68' - option target 'ACCEPT' - option family 'ipv4' - -config rule - option name 'Allow-Ping' - option src 'wan' - option proto 'icmp' - option icmp_type 'echo-request' - option family 'ipv4' - option target 'ACCEPT' -FIREWALL - -cat >"\$tmp_dir/system" <"\$tmp_dir/authorized_keys" - sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys" - sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" - sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" -fi -sync -sudo umount "\$mnt_dir" -sudo losetup -d "\$loopdev" -loopdev="" - -sudo "\$qm_cmd" create "\$vmid" \\ - --name "\$vm_name" \\ - --bios ovmf \\ - --cores "\$cores" \\ - --memory "\$memory" \\ - --net0 "virtio,bridge=\$wan_bridge" \\ - --net1 "virtio,bridge=\$lan_bridge" \\ - --numa 0 \\ - --ostype l26 \\ - --scsihw virtio-scsi-pci \\ - --sockets 1 \\ - --vga virtio \\ - --onboot "\$onboot" - -sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0" -sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null -disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')" -if [ -z "\$disk_volume" ]; then - echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2 - exit 1 -fi -sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume" -sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0" - -if [ "\$start_vm" = "true" ]; then - sudo "\$qm_cmd" start "\$vmid" -fi -EOF -} - -openwrt() { - require_debian_server "openwrt" - - LAB_OPENWRT_VM=true run_openwrt_pipeline -} - -cleanup_calico_links() { - ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true - sudo ip link delete vxlan.calico 2>/dev/null || true - sudo ip link delete tunl0 2>/dev/null || true - sudo ip link delete cni0 2>/dev/null || true - sudo ip link delete kube-ipvs0 2>/dev/null || true - ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true -} - -cleanup_iptables() { - sudo iptables -F || true - sudo iptables -X || true - sudo iptables -t nat -F || true - sudo iptables -t nat -X || true - sudo iptables -t mangle -F || true - sudo iptables -t mangle -X || true - sudo iptables -t raw -F || true - sudo iptables -t raw -X || true - if command -v ipvsadm >/dev/null 2>&1; then - sudo ipvsadm --clear || true - fi -} - -cleanup_calico_runtime_files() { - local path - - for path in /run/calico /var/run/calico; do - if sudo test -e "${path}"; then - sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true - sudo rmdir "${path}" 2>/dev/null || true - fi - done -} - -restore_node_dns() { - sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf - if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then - sudo rm -f /etc/resolv.conf - sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf - fi - sudo systemctl restart systemd-resolved 2>/dev/null || true -} - -cleanup_mounts() { - if command -v findmnt >/dev/null 2>&1; then - local mount_root - while IFS= read -r mountpoint; do - sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true - done < <( - for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do - findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true - done | sort -ru - ) - fi - while IFS= read -r mountpoint; do - sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true - done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) - sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true -} - -cleanup_node() { - sudo kubeadm reset --force || true - sudo systemctl stop kubelet 2>/dev/null || true - sudo systemctl stop containerd 2>/dev/null || true - sudo killall containerd-shim-runc-v2 2>/dev/null || true - - cleanup_mounts - - sudo rm -rf \ - /etc/kubernetes/ \ - /var/lib/etcd/ \ - /var/lib/kubelet/ \ - /var/lib/cni/ \ - /etc/cni/net.d \ - /run/flannel \ - /var/lib/calico \ - /var/log/calico \ - /var/lib/containerd/* \ - /run/containerd/* \ - /etc/containerd/certs.d \ - /etc/containerd/config.toml - cleanup_calico_runtime_files - sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam - - cleanup_iptables - cleanup_calico_links - restore_node_dns - - sudo mkdir -p /etc/containerd/certs.d - sudo systemctl reset-failed kubelet containerd 2>/dev/null || true - sudo systemctl start containerd 2>/dev/null || true -} - -image_ref_tag() { - local image_ref="$1" - local tag - - tag="${image_ref##*:}" - if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then - echo "Image reference ${image_ref} must include an immutable tag." >&2 - exit 1 - fi - - printf '%s\n' "${tag}" -} - -website_image_tag() { - local source_hash="$1" - - printf 'src-%s\n' "${source_hash:0:12}" -} - -apps_registry_endpoint() { - if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then - printf '%s\n' "${TF_VAR_registry_endpoint}" - return 0 - fi - - demos_registry_endpoint -} - -demos_registry_endpoint() { - local image - - image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")" - if [[ -z "${image}" || "${image}" != */* ]]; then - echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2 - exit 1 - fi - - printf '%s\n' "${image%%/*}" -} - -website_source_hash() { - ( - cd "${REPO_ROOT}" - find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' - ) -} - -demos_source_hash() { - ( - cd "${REPO_ROOT}" - find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}' - ) -} - -registry_image_exists() { - local registry_endpoint="$1" - local repository="$2" - local tag="$3" - local accept_header - - if ! command -v curl >/dev/null 2>&1; then - return 1 - fi - - accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json" - curl -fsS \ - -H "Accept: ${accept_header}" \ - "http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null -} - -image_state_value() { - local state_file="$1" - local key="$2" - - awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true -} - -website_image_is_current() { - local state_file="$1" - local source_hash="$2" - local platforms="$3" - local image_ref="$4" - local registry_endpoint="$5" - local image_tag - local saved_hash - local saved_platforms - local saved_image - - [[ -f "${state_file}" ]] || return 1 - - saved_hash="$(image_state_value "${state_file}" source_hash)" - saved_platforms="$(image_state_value "${state_file}" platforms)" - saved_image="$(image_state_value "${state_file}" image)" - - [[ "${saved_hash}" == "${source_hash}" ]] || return 1 - [[ "${saved_platforms}" == "${platforms}" ]] || return 1 - [[ "${saved_image}" == "${image_ref}" ]] || return 1 - - image_tag="$(image_ref_tag "${image_ref}")" - registry_image_exists "${registry_endpoint}" php-website "${image_tag}" -} - -ensure_website_image_tag_not_reused() { - local state_file="$1" - local source_hash="$2" - local image_ref="$3" - local saved_hash - local saved_image - - [[ -f "${state_file}" ]] || return 0 - - saved_hash="$(image_state_value "${state_file}" source_hash)" - saved_image="$(image_state_value "${state_file}" image)" - - if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then - echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2 - echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2 - exit 1 - fi -} - -demos_image_is_current() { - local state_file="$1" - local source_hash="$2" - local platforms="$3" - local image_ref="$4" - local registry_endpoint="$5" - local saved_hash - local saved_platforms - local saved_image - - [[ -f "${state_file}" ]] || return 1 - - saved_hash="$(image_state_value "${state_file}" source_hash)" - saved_platforms="$(image_state_value "${state_file}" platforms)" - saved_image="$(image_state_value "${state_file}" image)" - - [[ "${saved_hash}" == "${source_hash}" ]] || return 1 - [[ "${saved_platforms}" == "${platforms}" ]] || return 1 - [[ "${saved_image}" == "${image_ref}" ]] || return 1 - - registry_image_exists "${registry_endpoint}" demos-static latest -} - -write_website_image_state() { - local state_file="$1" - local source_hash="$2" - local platforms="$3" - local image_ref="$4" - - mkdir -p "$(dirname "${state_file}")" - { - printf 'source_hash=%s\n' "${source_hash}" - printf 'platforms=%s\n' "${platforms}" - printf 'image=%s\n' "${image_ref}" - } > "${state_file}" -} - -write_demos_image_state() { - local state_file="$1" - local source_hash="$2" - local platforms="$3" - local image_ref="$4" - - mkdir -p "$(dirname "${state_file}")" - { - printf 'source_hash=%s\n' "${source_hash}" - printf 'platforms=%s\n' "${platforms}" - printf 'image=%s\n' "${image_ref}" - } > "${state_file}" -} - -path_available_mb() { - local path="$1" - - while [[ ! -e "${path}" && "${path}" != "/" ]]; do - path="$(dirname "${path}")" - done - - df -Pm "${path}" | awk 'NR == 2 {print $4}' -} - -docker_root_dir() { - docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n' -} - -prune_unused_docker_build_data() { - docker buildx rm lab-builder 2>/dev/null || true - docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true - docker builder prune -af 2>/dev/null || true - docker system prune -af 2>/dev/null || true -} - -ensure_docker_build_space() { - local docker_root - local free_mb - local min_free_mb - - min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}" - docker_root="$(docker_root_dir)" - free_mb="$(path_available_mb "${docker_root}")" - - if (( free_mb >= min_free_mb )); then - return 0 - fi - - echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..." - prune_unused_docker_build_data - free_mb="$(path_available_mb "${docker_root}")" - - if (( free_mb < min_free_mb )); then - echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2 - echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2 - echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2 - exit 1 - fi -} - -prepare_buildx_builder() { - local registry_endpoint="$1" - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - cat < "${BUILDX_CONFIG}" -[registry."${registry_endpoint}"] - http = true - insecure = true -[registry."127.0.0.1:30500"] - http = true - insecure = true -[registry."localhost:30500"] - http = true - insecure = true -EOF - - docker buildx rm lab-builder 2>/dev/null || true - docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use - docker buildx inspect --bootstrap -} - -ensure_cosign_available() { - local arch - local cosign_url - local target_dir - local target_path - local tmp_path - - if [[ -n "${COSIGN_BIN}" ]]; then - if [[ -x "${COSIGN_BIN}" ]]; then - return 0 - fi - echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2 - exit 1 - fi - - if command -v cosign >/dev/null 2>&1; then - COSIGN_BIN="$(command -v cosign)" - return 0 - fi - - case "$(uname -m)" in - x86_64 | amd64) - arch="amd64" - ;; - aarch64 | arm64) - arch="arm64" - ;; - *) - echo "Unsupported Cosign install architecture: $(uname -m)" >&2 - exit 1 - ;; - esac - - target_dir="${HOMELAB_STATE_DIR}/bin" - target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}" - if [[ -x "${target_path}" ]]; then - COSIGN_BIN="${target_path}" - return 0 - fi - - if ! command -v curl >/dev/null 2>&1; then - echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2 - exit 1 - fi - - mkdir -p "${target_dir}" - tmp_path="${target_path}.tmp" - cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}" - echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..." - curl -fsSL "${cosign_url}" -o "${tmp_path}" - chmod 0755 "${tmp_path}" - mv "${tmp_path}" "${target_path}" - COSIGN_BIN="${target_path}" -} - -ensure_cosign_password() { - if [[ -n "${COSIGN_PASSWORD:-}" ]]; then - return 0 - fi - - if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then - COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" - export COSIGN_PASSWORD - return 0 - fi - - mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")" - if command -v openssl >/dev/null 2>&1; then - openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}" - elif command -v python3 >/dev/null 2>&1; then - python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}" -import secrets - -print(secrets.token_hex(32)) -PY - else - echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2 - exit 1 - fi - chmod 0600 "${COSIGN_PASSWORD_FILE}" - COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")" - export COSIGN_PASSWORD -} - -run_cosign() { - ensure_cosign_available - ensure_cosign_password - - COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@" -} - -ensure_cosign_keypair() { - ensure_cosign_available - ensure_cosign_password - - if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then - return 0 - fi - - if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then - echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2 - exit 1 - fi - - mkdir -p "$(dirname "${COSIGN_KEY_PATH}")" - echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..." - COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}" - chmod 0600 "${COSIGN_KEY_PATH}" - chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}" -} - -publish_cosign_public_key_config_map() { - kubectl --kubeconfig "${KUBECONFIG}" create namespace kyverno --dry-run=client -o yaml | - kubectl --kubeconfig "${KUBECONFIG}" apply -f - - - kubectl --kubeconfig "${KUBECONFIG}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \ - --from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \ - --dry-run=client -o yaml | - kubectl --kubeconfig "${KUBECONFIG}" apply -f - -} - -cosign_registry_flags() { - local registry_endpoint="$1" - local image_ref="$2" - - if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then - printf '%s\n' "--allow-http-registry" - fi -} - -image_supply_chain_metadata_exists() { - local image_ref="$1" - local registry_endpoint="$2" - local digest_ref - local -a registry_flags=() - - if ! digest_ref="$(image_digest_ref "${image_ref}")"; then - return 1 - fi - - mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") - - run_cosign verify \ - --key "${COSIGN_PUBLIC_KEY_PATH}" \ - --insecure-ignore-tlog=true \ - "${registry_flags[@]}" \ - "${digest_ref}" >/dev/null 2>&1 && - run_cosign verify-attestation \ - --key "${COSIGN_PUBLIC_KEY_PATH}" \ - --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ - --insecure-ignore-tlog=true \ - "${registry_flags[@]}" \ - "${digest_ref}" >/dev/null 2>&1 -} - -write_image_sbom_predicate() { - local image_ref="$1" - local output_file="$2" - local source_hash="$3" - - docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}" - python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY' -from datetime import datetime, timezone -import json -import re -import sys - -path = sys.argv[1] -image_ref = sys.argv[2] -source_hash = sys.argv[3] - -try: - with open(path, encoding="utf-8") as handle: - document = json.load(handle) -except json.JSONDecodeError: - document = None - -if not isinstance(document, dict): - safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-") - document = { - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": f"homelab image SBOM for {image_ref}", - "documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}", - "creationInfo": { - "created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "creators": ["Tool: jeannie"], - }, - "packages": [ - { - "name": image_ref, - "SPDXID": "SPDXRef-Image", - "downloadLocation": "NOASSERTION", - "filesAnalyzed": False, - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", - "copyrightText": "NOASSERTION", - } - ], - "relationships": [ - { - "spdxElementId": "SPDXRef-DOCUMENT", - "relationshipType": "DESCRIBES", - "relatedSpdxElement": "SPDXRef-Image", - } - ], - } - with open(path, "w", encoding="utf-8") as handle: - json.dump(document, handle, sort_keys=True) - handle.write("\n") - -if not isinstance(document, dict): - raise SystemExit("image SBOM predicate is not a JSON object") -if document.get("SPDXID") != "SPDXRef-DOCUMENT": - raise SystemExit("image SBOM predicate is not an SPDX document") -if not document.get("packages"): - raise SystemExit("image SBOM predicate does not include packages") -PY -} - -image_digest_ref() { - local image_ref="$1" - local digest - local repository - - if [[ "${image_ref}" == *@sha256:* ]]; then - printf '%s\n' "${image_ref}" - return 0 - fi - - digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')" - if [[ -z "${digest}" || "${digest}" == "" ]]; then - echo "Unable to resolve image digest for ${image_ref}." >&2 - return 1 - fi - - if [[ "${image_ref##*/}" == *:* ]]; then - repository="${image_ref%:*}" - else - repository="${image_ref}" - fi - printf '%s@%s\n' "${repository}" "${digest}" -} - -publish_image_supply_chain_metadata() { - local image_ref="$1" - local registry_endpoint="$2" - local source_hash="$3" - local digest_ref - local sbom_file - local -a registry_flags=() - - if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then - echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM." - return 0 - fi - - digest_ref="$(image_digest_ref "${image_ref}")" - mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}") - sbom_file="$(mktemp)" - write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}" - - echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..." - run_cosign sign \ - --yes \ - --key "${COSIGN_KEY_PATH}" \ - --tlog-upload=false \ - -a "homelab.dev/source-hash=${source_hash}" \ - "${registry_flags[@]}" \ - "${digest_ref}" - run_cosign attest \ - --yes \ - --key "${COSIGN_KEY_PATH}" \ - --predicate "${sbom_file}" \ - --type "${HOMELAB_SBOM_PREDICATE_TYPE}" \ - --tlog-upload=false \ - "${registry_flags[@]}" \ - "${digest_ref}" - rm -f "${sbom_file}" - - if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then - echo "Cosign verification failed after signing ${digest_ref}." >&2 - exit 1 - fi -} - -dump_argocd_debug() { - local app="$1" - - kubectl --kubeconfig "${KUBECONFIG}" -n argocd get application "${app}" -o yaml || true - kubectl --kubeconfig "${KUBECONFIG}" -n argocd describe application "${app}" || true - kubectl --kubeconfig "${KUBECONFIG}" -n argocd get pods -o wide || true - kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs deployment/argocd-repo-server --tail=120 || true - kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true -} - -dump_namespace_debug() { - local namespace="$1" - - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get all -o wide || true - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pvc -o wide || true - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe pods || true - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true -} - -wait_for_namespace() { - local namespace="$1" - local app="$2" - local timeout_seconds="$3" - local elapsed=0 - - until kubectl --kubeconfig "${KUBECONFIG}" get namespace "${namespace}" >/dev/null 2>&1; do - if ((elapsed >= timeout_seconds)); then - echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2 - dump_argocd_debug "${app}" - exit 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} - -wait_for_namespaced_resource() { - local namespace="$1" - local kind="$2" - local name="$3" - local app="$4" - local timeout_seconds="$5" - local elapsed=0 - - until kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do - if ((elapsed >= timeout_seconds)); then - echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2 - dump_argocd_debug "${app}" - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true - exit 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} - -wait_for_cluster_resource() { - local kind="$1" - local name="$2" - local app="$3" - local timeout_seconds="$4" - local elapsed=0 - local api_resource - - api_resource="${kind%%/*}" - if ! kubectl --kubeconfig "${KUBECONFIG}" get crd "${api_resource}" >/dev/null 2>&1; then - echo "Required CRD ${api_resource} is not installed; cannot wait for ${kind}/${name} from Argo CD app ${app}." >&2 - kubectl --kubeconfig "${KUBECONFIG}" -n argocd get application "${app}" \ - -o jsonpath='sync={.status.sync.status} health={.status.health.status} message={.status.operationState.message}{"\n"}' 2>/dev/null || true - dump_argocd_debug "${app}" - exit 1 - fi - - until kubectl --kubeconfig "${KUBECONFIG}" get "${kind}/${name}" >/dev/null 2>&1; do - if ((elapsed >= timeout_seconds)); then - echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2 - dump_argocd_debug "${app}" - exit 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} - -wait_for_deployment_ready() { - local namespace="$1" - local deployment="$2" - local app="$3" - local timeout_seconds="$4" - local desired_replicas - local ready_replicas - local elapsed=0 - - desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)" - desired_replicas="${desired_replicas:-1}" - - until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \ - (( ${ready_replicas:-0} >= desired_replicas )); do - if ((elapsed >= timeout_seconds)); then - echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2 - dump_argocd_debug "${app}" - dump_namespace_debug "${namespace}" - exit 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} - -deploy_gitea() { - local mode="${LAB_GITEA_DEPLOY:-true}" - local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}" - local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}" - local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}" - local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}" - local http_port="${LAB_GITEA_HTTP_PORT:-3000}" - local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" - local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}}" - local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" - local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" - local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml" - local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}" - - require_debian_server "deploy-gitea" - - if disabled_value "${mode}"; then - install_gitea_backup_timer - return 0 - fi - - if [[ ! -s "${compose_file}" ]]; then - echo "Missing ${compose_file}" >&2 - exit 1 - fi - - echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..." - - ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea" - scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml" - - ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu -install_dir='${install_dir}' -install_docker='${install_docker}' - -install_missing_packages() { - missing_packages='' - for package in \"\$@\"; do - if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then - missing_packages=\"\$missing_packages \$package\" - fi - done - if [ -n \"\$missing_packages\" ]; then - sudo apt-get update - sudo apt-get install -y --no-install-recommends \$missing_packages - fi -} - -install_missing_packages ca-certificates curl iptables - -if ! command -v docker >/dev/null 2>&1; then - if [ \"\$install_docker\" = 'true' ]; then - curl -fsSL https://get.docker.com | sudo sh - else - echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2 - exit 1 - fi -fi - -if ! sudo docker compose version >/dev/null 2>&1; then - install_missing_packages docker-compose-plugin -fi - -repair_docker_iptables() { - if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then - return 0 - fi - - echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...' - sudo systemctl restart docker - sleep 3 - - if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then - return 0 - fi - - echo 'Docker NAT chain is still missing after restarting Docker.' >&2 - sudo iptables -t nat -S >&2 || true - sudo systemctl status docker --no-pager -l >&2 || true - exit 1 -} - -repair_docker_iptables - -sudo mkdir -p \"\$install_dir/data\" -sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\" -sudo chown -R 1000:1000 \"\$install_dir/data\" -sudo tee \"\$install_dir/.env\" >/dev/null </dev/null | grep -q "install ok installed"; then - missing_packages+=("${package}") - fi - done - - if ((${#missing_packages[@]})); then - sudo apt-get update - sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" - fi -} - -ensure_local_docker_compose() { - local command_name="$1" - local install_docker="$2" - - install_missing_debian_packages ca-certificates curl iptables - - if ! command -v docker >/dev/null 2>&1; then - if [[ "${install_docker}" == "true" ]]; then - curl -fsSL https://get.docker.com | sudo sh - else - echo "Docker is not installed on the Debian host. Install Docker through host bootstrap first, or rerun with LAB_${command_name}_INSTALL_DOCKER=true to allow get.docker.com installation." >&2 - exit 1 - fi - fi - - if ! sudo docker compose version >/dev/null 2>&1; then - install_missing_debian_packages docker-compose-plugin - fi -} - -repair_local_docker_iptables() { - if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then - return 0 - fi - - echo "Docker NAT chain is missing on the Debian host; restarting Docker once to restore iptables state..." - sudo systemctl restart docker - sleep 3 - - if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then - return 0 - fi - - echo "Docker NAT chain is still missing after restarting Docker." >&2 - sudo iptables -t nat -S >&2 || true - sudo systemctl status docker --no-pager -l >&2 || true - exit 1 -} - -deploy_heimdall() { - local mode="${LAB_HEIMDALL_DEPLOY:-true}" - local install_dir="${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" - local data_dir="${LAB_HEIMDALL_DATA_DIR:-${install_dir}/data}" - local config_dir="${LAB_HEIMDALL_CONFIG_DIR:-${data_dir}/config}" - local http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" - local bind_ip="${LAB_HEIMDALL_BIND_IP:-0.0.0.0}" - local public_url="${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:-lab2025.duckdns.org}/}" - local image="${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347}" - local container_name="${HEIMDALL_CONTAINER_NAME:-homelab-heimdall}" - local seeder_container_name="${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder}" - local network="${HEIMDALL_NETWORK:-homelab-heimdall}" - local install_docker="${LAB_HEIMDALL_INSTALL_DOCKER:-false}" - local source_dir="${REPO_ROOT}/infra/heimdall" - local elapsed=0 - local value_name - local value - - require_debian_server "deploy-heimdall" - - if disabled_value "${mode}"; then - return 0 - fi - - for value_name in install_dir data_dir config_dir http_port bind_ip public_url image container_name seeder_container_name network install_docker; do - value="${!value_name}" - if [[ "${value}" == *$'\n'* ]]; then - echo "${value_name} cannot contain a newline." >&2 - exit 1 - fi - done - - if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/links.json" || ! -s "${source_dir}/seed-heimdall.py" ]]; then - echo "Missing Heimdall source files in ${source_dir}" >&2 - exit 1 - fi - - echo "Deploying external Heimdall on the Debian host at ${install_dir}..." - - ensure_local_docker_compose "HEIMDALL" "${install_docker}" - repair_local_docker_iptables - - sudo mkdir -p "${install_dir}" "${config_dir}" - sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" - sudo cp "${source_dir}/links.json" "${install_dir}/links.json" - sudo cp "${source_dir}/seed-heimdall.py" "${install_dir}/seed-heimdall.py" - sudo chown -R 1000:1000 "${data_dir}" - sudo tee "${install_dir}/.env" >/dev/null </dev/null; do - if ((elapsed >= 180)); then - echo "Timed out waiting for Heimdall on http://127.0.0.1:${http_port}/" >&2 - (cd "${install_dir}" && sudo docker compose ps && sudo docker compose logs --tail=120) >&2 || true - exit 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done - - echo "Heimdall is reachable at http://127.0.0.1:${http_port}/ and ${public_url%/}/." -} - -deploy_matrix() { - local mode="${LAB_MATRIX_DEPLOY:-true}" - local install_dir="${LAB_MATRIX_INSTALL_DIR:-/home/jv/matrix-server}" - local image="" - local container_name="" - local http_port="" - local ssh_port="" - local domain="" - local install_docker="false" - local source_dir="${REPO_ROOT}/infra/matrix-server" - local value_name - local value - - require_debian_server "deploy-matrix" - - if disabled_value "${mode}"; then - return 0 - fi - - for value_name in install_dir; do - value="${!value_name}" - if [[ "${value}" == *$'\n'* ]]; then - echo "${value_name} cannot contain a newline." >&2 - exit 1 - fi - done - - if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/homeserver.yaml.tpl" || ! -s "${source_dir}/bootstrap.sh" ]]; then - echo "Missing Matrix source files in ${source_dir}" >&2 - exit 1 - fi - - echo "Deploying Matrix stack on the Debian host at ${install_dir}..." - - ensure_local_docker_compose "MATRIX" "${install_docker}" - repair_local_docker_iptables - - sudo mkdir -p "${install_dir}" - sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" - sudo cp "${source_dir}/homeserver.yaml.tpl" "${install_dir}/homeserver.yaml.tpl" - sudo cp "${source_dir}/log_config.yaml" "${install_dir}/log_config.yaml" - sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" 2>/dev/null || { - sudo mkdir -p "${install_dir}/element-config" - sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" - } - sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh" - sudo chmod +x "${install_dir}/bootstrap.sh" - - # Secrets come from the SOPS-encrypted repo file, decrypted at deploy time. - # (Env vars override the file, for manual/ops overrides.) - if ! sops_load_secrets "${REPO_ROOT}/infra/matrix-server/matrix.secret.yaml"; then - exit 1 - fi - - if [[ -z "${MATRIX_POSTGRES_PASSWORD:-}" || -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]]; then - echo "MATRIX_POSTGRES_PASSWORD and MATRIX_REGISTRATION_SHARED_SECRET missing after decrypt." >&2 - exit 1 - fi - - local matrix_server_name="${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}" - local matrix_synapse_port="${LAB_MATRIX_SYNAPSE_PORT:-8008}" - local matrix_element_port="${LAB_MATRIX_ELEMENT_PORT:-8081}" - local postgres_db="${LAB_MATRIX_POSTGRES_DB:-synapse}" - local postgres_user="${LAB_MATRIX_POSTGRES_USER:-synapse}" - - sudo tee "${install_dir}/.env" >/dev/null <&2 - exit 1 - fi - - echo "Deploying OmniRoute LLM gateway on the Debian host at ${install_dir}..." - - ensure_local_docker_compose "OMNIROUTE" "${install_docker}" - repair_local_docker_iptables - - sudo mkdir -p "${install_dir}" - sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" - sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh" - sudo chmod +x "${install_dir}/bootstrap.sh" - - sudo tee "${install_dir}/.env" >/dev/null <&2 - exit 1 - fi - done - - echo "Deploying Hermes agent on the Debian host at ${install_dir}..." - - ensure_local_docker_compose "HERMES" "${install_docker}" - - # Install uv if missing. - if [[ ! -x "${uv_bin}" ]]; then - mkdir -p "$(dirname "${uv_bin}")" - echo "Installing uv..." - curl -LsSf https://astral.sh/uv/install.sh | bash - # uv installs to ~/.local/bin; copy/alias into HERMES bin if that path differs. - if command -v uv >/dev/null 2>&1 && [[ ! -x "${uv_bin}" ]]; then - cp "$(command -v uv)" "${uv_bin}" - fi - fi - - # Install Hermes via its official installer (git method, managed venv layout). - if [[ ! -x "${hermes_home}/hermes-agent/venv/bin/python" ]]; then - echo "Installing Hermes agent..." - curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash - fi - - # Ensure the Matrix platform dependencies (E2EE) are present in the venv. - if [[ -x "${uv_bin}" && -f "${install_dir}/pyproject.toml" ]]; then - cd "${install_dir}" - "${uv_bin}" pip install 'mautrix[encryption]' -p venv/bin/python - fi - - # Secret env for the gateway (.env under HERMES_HOME, mode 600, never committed). - # Pull the hermes secrets from the SOPS-encrypted repo file; env vars override. - if ! sops_load_secrets "${REPO_ROOT}/infra/hermes.secret.yaml"; then - echo "WARNING: could not load hermes secrets; MATRIX_PASSWORD may be blank." >&2 - fi - - local secrets_file="${hermes_home}/.env" - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "${hermes_home}" - if [[ -f "${secrets_file}" ]]; then - sudo cp "${secrets_file}" "${secrets_file}.bak-pre-jeannie" - fi - sudo tee "${secrets_file}" >/dev/null </dev/null 2>&1 && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^matrix-synapse$'; then - if [[ -n "${MATRIX_BOT_PASSWORD:-}" ]]; then - if ! docker exec matrix-synapse register_new_matrix_user -c /data/homeserver.yaml \ - -u hermes -p "${MATRIX_BOT_PASSWORD}" --no-admin >/dev/null 2>&1; then - echo "Hermes bot account already exists (expected on re-run) or auto-registration skipped." >&2 - else - echo "Hermes Matrix bot account ready." - fi - else - echo "MATRIX_BOT_PASSWORD not set; skipping Hermes bot account registration (needed only on fresh install)." >&2 - fi - fi - - # Install/start the gateway systemd user service (Hermes manages the unit). - "${hermes_home}/hermes-agent/venv/bin/python" "${hermes_home}/hermes-agent/hermes" gateway install - systemctl --user daemon-reload - systemctl --user enable --now hermes-gateway 2>/dev/null || true - - echo "Hermes agent deployed. Run 'hermes' to configure the model, or check gateway logs at ${hermes_home}/logs/gateway.log." -} - -deploy_rpi_services() { - local mode="${LAB_RPI_SERVICES_DEPLOY:-true}" - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" - local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}" - local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" - local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" - local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}" - local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}" - local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}" - local source_dir="${REPO_ROOT}/infra/rpi-services" - local value_name - local value - - require_debian_server "rpi-services" - - if disabled_value "${mode}"; then - return 0 - fi - - if [[ ! -d "${source_dir}" ]]; then - echo "Missing ${source_dir}" >&2 - exit 1 - fi - - for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do - value="${!value_name}" - if [[ "${value}" == *"'"* ]]; then - echo "${value_name} cannot contain a single quote." >&2 - exit 1 - fi - done - - echo "Deploying RPi services on ${rpi_user}@${rpi_host}..." - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services" - scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ - "${source_dir}/docker-compose.yml" \ - "${source_dir}/adlists.txt" \ - "${source_dir}/local-dns-records.txt" \ - "${source_dir}/cname-records.txt" \ - "${source_dir}/static-dhcp-hosts.txt" \ - "${source_dir}/uptime-kuma-monitors.json" \ - "${source_dir}/seed-uptime-kuma.js" \ - "${source_dir}/bootstrap.sh" \ - "${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/" - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " -LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \ -LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \ -LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \ -LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \ -LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \ -LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \ -LAB_RPI_INSTALL_DOCKER='${install_docker}' \ -PIHOLE_WEBPASSWORD='${pihole_webpassword}' \ -bash /tmp/homelab-rpi-services/bootstrap.sh" -} - -gitea_bootstrap_password() { - if command -v openssl >/dev/null 2>&1; then - openssl rand -hex 32 - return 0 - fi - - python3 - <<'PY' -import secrets - -print(secrets.token_hex(32)) -PY -} - -gitea_api_base_url() { - local gitea_host="$1" - local http_port="$2" - local candidate - local api_base_override="${LAB_GITEA_API_BASE_URL:-}" - - if [[ -n "${api_base_override}" ]]; then - printf '%s\n' "${api_base_override%/}" - return 0 - fi - - for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do - if curl -fsS "${candidate}/version" >/dev/null 2>&1; then - printf '%s\n' "${candidate}" - return 0 - fi - done - - echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2 - exit 1 -} - -gitea_repo_exists() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local owner="$4" - local repo_name="$5" - local status - - status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")" - case "${status}" in - 200) - return 0 - ;; - 404) - return 1 - ;; - 401 | 403) - echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2 - exit 1 - ;; - *) - echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2 - exit 1 - ;; - esac -} - -gitea_branch_exists() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local owner="$4" - local repo_name="$5" - local branch="$6" - local status - - status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")" - case "${status}" in - 200) - return 0 - ;; - 404) - return 1 - ;; - 401 | 403) - echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2 - exit 1 - ;; - *) - echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2 - exit 1 - ;; - esac -} - -create_gitea_repo() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local repo_name="$4" - local default_branch="$5" - local payload - - payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY' -import json -import sys - -repo_name, default_branch = sys.argv[1:3] -print(json.dumps({ - "name": repo_name, - "private": False, - "auto_init": False, - "default_branch": default_branch, - "description": "Homelab infrastructure configuration", -})) -PY -)" - - curl -fsS \ - -u "${auth_user}:${auth_password}" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "${payload}" \ - "${api_base}/user/repos" >/dev/null -} - -gitea_public_key_registered() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local owner="$4" - local repo_name="$5" - local public_key_path="$6" - local repo_keys - local user_keys - - user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")" - repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")" - - GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ - GITEA_USER_KEYS="${user_keys}" \ - GITEA_REPO_KEYS="${repo_keys}" \ - python3 - <<'PY' -import json -import os -import sys - -public_key = os.environ["GITEA_PUBLIC_KEY"].strip() -for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"): - for key in json.loads(os.environ[env_name]) or []: - if key.get("key", "").strip() == public_key: - sys.exit(0) -sys.exit(1) -PY -} - -create_gitea_repo_deploy_key() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local owner="$4" - local repo_name="$5" - local title="$6" - local public_key_path="$7" - local read_only="$8" - local payload - - payload="$( - GITEA_DEPLOY_KEY_TITLE="${title}" \ - GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \ - GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \ - python3 - <<'PY' -import json -import os - -print(json.dumps({ - "title": os.environ["GITEA_DEPLOY_KEY_TITLE"], - "key": os.environ["GITEA_PUBLIC_KEY"].strip(), - "read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true", -})) -PY - )" - - curl -fsS \ - -u "${auth_user}:${auth_password}" \ - -H "Content-Type: application/json" \ - -X POST \ - -d "${payload}" \ - "${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null -} - -ensure_gitea_repo_ssh_access() { - local api_base="$1" - local auth_user="$2" - local auth_password="$3" - local owner="$4" - local repo_name="$5" - local ssh_host="$6" - local ssh_port="$7" - local key_path="$8" - local key_title="$9" - local key_read_only="${10}" - local key_dir - local known_hosts - local public_key_path - local read_only_json="false" - local ssh_repo_url - - if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then - echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2 - exit 1 - fi - - key_dir="$(dirname "${key_path}")" - public_key_path="${key_path}.pub" - mkdir -p "${key_dir}" - chmod 0700 "${key_dir}" - - if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then - ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null - elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then - ssh-keygen -y -f "${key_path}" >"${public_key_path}" - elif [[ ! -s "${key_path}" ]]; then - echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2 - exit 1 - fi - - chmod 0600 "${key_path}" - chmod 0644 "${public_key_path}" - - if truthy "${key_read_only}"; then - read_only_json="true" - fi - - if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then - echo "Gitea already has Debian host SSH key ${public_key_path}." - else - create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}" - echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}." - fi - - known_hosts="${HOME}/.ssh/known_hosts" - touch "${known_hosts}" - chmod 0644 "${known_hosts}" - if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then - ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null - fi - - ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git" - git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null || - git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}" - git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new" - git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null - echo "Gitea SSH remote: ${ssh_repo_url}" -} - -bootstrap_gitea_repo() { - local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}" - local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" - local gitea_user="${LAB_GITEA_USER:-jv}" - local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}" - local http_port="${LAB_GITEA_HTTP_PORT:-3000}" - local ssh_port="${LAB_GITEA_SSH_PORT:-32222}" - local root_url="${LAB_GITEA_ROOT_URL:?LAB_GITEA_ROOT_URL is required from homelab.yml}" - local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}" - local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}" - local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}" - local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}" - local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}" - local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}" - local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}" - local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}" - local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}" - local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" - local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}" - local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}" - local api_base - local public_repo_url - local direct_repo_url - local push_url - local askpass - local credentials_dir - local remote_status - local worktree_status - - require_debian_server "bootstrap-gitea-repo" - - if disabled_value "${mode}"; then - return 0 - fi - - ensure_python3 - for value_name in repo_owner repo_name default_branch bootstrap_user; do - local value="${!value_name}" - if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then - echo "${value_name} contains unsupported characters." >&2 - exit 1 - fi - done - if [[ "${bootstrap_email}" == *"'"* ]]; then - echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2 - exit 1 - fi - if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then - echo "LAB_GITEA_SSH_PORT must be numeric." >&2 - exit 1 - fi - - if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then - # shellcheck disable=SC1090 - source "${credentials_file}" - bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}" - bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}" - bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}" - fi - - if [[ -z "${bootstrap_password}" ]]; then - bootstrap_password="$(gitea_bootstrap_password)" - credentials_dir="$(dirname "${credentials_file}")" - mkdir -p "${credentials_dir}" - chmod 0700 "${credentials_dir}" - { - printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}" - printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}" - printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}" - } > "${credentials_file}" - chmod 0600 "${credentials_file}" - echo "Generated Gitea bootstrap credentials at ${credentials_file}." - fi - - for value_name in repo_owner repo_name default_branch bootstrap_user; do - local value="${!value_name}" - if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then - echo "${value_name} contains unsupported characters." >&2 - exit 1 - fi - done - for value_name in bootstrap_email bootstrap_password; do - local value="${!value_name}" - if [[ "${value}" == *"'"* ]]; then - echo "${value_name} cannot contain a single quote." >&2 - exit 1 - fi - done - - echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..." - - # shellcheck disable=SC2087 - ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" </dev/null 2>&1; then - echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2 - exit 1 -fi - -for attempt in \$(seq 1 60); do - if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 || - curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then - break - fi - if [ "\${attempt}" = "60" ]; then - echo "Timed out waiting for Gitea API inside \${container_name}." >&2 - exit 1 - fi - sleep 2 -done - -if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \ - --username "\${bootstrap_user}" \ - --password "\${bootstrap_password}" \ - --email "\${bootstrap_email}" \ - --admin \ - --must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then - if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then - sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \ - --username "\${bootstrap_user}" \ - --password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || { - cat /tmp/homelab-gitea-user-password.log >&2 - exit 1 - } - else - cat /tmp/homelab-gitea-user-create.log >&2 - exit 1 - fi -fi -EOF - - api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")" - - if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then - echo "Gitea repository ${repo_owner}/${repo_name} already exists." - else - if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then - echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2 - exit 1 - fi - create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}" - echo "Created Gitea repository ${repo_owner}/${repo_name}." - fi - - public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git" - if [[ "${api_base}" == */git/api/v1 ]]; then - direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git" - else - direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git" - fi - push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}" - - git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null - git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null || - git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}" - - if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then - echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged." - else - worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)" - if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then - echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2 - echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2 - exit 1 - fi - - askpass="$(mktemp)" - trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT - cat > "${askpass}" </dev/null </dev/null 2>&1 || true" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -ssh_gitea "set -eu -sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true -sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories' -sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}' -sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}' -sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}' -sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" - -scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \ - "\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}" - -sudo mkdir -p "\${GITEA_BACKUP_DIR}" -sudo chown jv:jv "\${GITEA_BACKUP_DIR}" -sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}" -sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete - -echo "Created \${backup_archive}" -BACKUP_SCRIPT_EOT - sudo chmod 0755 "${backup_script}" - - sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT' -[Unit] -Description=Back up external Homelab Gitea to Debian host storage -After=network-online.target -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-gitea-backup.sh -SERVICE_EOT - - sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT' -[Unit] -Description=Run daily Homelab Gitea backups - -[Timer] -OnCalendar=*-*-* 02:35:00 -RandomizedDelaySec=20m -Persistent=true - -[Install] -WantedBy=timers.target -TIMER_EOT - - sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT' -#!/usr/bin/env bash -set -euo pipefail - -GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}" -GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}" -GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}" - -if ! command -v python3 >/dev/null 2>&1; then - echo "python3 is required for Gitea restore drills." >&2 - exit 1 -fi - -latest_archive="$( - { find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } | - sort -nr | - awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }' -)" - -if [[ -z "${latest_archive}" ]]; then - echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}." - exit 0 -fi - -timestamp="$(date -u +%Y%m%dT%H%M%SZ)" -tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")" -tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")" -report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt" - -cleanup() { - rm -rf "${tmp_dir}" - rm -f "${tmp_report}" -} -trap cleanup EXIT - -python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY' -import os -import sys -import zipfile - -archive_path, extract_dir, report_path = sys.argv[1:4] - -with zipfile.ZipFile(archive_path) as archive: - bad_member = archive.testzip() - if bad_member: - raise SystemExit(f"ZIP integrity check failed at {bad_member}") - - members = archive.infolist() - if not members: - raise SystemExit("ZIP archive is empty") - - extract_root = os.path.abspath(extract_dir) - for member in members: - target = os.path.abspath(os.path.join(extract_root, member.filename)) - if target != extract_root and not target.startswith(extract_root + os.sep): - raise SystemExit(f"Unsafe archive path: {member.filename}") - - archive.extractall(extract_root) - -file_count = 0 -total_bytes = 0 -for root, _, files in os.walk(extract_dir): - for name in files: - file_count += 1 - total_bytes += os.path.getsize(os.path.join(root, name)) - -if file_count == 0: - raise SystemExit("Archive extracted no files") - -with open(report_path, "w", encoding="utf-8") as handle: - handle.write("Gitea restore drill report\n") - handle.write(f"archive={archive_path}\n") - handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n") - handle.write(f"extracted_files={file_count}\n") - handle.write(f"extracted_bytes={total_bytes}\n") - handle.write("result=ok\n") -PY - -sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}" -sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}" -sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete - -echo "Created ${report_path}" -RESTORE_DRILL_SCRIPT_EOT - sudo chmod 0755 "${restore_drill_script}" - - sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT' -[Unit] -Description=Run a non-destructive Gitea backup restore drill -After=network-online.target homelab-gitea-backup.service -Wants=network-online.target - -[Service] -Type=oneshot -ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh -RESTORE_DRILL_SERVICE_EOT - - sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT' -[Unit] -Description=Run monthly Homelab Gitea restore drills - -[Timer] -OnCalendar=monthly -RandomizedDelaySec=2h -Persistent=true - -[Install] -WantedBy=timers.target -RESTORE_DRILL_TIMER_EOT - - sudo systemctl daemon-reload - sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null - sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null -} - -backup_gitea() { - require_debian_server "backup-gitea" - - install_gitea_backup_timer - sudo /usr/local/sbin/homelab-gitea-backup.sh -} - -drill_gitea_restore() { - require_debian_server "drill-gitea-restore" - - install_gitea_backup_timer - sudo /usr/local/sbin/homelab-gitea-restore-drill.sh -} - -install_gitea_runner() { - local runner_arch - local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}" - local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}" - local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}" - local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}" - local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}" - local runner_user="${GITEA_RUNNER_USER:-jv}" - local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}" - local missing_packages=() - - require_debian_server "install-gitea-runner" - - case "$(dpkg --print-architecture)" in - amd64) - runner_arch="linux-amd64" - ;; - arm64) - runner_arch="linux-arm64" - ;; - *) - echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2 - exit 1 - ;; - esac - - for package in ca-certificates curl git nodejs python3; do - if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then - missing_packages+=("$package") - fi - done - if [[ ${#missing_packages[@]} -gt 0 ]]; then - sudo apt-get update - sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" - fi - - sudo curl -fsSL \ - -o /usr/local/bin/act_runner \ - "https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}" - sudo chmod 0755 /usr/local/bin/act_runner - sudo chown root:root /usr/local/bin/act_runner - - sudo -u "${runner_user}" mkdir -p "${runner_home}" - - if [[ ! -f "${runner_home}/.runner" ]]; then - if [[ -z "${runner_token}" ]]; then - echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2 - exit 1 - fi - - sudo -u "${runner_user}" env \ - HOME="/home/${runner_user}" \ - GITEA_RUNNER_HOME="${runner_home}" \ - GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \ - GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \ - GITEA_RUNNER_NAME="${runner_name}" \ - GITEA_RUNNER_LABELS="${runner_labels}" \ - bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"' - else - echo "Existing runner registration found at ${runner_home}/.runner; keeping it." - fi - - sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null </dev/null - sudo systemctl status homelab-gitea-runner.service --no-pager -l -} - -recreate_pods_for_selector() { - local namespace="$1" - local selector="$2" - local app="$3" - - if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then - echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2 - dump_argocd_debug "${app}" - dump_namespace_debug "${namespace}" - exit 1 - fi -} - -refresh_argocd_application() { - local app="$1" - - kubectl --kubeconfig "${KUBECONFIG}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null -} - -website_translation_model() { - local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}" - local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}" - - require_debian_server "website-translation-model" - - if ! command -v ollama >/dev/null 2>&1; then - echo "ollama is not installed or not in PATH on this Debian server." >&2 - exit 1 - fi - - if [[ ! -r "${modelfile}" ]]; then - echo "Ollama Modelfile is not readable: ${modelfile}" >&2 - exit 1 - fi - - echo "Creating or updating Ollama model ${model_name} from ${modelfile}..." - ollama create "${model_name}" -f "${modelfile}" - - if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then - echo "Ollama model ${model_name} was not found after creation." >&2 - exit 1 - fi - - echo "Ollama model ${model_name} is ready." -} - -website_ollama_listen() { - local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}" - local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" - local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}" - local dropin_dir="/etc/systemd/system/ollama.service.d" - local dropin_file="${dropin_dir}/homelab.conf" - local waited=0 - - require_debian_server "website-ollama-listen" - - bind_address="${bind_address#http://}" - bind_address="${bind_address#https://}" - - if ! systemctl cat ollama.service >/dev/null 2>&1; then - echo "ollama.service was not found on this Debian server." >&2 - exit 1 - fi - - echo "Configuring ollama.service to listen on ${bind_address}..." - sudo mkdir -p "${dropin_dir}" - sudo mkdir -p "${models_dir}" - if id ollama >/dev/null 2>&1; then - sudo chown ollama:ollama "${models_dir}" - fi - printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" | - sudo tee "${dropin_file}" >/dev/null - sudo systemctl daemon-reload - sudo systemctl restart ollama.service - sudo systemctl is-active --quiet ollama.service - - echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..." - until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do - waited=$((waited + 2)) - if ((waited >= 60)); then - echo "Ollama service did not expose the local API after ${waited}s." >&2 - echo "Recent service logs:" >&2 - sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true - exit 1 - fi - sleep 2 - done - echo "Ollama API is reachable on the Debian host." -} - -ollama_linux_arch() { - case "$(uname -m)" in - x86_64 | amd64) - printf 'amd64\n' - ;; - aarch64 | arm64) - printf 'arm64\n' - ;; - *) - echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2 - return 1 - ;; - esac -} - -ollama_write_systemd_service() { - sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF' -[Unit] -Description=Ollama Service -After=network-online.target - -[Service] -ExecStart=/usr/bin/ollama serve -User=ollama -Group=ollama -Restart=always -RestartSec=3 -Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" - -[Install] -WantedBy=default.target -EOF -} - -ollama_install_from_tgz() { - local tarball="$1" - - if [[ ! -r "${tarball}" ]]; then - echo "Ollama tarball is not readable: ${tarball}" >&2 - exit 1 - fi - - echo "Installing Ollama from local tarball ${tarball}..." - sudo tar -C /usr -xzf "${tarball}" - if ! id ollama >/dev/null 2>&1; then - sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama - fi - if getent group render >/dev/null 2>&1; then - sudo usermod -aG render ollama - fi - if getent group video >/dev/null 2>&1; then - sudo usermod -aG video ollama - fi - ollama_write_systemd_service -} - -ollama_setup() { - local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}" - local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}" - local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" - local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}" - local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}" - local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}" - local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}" - local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}" - local arch - local tmp_installer - local curl_ip_args=() - - require_debian_server "ollama-setup" - - if ! command -v curl >/dev/null 2>&1; then - echo "curl is required to install Ollama." >&2 - exit 1 - fi - if [[ -n "${curl_ip_version}" ]]; then - curl_ip_args+=("${curl_ip_version}") - fi - - if ! command -v ollama >/dev/null 2>&1; then - if [[ -n "${local_tarball}" ]]; then - ollama_install_from_tgz "${local_tarball}" - else - echo "Installing Ollama from ${installer_url}..." - tmp_installer="$(mktemp)" - if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then - rm -f "${tmp_installer}" - arch="$(ollama_linux_arch || true)" - cat >&2 < 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then - echo "Ollama model ${model} is already present." - else - OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}" - fi - - echo "Ollama homelab setup is ready." -} - -require_argocd_application_controller() { - local namespace="${TF_VAR_argocd_namespace:-argocd}" - local expected_resources="10m|1|2Gi" - local actual_resources - - if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then - echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2 - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pods -o wide >&2 || true - kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true - exit 1 - fi - - actual_resources="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get statefulset argocd-application-controller \ - -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')" - if [[ "${actual_resources}" != "${expected_resources}" ]]; then - echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2 - echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2 - exit 1 - fi -} - -require_pimox_app_worker_capacity() { - local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}" - local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" - local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}" - local worker_key - local worker_host - local worker_user - local worker_node - local worker_key_path - local capacity - local ready - local failures=0 - - if disabled_value "${enabled}"; then - echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}." - return 0 - fi - if [[ ! -s "${spec_file}" ]]; then - echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate." - return 0 - fi - if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then - echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2 - exit 1 - fi - - while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do - [[ -n "${worker_node}" ]] || continue - capacity="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)" - ready="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" - if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then - echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2 - failures=$((failures + 1)) - fi - done <"${spec_file}" - - if ((failures > 0)); then - echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2 - kubectl --kubeconfig "${KUBECONFIG}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true - exit 1 - fi -} - -apps() { - local buildx_builder_ready=false - local demos_image_built=false - local demos_image_ref - local demos_image_state_file - local demos_platforms - local demos_registry_endpoint - local demos_source_hash - local registry_endpoint - local website_image_built=false - local website_image_ref - local website_image_state_file - local website_platforms - local website_source_hash - - require_debian_server "apps" - - registry_endpoint="$(apps_registry_endpoint)" - demos_registry_endpoint="$(demos_registry_endpoint)" - demos_image_ref="${registry_endpoint}/demos-static:latest" - demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state" - demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" - demos_source_hash="$(demos_source_hash)" - website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state" - website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}" - website_source_hash="$(website_source_hash)" - website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}" - export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}" - export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}" - export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" - export KUBECONFIG="${TF_VAR_kubeconfig_path}" - - require_argocd_application_controller - require_pimox_app_worker_capacity - - ensure_cosign_keypair - publish_cosign_public_key_config_map - - if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then - echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2 - exit 1 - fi - - if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then - echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2 - exit 1 - fi - - if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then - echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2 - exit 1 - fi - - if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then - echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2 - exit 1 - fi - ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}" - - echo "Deploying homelab applications..." - - run_tofu_stack "bootstrap/apps" - - refresh_argocd_application container-registry - refresh_argocd_application supply-chain-policy - - wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300 - wait_for_namespace container-registry container-registry 300 - wait_for_namespaced_resource container-registry deployment local-registry container-registry 300 - wait_for_deployment_ready container-registry local-registry container-registry 300 - - if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" && - image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then - echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build." - else - echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..." - ensure_docker_build_space - if [[ "${buildx_builder_ready}" != "true" ]]; then - prepare_buildx_builder "${registry_endpoint}" - buildx_builder_ready=true - fi - - docker buildx build \ - --network host \ - --platform "${website_platforms}" \ - --provenance=false \ - --sbom=true \ - --label "dev.homelab.website.source-hash=${website_source_hash}" \ - -t "${website_image_ref}" \ - -f "${REPO_ROOT}/apps/website/Dockerfile" \ - "${REPO_ROOT}/apps/website/" \ - --push - website_image_built=true - fi - publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}" - - if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" && - image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then - echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build." - else - echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..." - ensure_docker_build_space - if [[ "${buildx_builder_ready}" != "true" ]]; then - prepare_buildx_builder "${registry_endpoint}" - buildx_builder_ready=true - fi - - docker buildx build \ - --network host \ - --platform "${demos_platforms}" \ - --provenance=false \ - --sbom=true \ - --label "dev.homelab.demos.source-hash=${demos_source_hash}" \ - -t "${demos_image_ref}" \ - -f "${REPO_ROOT}/apps/demos-static/Dockerfile" \ - "${REPO_ROOT}/apps/demos-static/" \ - --push - demos_image_built=true - fi - publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}" - - refresh_argocd_application website-production - wait_for_namespace website-production website-production 300 - wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300 - if [[ "${website_image_built}" == "true" ]]; then - recreate_pods_for_selector website-production app=php-website website-production - else - echo "Skipping website pod restart because the image did not change." - fi - wait_for_deployment_ready website-production php-website-deployment website-production 300 - if [[ "${website_image_built}" == "true" ]]; then - write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" - fi - - refresh_argocd_application demos-static - wait_for_namespace demos-static demos-static 300 - wait_for_namespaced_resource demos-static deployment demos-static demos-static 300 - if [[ "${demos_image_built}" == "true" ]]; then - recreate_pods_for_selector demos-static app=demos-static demos-static - else - echo "Skipping demos pod restart because the image did not change." - fi - wait_for_deployment_ready demos-static demos-static demos-static 300 - if [[ "${demos_image_built}" == "true" ]]; then - write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" - fi - - refresh_argocd_application heimdall - wait_for_namespaced_resource monitoring ingress grafana heimdall 300 - wait_for_namespaced_resource monitoring ingress prometheus heimdall 300 - wait_for_namespaced_resource monitoring ingress alertmanager heimdall 300 - wait_for_namespaced_resource argocd ingress argocd-server heimdall 300 - - refresh_argocd_application n8n - wait_for_namespace n8n n8n 300 - wait_for_namespaced_resource n8n deployment n8n n8n 300 - wait_for_deployment_ready n8n n8n n8n 300 - - echo "Application deployment successfully completed." -} - -platform_apply() { - require_debian_server "platform" - - run_tofu_stack "bootstrap/platform" -} - -edge_apply() { - require_debian_server "edge" - - run_tofu_stack "bootstrap/edge" -} - -edge_haproxy_stats_password_valid() { - local password="${1:-}" - - [[ -n "${password}" ]] && ((${#password} >= 12)) && [[ "${password}" != "adminpassword" ]] -} - -ensure_edge_tailscale_routes() { - local enabled="${LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES:-true}" - local route="${LAB_EDGE_TAILSCALE_SUBNET_ROUTE:-${LAB_LAN_CIDR:-192.168.100.0/24}}" - local router="${LAB_EDGE_TAILSCALE_ROUTER:-debian}" - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}" - local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}" - local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - - if disabled_value "${enabled}"; then - return 0 - fi - if ! truthy "${enabled}"; then - echo "LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES must be true or false." >&2 - exit 1 - fi - if ! validate_ipv4_cidr_or_host "${route}"; then - echo "Invalid LAB_EDGE_TAILSCALE_SUBNET_ROUTE '${route}'." >&2 - exit 1 - fi - if [[ -z "${edge_host}" ]]; then - echo "LAB_EDGE_HOST or TF_VAR_edge_host is required to configure edge Tailscale routes." >&2 - exit 1 - fi - - case "${router}" in - debian) - echo "Ensuring Debian advertises ${route} and OCI edge accepts Tailscale routes..." - if ! command -v tailscale >/dev/null 2>&1; then - echo "tailscale is not installed on the Debian subnet router." >&2 - exit 1 - fi - sudo mkdir -p /etc/sysctl.d - printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null - sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null - sudo tailscale set --advertise-routes="${route}" - ;; - rpi | rpi4 | raspberrypi) - echo "Ensuring RPi advertises ${route} and OCI edge accepts Tailscale routes..." - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu -if ! command -v tailscale >/dev/null 2>&1; then - echo 'tailscale is not installed on the RPi subnet router.' >&2 - exit 1 -fi -sysctl_bin=\"\$(command -v sysctl 2>/dev/null || true)\" -if [ -z \"\$sysctl_bin\" ] && [ -x /usr/sbin/sysctl ]; then - sysctl_bin=/usr/sbin/sysctl -fi -if [ -z \"\$sysctl_bin\" ]; then - echo 'sysctl is not installed on the RPi subnet router.' >&2 - exit 1 -fi -sudo mkdir -p /etc/sysctl.d -printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null -sudo \"\$sysctl_bin\" -w net.ipv4.ip_forward=1 >/dev/null -sudo tailscale set --advertise-routes='${route}' -" - ;; - *) - echo "LAB_EDGE_TAILSCALE_ROUTER must be debian or rpi." >&2 - exit 1 - ;; - esac - - ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu -if ! command -v tailscale >/dev/null 2>&1; then - echo 'tailscale is not installed on the OCI edge host.' >&2 - exit 1 -fi -sudo tailscale set --accept-routes=true -" -} - -edge_haproxy_stats_credentials_file() { - printf '%s\n' "${LAB_EDGE_HAPROXY_STATS_CREDENTIALS_FILE:-${HOME}/.config/homelab/edge-haproxy.env}" -} - -generate_edge_haproxy_stats_password() { - if command -v openssl >/dev/null 2>&1; then - openssl rand -hex 24 - elif command -v python3 >/dev/null 2>&1; then - python3 - <<'PY' -import secrets - -print(secrets.token_hex(24)) -PY - else - echo "openssl or python3 is required to generate an edge HAProxy stats password." >&2 - exit 1 - fi -} - -write_edge_haproxy_stats_credentials() { - local credentials_file="$1" - local password="$2" - local credentials_dir - - credentials_dir="$(dirname "${credentials_file}")" - mkdir -p "${credentials_dir}" - chmod 0700 "${credentials_dir}" - { - printf 'TF_VAR_haproxy_stats_password=' - printf '%q' "${password}" - printf '\n' - } > "${credentials_file}" - chmod 0600 "${credentials_file}" -} - -recover_edge_haproxy_stats_password() { - local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}" - local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}" - local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local edge_install_dir="${TF_VAR_edge_install_dir:-${LAB_EDGE_INSTALL_DIR:-/opt/homelab-edge}}" - local remote_auth - local remote_password - - if [[ -z "${edge_host}" || "${edge_install_dir}" == *"'"* ]]; then - return 1 - fi - - remote_auth="$(ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ - "sudo sed -n 's/^[[:space:]]*stats auth //p' '${edge_install_dir}/config_files/haproxy.cfg' | head -n 1" 2>/dev/null || true)" - remote_auth="${remote_auth//$'\r'/}" - if [[ "${remote_auth}" != *:* ]]; then - return 1 - fi - - remote_password="${remote_auth#*:}" - if ! edge_haproxy_stats_password_valid "${remote_password}"; then - return 1 - fi - - printf '%s\n' "${remote_password}" -} - -ensure_edge_haproxy_stats_password() { - local credentials_file - local recovered_password - - if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then - export TF_VAR_haproxy_stats_password - return 0 - fi - - credentials_file="$(edge_haproxy_stats_credentials_file)" - if [[ -r "${credentials_file}" ]]; then - # shellcheck disable=SC1090 - source "${credentials_file}" - if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then - export TF_VAR_haproxy_stats_password - return 0 - fi - echo "Ignoring invalid edge HAProxy stats credentials in ${credentials_file}." >&2 - fi - - if recovered_password="$(recover_edge_haproxy_stats_password)"; then - TF_VAR_haproxy_stats_password="${recovered_password}" - export TF_VAR_haproxy_stats_password - write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}" - echo "Recovered edge HAProxy stats password from the live edge host and stored it at ${credentials_file}." - return 0 - fi - - TF_VAR_haproxy_stats_password="$(generate_edge_haproxy_stats_password)" - export TF_VAR_haproxy_stats_password - write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}" - echo "Generated edge HAProxy stats password at ${credentials_file}." -} - -ensure_cluster_worker_var_file() { - if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then - prepare_cluster_worker_var_file false - fi - if truthy "${TF_VAR_allow_empty_worker_nodes:-false}"; then - return 0 - fi - if ! cluster_worker_var_file_has_workers "${LAB_CLUSTER_VAR_FILE}"; then - echo "Cluster worker var file has no worker_nodes: ${LAB_CLUSTER_VAR_FILE}" >&2 - echo "Run the Pimox worker stage first or set LAB_PIMOX_WORKER_COUNT and LAB_PIMOX_WORKER_STATIC_IPS." >&2 - return 1 - fi -} - -run_up_parallel_stage() { - local spec_file - local parallelism="${JEANNIE_UP_PARALLELISM:-4}" - local status - - if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then - echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2 - return 1 - fi - - spec_file="$(mktemp)" - python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY' -import json -import sys - -spec_file, jeannie, parallelism = sys.argv[1:4] -spec = { - "parallelism": int(parallelism), - "tasks": [ - { - "id": "gitea-deploy", - "title": "Gitea deploy", - "command": [jeannie, "__up-task", "gitea-deploy"], - }, - { - "id": "gitea-bootstrap", - "title": "Gitea repo bootstrap", - "needs": ["gitea-deploy"], - "command": [jeannie, "__up-task", "gitea-bootstrap"], - }, - { - "id": "heimdall-deploy", - "title": "Heimdall deploy", - "command": [jeannie, "__up-task", "heimdall-deploy"], - }, - { - "id": "matrix-deploy", - "title": "Matrix deploy", - "command": [jeannie, "__up-task", "matrix-deploy"], - }, - { - "id": "omniroute-deploy", - "title": "OmniRoute deploy", - "command": [jeannie, "__up-task", "omniroute-deploy"], - }, - { - "id": "hermes-deploy", - "title": "Hermes deploy", - "needs": ["omniroute-deploy", "matrix-deploy"], - "command": [jeannie, "__up-task", "hermes-deploy"], - }, - { - "id": "rpi-services", - "title": "RPi services", - "command": [jeannie, "__up-task", "rpi-services"], - }, - { - "id": "pimox-workers", - "title": "Pimox provisioning and workers", - "command": [jeannie, "__up-task", "pimox-workers"], - }, - { - "id": "openwrt", - "title": "OpenWrt VM", - "command": [jeannie, "__up-task", "openwrt"], - }, - { - "id": "edge-host", - "title": "OCI edge host check", - "command": [jeannie, "__up-task", "edge-host"], - }, - ], -} -with open(spec_file, "w", encoding="utf-8") as handle: - json.dump(spec, handle, indent=2) - handle.write("\n") -PY - set +e - "${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}" - status=$? - set -e - rm -f "${spec_file}" - return "${status}" -} - -up_task() { - require_debian_server "__up-task" - - case "${1:-}" in - gitea-deploy) - deploy_gitea - ;; - gitea-bootstrap) - bootstrap_gitea_repo - ;; - heimdall-deploy) - deploy_heimdall - ;; - matrix-deploy) - deploy_matrix - ;; - omniroute-deploy) - deploy_omniroute - ;; - hermes-deploy) - deploy_hermes - ;; - rpi-services) - deploy_rpi_services - ;; - pimox-workers) - run_pimox_pipeline - ;; - openwrt) - run_openwrt_pipeline - ;; - edge-host) - check_edge_ssh - ;; - *) - echo "Unknown __up-task '${1:-}'." >&2 - return 1 - ;; - esac -} - -up() { - require_debian_server "up" - - echo "Deploying the homelab infrastructure..." - jeannie_log_start "up" - jeannie_step_plan 12 - - run_step "Early preflight" homelab_preflight early - run_step "Go toolchain" ensure_go_toolchain - run_step "Repo Map (Graphiphy)" graphiphy_map - run_step "Independent host services" run_up_parallel_stage - run_step "Full preflight" homelab_preflight full - run_step "Pre-apply doctor" doctor_preapply - run_step "Worker var file" ensure_cluster_worker_var_file - run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up - run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" - run_step "Version report" doctor_versions_report - run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" - run_step "Applications" apps - run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" - - echo "Deployment successfully completed." - echo "Log: ${JEANNIE_LOG_FILE}" -} - -tofu_stack_from_plan_target() { - case "$1" in - provisioning) - printf 'bootstrap/provisioning\n' - ;; - cluster) - printf 'bootstrap/cluster\n' - ;; - platform) - printf 'bootstrap/platform\n' - ;; - apps) - printf 'bootstrap/apps\n' - ;; - edge) - printf 'bootstrap/edge\n' - ;; - *) - echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2 - return 1 - ;; - esac -} - -plan_homelab() { - local target="${1:-all}" - local stack - local -a stacks=() - - require_debian_server "plan" - - if [[ "${target}" == "all" ]]; then - if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then - prepare_cluster_worker_var_file false - fi - stacks=( - "bootstrap/provisioning" - "bootstrap/cluster" - "bootstrap/platform" - "bootstrap/apps" - "bootstrap/edge" - ) - else - stack="$(tofu_stack_from_plan_target "${target}")" - if [[ "${stack}" == "bootstrap/cluster" && -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then - prepare_cluster_worker_var_file false - fi - stacks=("${stack}") - fi - - for stack in "${stacks[@]}"; do - echo - echo "Planning ${stack}..." - run_tofu_plan_stack "${stack}" - done -} - -rebuild_cluster() { - require_debian_server "rebuild-cluster" - - export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}" - export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" - export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}" - export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}" - export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}" - export TF_VAR_force_worker_rejoin="${TF_VAR_force_worker_rejoin:-true}" - - echo "Rebuilding the Kubernetes cluster without touching external Gitea..." - jeannie_log_start "rebuild-cluster" - jeannie_step_plan 10 - - run_step "Preflight" homelab_preflight - run_step "Nuke existing cluster state" nuke_for_rebuild - run_step "Pimox provisioning and workers" run_pimox_pipeline - run_step "OpenWrt VM" run_openwrt_pipeline - run_step "Worker var file" ensure_cluster_worker_var_file - run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster" - run_step "Version report" doctor_versions_report - run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform" - run_step "Applications" apps - run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge" - - echo "Cluster rebuild successfully completed." - echo "Log: ${JEANNIE_LOG_FILE}" -} - -nuke_for_rebuild() { - LAB_NUKE_SKIP_CONFIRM=true nuke -} - -cluster_worker_targets() { - local worker_ssh_targets="${WORKER_SSH_TARGETS-}" - local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - - CLUSTER_WORKER_TARGETS=() - read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}" - - if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then - while IFS= read -r target; do - [[ -n "${target}" ]] || continue - if [[ ! " ${CLUSTER_WORKER_TARGETS[*]-} " =~ [[:space:]]${target}[[:space:]] ]]; then - CLUSTER_WORKER_TARGETS+=("${target}") - fi - done < <(python3 - "${var_file}" <<'PY' -import json -import sys - -var_file = sys.argv[1] -with open(var_file, encoding="utf-8") as handle: - document = json.load(handle) - -for key, node in sorted((document.get("worker_nodes") or {}).items()): - host = node.get("host") - user = node.get("user") - if host and user: - print(f"{user}@{host}") -PY -) - fi -} - -stop_local_kubernetes_services() { - echo "--> Stopping local Kubernetes services on the Debian control plane..." - sudo systemctl stop kubelet 2>/dev/null || true - if command -v crictl >/dev/null 2>&1; then - sudo crictl stop --all 2>/dev/null || true - elif command -v ctr >/dev/null 2>&1; then - sudo ctr -n k8s.io tasks ls -q 2>/dev/null | - while IFS= read -r task; do - [[ -n "${task}" ]] || continue - sudo ctr -n k8s.io tasks kill -s SIGTERM "${task}" 2>/dev/null || true - done - sleep 3 - sudo ctr -n k8s.io tasks ls -q 2>/dev/null | - while IFS= read -r task; do - [[ -n "${task}" ]] || continue - sudo ctr -n k8s.io tasks kill -s SIGKILL "${task}" 2>/dev/null || true - sudo ctr -n k8s.io tasks rm -f "${task}" 2>/dev/null || true - done - fi - if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then - sudo systemctl stop containerd 2>/dev/null || true - sudo systemctl reset-failed containerd 2>/dev/null || true - fi - sudo systemctl reset-failed kubelet 2>/dev/null || true -} - -start_local_kubernetes_services() { - echo "--> Starting local Kubernetes services on the Debian control plane..." - sudo systemctl start containerd - sudo systemctl start docker 2>/dev/null || true - sudo systemctl start kubelet -} - -stop_remote_kubernetes_services() { - local target="$1" - - echo "--> Stopping Kubernetes services on remote worker ${target}..." - if ! ssh -o ConnectTimeout=5 "${target}" \ - "sudo systemctl stop kubelet 2>/dev/null || true -if command -v crictl >/dev/null 2>&1; then - sudo crictl stop --all 2>/dev/null || true -elif command -v ctr >/dev/null 2>&1; then - sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do - [ -n \"\$task\" ] || continue - sudo ctr -n k8s.io tasks kill -s SIGTERM \"\$task\" 2>/dev/null || true - done - sleep 3 - sudo ctr -n k8s.io tasks ls -q 2>/dev/null | while IFS= read -r task; do - [ -n \"\$task\" ] || continue - sudo ctr -n k8s.io tasks kill -s SIGKILL \"\$task\" 2>/dev/null || true - sudo ctr -n k8s.io tasks rm -f \"\$task\" 2>/dev/null || true - done -fi -if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi -sudo systemctl reset-failed kubelet 2>/dev/null || true"; then - echo "Warning: could not SSH to ${target}; continuing because Pimox VM shutdown follows." >&2 - fi -} - -start_remote_kubernetes_services() { - local target="$1" - - echo "--> Starting Kubernetes services on remote worker ${target}..." - ssh -o ConnectTimeout=5 "${target}" \ - "sudo systemctl start containerd && sudo systemctl start kubelet" -} - -pimox_worker_count_default() { - local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" - local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" - local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}" - local max_count=0 - local count - local state_count - - if [[ -s "${spec_file}" ]]; then - count="$(awk -F'\t' -v key_prefix="${worker_key_prefix}" -v node_prefix="${worker_node_prefix}-" ' - $1 ~ "^" key_prefix "[0-9]+$" { - worker_index = substr($1, length(key_prefix) + 1) + 0 - if (worker_index > max) max = worker_index - } - $4 ~ "^" node_prefix "[0-9]+$" { - worker_index = substr($4, length(node_prefix) + 1) + 0 - if (worker_index > max) max = worker_index - } - END { print max + 0 } - ' "${spec_file}")" - if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then - max_count="${count}" - fi - fi - - if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then - count="$(python3 - "${var_file}" "${worker_key_prefix}" "${worker_node_prefix}" <<'PY' -import json -import re -import sys - -var_file, worker_key_prefix, worker_node_prefix = sys.argv[1:4] -with open(var_file, encoding="utf-8") as handle: - document = json.load(handle) -nodes = document.get("worker_nodes") or {} -highest = 0 -key_pattern = re.compile(rf"^{re.escape(worker_key_prefix)}(\d+)$") -node_pattern = re.compile(rf"^{re.escape(worker_node_prefix)}-(\d+)$") -for key, node in nodes.items(): - for candidate in (key, str(node.get("node_name", ""))): - match = key_pattern.match(candidate) or node_pattern.match(candidate) - if match: - highest = max(highest, int(match.group(1))) -print(highest) -PY -)" - if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then - max_count="${count}" - fi - fi - - state_count="$(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" state show null_resource.worker_nodes_required 2>/dev/null | - awk -F'"' '/"worker_count"[[:space:]]*=/ { print $4; found = 1 } END { exit found ? 0 : 1 }' || true)" - if [[ "${state_count}" =~ ^[0-9]+$ && "${state_count}" -gt "${max_count}" ]]; then - max_count="${state_count}" - fi - - count="${LAB_PIMOX_DEFAULT_WORKER_COUNT:-1}" - if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then - max_count="${count}" - fi - - printf '%s\n' "${max_count}" -} - -pimox_worker_count_effective() { - local configured="${LAB_PIMOX_WORKER_COUNT:-}" - local detected - - detected="$(pimox_worker_count_default)" - if [[ "${configured}" =~ ^[0-9]+$ && "${configured}" -gt "${detected}" ]]; then - printf '%s\n' "${configured}" - return 0 - fi - printf '%s\n' "${detected}" -} - -stop_pimox_worker_vms() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}" - local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}" - local index - local vmid - - worker_count="$(pimox_worker_count_effective)" - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 - exit 1 - fi - if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then - echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2 - exit 1 - fi - if ((worker_count == 0)); then - return 0 - fi - - echo "--> Stopping Pimox worker VMs on ${pimox_host}..." - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." - continue - fi - vmid=$((worker_base_vmid + index - 1)) - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then - echo 'Pimox worker VM ${vmid} does not exist; skipping.' - exit 0 -fi -if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then - echo 'Pimox worker VM ${vmid} is already stopped.' - exit 0 -fi -echo 'Gracefully shutting down Pimox worker VM ${vmid}...' -if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then - exit 0 -fi -if [ '${force_stop}' = 'true' ]; then - echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.' - sudo '${qm_bin}' stop '${vmid}' -else - echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2 - exit 1 -fi" - done -} - -verify_pimox_worker_vms_stopped() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local index - local vmid - local failures=0 - - worker_count="$(pimox_worker_count_effective)" - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - continue - fi - vmid=$((worker_base_vmid + index - 1)) - if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' 2>/dev/null | grep -q 'status: stopped'"; then - echo "Pimox worker VM ${vmid} is not stopped." >&2 - failures=$((failures + 1)) - fi - done - - ((failures == 0)) -} - -destroy_pimox_worker_vms() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local index - local vmid - - if ! truthy "${LAB_NUKE_DESTROY_PIMOX_WORKERS:-true}"; then - echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}." - return 0 - fi - worker_count="$(pimox_worker_count_effective)" - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 - exit 1 - fi - if ((worker_count == 0)); then - return 0 - fi - - echo "--> Destroying Pimox worker VMs on ${pimox_host}..." - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." - continue - fi - vmid=$((worker_base_vmid + index - 1)) - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then - echo 'Pimox worker VM ${vmid} does not exist; skipping.' - exit 0 -fi -if sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'; then - echo 'Pimox VM ${vmid} is a template; refusing to destroy it as a worker.' >&2 - exit 1 -fi -sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true -sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" - done -} - -start_pimox_worker_vms() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local index - local vmid - - worker_count="$(pimox_worker_count_effective)" - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2 - exit 1 - fi - if ((worker_count == 0)); then - return 0 - fi - - echo "--> Starting Pimox worker VMs on ${pimox_host}..." - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}." - continue - fi - vmid=$((worker_base_vmid + index - 1)) - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then - echo 'Pimox worker VM ${vmid} does not exist; skipping.' - exit 0 -fi -if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then - echo 'Pimox worker VM ${vmid} is already running.' - exit 0 -fi -echo 'Starting Pimox worker VM ${vmid}...' -sudo '${qm_bin}' start '${vmid}'" - done -} - -stop_cluster() { - local target - local failures=0 - declare -a CLUSTER_WORKER_TARGETS=() - - require_debian_server "stop-cluster" - cluster_worker_targets - - echo "Stopping Kubernetes cluster runtime without destroying state..." - for target in "${CLUSTER_WORKER_TARGETS[@]}"; do - stop_remote_kubernetes_services "${target}" - done - stop_pimox_worker_vms - stop_local_kubernetes_services - if kubernetes_api_reachable; then - echo "Kubernetes API is still reachable after stop; control-plane containers may still be running." >&2 - failures=$((failures + 1)) - fi - if ! verify_pimox_worker_vms_stopped; then - failures=$((failures + 1)) - fi - if ((failures > 0)); then - echo "Cluster stop finished with ${failures} verification failure(s)." >&2 - return 1 - fi - echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact." -} - -start_cluster() { - local target - declare -a CLUSTER_WORKER_TARGETS=() - - require_debian_server "start-cluster" - cluster_worker_targets - - echo "Starting Kubernetes cluster runtime without rebuilding state..." - start_local_kubernetes_services - start_pimox_worker_vms - for target in "${CLUSTER_WORKER_TARGETS[@]}"; do - start_remote_kubernetes_services "${target}" - done - echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness." -} - -kubernetes_api_reachable() { - kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1 -} - -wait_for_kubernetes_api_stable() { - local timeout_seconds="${1:-300}" - local stable_seconds="${2:-30}" - local interval_seconds="${3:-5}" - local elapsed=0 - local stable_elapsed=0 - - echo "Waiting for Kubernetes API to stay ready for ${stable_seconds}s..." - until ((stable_elapsed >= stable_seconds)); do - if kubernetes_api_reachable; then - stable_elapsed=$((stable_elapsed + interval_seconds)) - else - stable_elapsed=0 - fi - - if ((stable_elapsed >= stable_seconds)); then - return 0 - fi - if ((elapsed >= timeout_seconds)); then - echo "Kubernetes API did not stay ready for ${stable_seconds}s within ${timeout_seconds}s." >&2 - return 1 - fi - - sleep "${interval_seconds}" - elapsed=$((elapsed + interval_seconds)) - done -} - -ensure_kubernetes_api_ready_for_tofu_stack() { - local stack="$1" - - case "${stack}" in - bootstrap/platform|bootstrap/apps) - wait_for_kubernetes_api_stable \ - "${LAB_KUBERNETES_API_WAIT_TIMEOUT_SECONDS:-300}" \ - "${LAB_KUBERNETES_API_STABLE_SECONDS:-30}" \ - "${LAB_KUBERNETES_API_STABLE_INTERVAL_SECONDS:-5}" - ;; - esac -} - -cluster_control_plane_tracked() { - tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane" -} - -cluster_admin_kubeconfig_present() { - [[ -s "${KUBECONFIG_PATH}" ]] || - sudo test -s /etc/kubernetes/admin.conf 2>/dev/null || - [[ -n "${KUBECONFIG:-}" && -s "${KUBECONFIG}" ]] -} - -wait_for_kubernetes_api() { - local timeout_seconds="${1:-180}" - local elapsed=0 - - until kubernetes_api_reachable; do - if ((elapsed >= timeout_seconds)); then - echo "Kubernetes API did not become reachable after ${timeout_seconds}s." >&2 - return 1 - fi - sleep 5 - elapsed=$((elapsed + 5)) - done -} - -ensure_existing_cluster_started_for_up() { - if ! cluster_control_plane_tracked; then - echo "No tracked kubeadm control plane found; bootstrap/cluster will create one." - return 0 - fi - - if ! cluster_admin_kubeconfig_present; then - cat >&2 <&2 - fi -} - -check_pimox_worker_tailnet_egress() { - local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" - local probe_host="${LAB_PIMOX_WORKER_TAILNET_PROBE_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" - local probe_port="${LAB_PIMOX_WORKER_TAILNET_PROBE_PORT:-${LAB_GITEA_HTTP_PORT:-3000}}" - local failures=0 - local found=0 - local worker_key - local host - local user - local node_name - local key_path - local output - - if [[ ! -s "${spec_file}" ]]; then - echo "No Pimox worker spec found at ${spec_file}; run ./jeannie up or set LAB_CLUSTER_VAR_FILE." - return 1 - fi - if [[ -z "${probe_host}" ]]; then - echo "No tailnet probe host configured; set LAB_PIMOX_WORKER_TAILNET_PROBE_HOST or LAB_DEBIAN_TAILSCALE_IP." - return 1 - fi - - while IFS=$'\t' read -r worker_key host user node_name key_path; do - [[ -n "${worker_key}" && -n "${host}" && -n "${user}" && -n "${node_name}" && -n "${key_path}" ]] || continue - found=1 - printf '%-28s ' "${node_name}" - if output="$(ssh -n -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "${user}@${host}" " -set -eu -command -v tailscale >/dev/null || { echo 'tailscale command missing'; exit 1; } -sudo systemctl is-active --quiet tailscaled || { echo 'tailscaled is not active'; exit 1; } -tailscale_ip=\"\$(tailscale ip -4 2>/dev/null | head -n 1 || true)\" -[ -n \"\$tailscale_ip\" ] || { echo 'worker is not joined to Tailscale'; exit 1; } -timeout 6 bash -c 'exec 3<>/dev/tcp/${probe_host}/${probe_port}' || { echo 'tailnet TCP probe failed'; exit 1; } -echo \"tailscale_ip=\$tailscale_ip\" -" 2>&1)"; then - printf 'ok - %s:%s reachable over tailnet (%s)\n' "${probe_host}" "${probe_port}" "${output}" - else - printf 'fail - %s\n' "$(printf '%s' "${output}" | head -n 1)" - failures=$((failures + 1)) - fi - done <"${spec_file}" - - if ((found == 0)); then - echo "No Pimox workers were listed in ${spec_file}." - return 1 - fi - - ((failures == 0)) -} - -status_http() { - local name="$1" - local url="$2" - - if command -v curl >/dev/null 2>&1; then - printf '%-28s ' "${name}" - curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code} %{url_effective}\n' "${url}" || - printf 'unreachable %s\n' "${url}" - else - printf '%-28s curl not installed\n' "${name}" - fi -} - -status_http_ok() { - local url="$1" - local status - - if ! command -v curl >/dev/null 2>&1; then - echo "curl not installed" >&2 - return 1 - fi - - status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { - echo "unreachable ${url}" >&2 - return 1 - } - case "${status}" in - 2* | 3*) - return 0 - ;; - *) - echo "HTTP ${status} from ${url}" >&2 - return 1 - ;; - esac -} - -status_http_reachable() { - local url="$1" - local status - - if ! command -v curl >/dev/null 2>&1; then - echo "curl not installed" >&2 - return 1 - fi - - status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || { - echo "unreachable ${url}" >&2 - return 1 - } - case "${status}" in - 2* | 3* | 4*) - return 0 - ;; - *) - echo "HTTP ${status} from ${url}" >&2 - return 1 - ;; - esac -} - -status_gitea_public_ok() { - local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" - - status_http_reachable "${root_url}" -} - -status_cascade_check() { - local description="$1" - local output - shift - - if truthy "${JEANNIE_REPORT_MODE:-false}"; then - report_ui_check fail "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@" || return 1 - return 0 - fi - - printf '%-36s ' "${description}" - if output="$("$@" 2>&1)"; then - printf 'ok\n' - if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' - fi - return 0 - fi - - printf 'fail\n' - if [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /' - fi - return 1 -} - -status_cascade_warn() { - local description="$1" - local output - shift - - if truthy "${JEANNIE_REPORT_MODE:-false}"; then - report_ui_check warn "${STATUS_CURRENT_SECTION:-Status}" "${description}" "./jeannie explain status" "" "" "$@" - return 0 - fi - - printf '%-36s ' "${description}" - if output="$("$@" 2>&1)"; then - printf 'ok\n' - if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' - fi - return 0 - fi - - printf 'warn\n' - if [[ -n "${output}" ]]; then - printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /' - fi - return 0 -} - -status_container_running() { - local container="$1" - - if ! command -v docker >/dev/null 2>&1; then - echo "docker not installed" >&2 - return 1 - fi - if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then - echo "container ${container} is not running" >&2 - return 1 - fi -} - -status_kubernetes_api_ok() { - if ! command -v kubectl >/dev/null 2>&1; then - echo "kubectl not installed" >&2 - return 1 - fi - if [[ ! -s "${KUBECONFIG_PATH}" ]]; then - echo "kubeconfig missing: ${KUBECONFIG_PATH}" >&2 - return 1 - fi - kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null -} - -status_kubernetes_nodes_ready() { - local nodes - local not_ready - - nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes --no-headers 2>/dev/null)" || return 1 - if [[ -z "${nodes}" ]]; then - echo "no Kubernetes nodes found" >&2 - return 1 - fi - not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")" - if [[ -n "${not_ready}" ]]; then - echo "not ready: ${not_ready}" >&2 - return 1 - fi -} - -status_kubernetes_deployment_ready() { - local namespace="$1" - local deployment="$2" - - kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null -} - -status_no_problem_pods() { - local rows - - rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | - awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }' - )" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" >&2 - return 1 - fi -} - -status_recent_deployments() { - local rows - - rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get deployments -A \ - -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.readyReplicas,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,AVAILABLE:.status.availableReplicas,AGE:.metadata.creationTimestamp' \ - --no-headers 2>/dev/null | - awk '$3 != $4 || $5 != $4 || $6 != $4 { print }' - )" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" >&2 - return 1 - fi -} - -status_pod_restart_pressure() { - local threshold="${LAB_STATUS_RESTART_THRESHOLD:-3}" - local rows - - rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null | - awk -v threshold="${threshold}" '$5 + 0 >= threshold { print $1 "/" $2 ": restarts=" $5 " age=" $6 }' - )" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" >&2 - return 1 - fi -} - -status_node_pressure() { - local rows - - rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}{"="}{.status}{" "}{end}{"\n"}{end}' 2>/dev/null | - awk '/MemoryPressure=True|DiskPressure=True|PIDPressure=True|NetworkUnavailable=True/ { print }' - )" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" >&2 - return 1 - fi -} - -status_disk_pressure() { - local threshold="${LAB_STATUS_DISK_USE_THRESHOLD:-85}" - local rows - - rows="$(df -P / /data 2>/dev/null | awk -v threshold="${threshold}" 'NR > 1 { gsub(/%/, "", $5); if ($5 + 0 >= threshold) print $6 ": " $5 "% used" }')" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" >&2 - return 1 - fi -} - -status_traefik_error_rates() { - local since="${LAB_STATUS_LOG_SINCE:-30m}" - local sample - local count_502 - local count_404 - - sample="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik logs -l app.kubernetes.io/name=traefik --since="${since}" --tail=2000 2>/dev/null || true)" - if [[ -z "${sample}" ]]; then - echo "no recent Traefik logs available" >&2 - return 1 - fi - - count_502="$(grep -Eac '(^|[^0-9])50[234]([^0-9]|$)|level=error' <<<"${sample}" || true)" - count_404="$(grep -Eac '(^|[^0-9])404([^0-9]|$)' <<<"${sample}" || true)" - if ((count_502 > 0)); then - printf '5xx/error lines in last %s: %s\n' "${since}" "${count_502}" >&2 - return 1 - fi - printf '404 lines in last %s: %s\n' "${since}" "${count_404}" >&2 -} - -status_gitea_recent_errors() { - local since="${LAB_STATUS_LOG_SINCE:-30m}" - local container="${GITEA_CONTAINER_NAME:-homelab-gitea}" - local sample - local matches - - if ! command -v docker >/dev/null 2>&1; then - echo "docker not installed" >&2 - return 1 - fi - if ! sudo docker inspect "${container}" >/dev/null 2>&1; then - echo "container ${container} not found" >&2 - return 1 - fi - - sample="$(sudo docker logs --since "${since}" "${container}" 2>&1 || true)" - matches="$(grep -Eai 'error|panic|fatal|authentication failed|denied| 50[0-9] ' <<<"${sample}" | tail -10 || true)" - if [[ -n "${matches}" ]]; then - printf '%s\n' "${matches}" >&2 - return 1 - fi -} - -status_registry_ok() { - status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/" -} - -status_rpi_uptime_kuma_ok() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" - - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \ - "curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null" -} - -status_pimox_workers_running() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local index - local vmid - local failures=0 - - worker_count="$(pimox_worker_count_effective)" - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - echo "invalid worker count ${worker_count}" >&2 - return 1 - fi - - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - continue - fi - vmid=$((worker_base_vmid + index - 1)) - if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then - echo "VM ${vmid} is not running" >&2 - failures=$((failures + 1)) - fi - done - - ((failures == 0)) -} - -status_cascade_report() { - local failures=0 - - status_section "Status Cascade" - status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1)) - status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1)) - status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1)) - status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1)) - status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1)) - - status_section "Git And Local Services" - status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1)) - status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1)) - status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/" - status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/" - status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/" - - status_section "RPi Bootstrap And DNS" - status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1)) - status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1)) - status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1)) - status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip - status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok - - status_section "Pimox And Cluster" - status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1)) - status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1)) - status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1)) - status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1)) - status_cascade_warn "No problem pods" status_no_problem_pods - - status_section "What Broke Signals" - status_cascade_warn "Backup freshness" backup_status - status_cascade_warn "Recent deployments healthy" status_recent_deployments - status_cascade_warn "Pod restart pressure" status_pod_restart_pressure - status_cascade_warn "Node pressure" status_node_pressure - status_cascade_warn "Disk usage" status_disk_pressure - status_cascade_warn "Traefik 5xx/404 signals" status_traefik_error_rates - status_cascade_warn "Gitea recent errors" status_gitea_recent_errors - - status_section "Platform And Apps" - status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1)) - status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1)) - status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1)) - status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server - - status_section "Edge And Public URLs" - status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1)) - status_cascade_check "Traefik LoadBalancer HTTP" status_http_reachable "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" || failures=$((failures + 1)) - status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1)) - status_cascade_check "Gitea public route" status_gitea_public_ok || failures=$((failures + 1)) - - if truthy "${JEANNIE_REPORT_MODE:-false}"; then - return "${failures}" - fi - - if ((failures > 0)); then - printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2 - return 1 - fi - - printf '\nStatus cascade passed.\n' -} - -backstage_brain_enabled() { - local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}" - local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}" - - [[ "${provider,,}" == "ollama" ]] && truthy "${enabled}" -} - -backstage_brain_note() { - local topic="$1" - local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" - local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" - local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}" - local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" - local context - local retrieved_context="" - - if ! backstage_brain_enabled; then - return 0 - fi - if ! command -v python3 >/dev/null 2>&1; then - return 0 - fi - - context="$(cat)" - if [[ -s "${index_dir}/index.json" ]]; then - retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)" - fi - if [[ -n "${retrieved_context}" ]]; then - context="${context} - -Retrieved homelab knowledge: -${retrieved_context}" - fi - BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true -import json -import os -import sys -import urllib.error -import urllib.request - -endpoint, model, timeout, topic = sys.argv[1:5] -context = os.environ.get("BACKSTAGE_CONTEXT", "").strip() -try: - timeout_seconds = int(timeout) -except ValueError: - timeout_seconds = 20 - -prompt = f"""You are the backstage helper for a personal homelab script named jeannie. -Be concise and operational. Do not invent facts. Use only the supplied context. -Return at most 5 bullets. Focus on likely cause, next command, and risk. - -Topic: {topic} - -Context: -{context} -""" - -payload = { - "model": model, - "prompt": prompt, - "stream": False, - "options": { - "temperature": 0.1, - "num_predict": 220, - }, -} - -try: - tags_request = urllib.request.Request(f"{endpoint.rstrip('/')}/api/tags") - with urllib.request.urlopen(tags_request, timeout=3) as response: - tags = json.loads(response.read().decode("utf-8")) - available = {item.get("name", "") for item in tags.get("models", [])} - if model not in available and f"{model}:latest" not in available: - print(f"\n== Backstage Notes ==\nOllama is reachable, but model '{model}' is not pulled. Run: ollama pull {model}") - raise SystemExit(0) - - request = urllib.request.Request( - f"{endpoint.rstrip('/')}/api/generate", - data=json.dumps(payload).encode("utf-8"), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(request, timeout=timeout_seconds) as response: - result = json.loads(response.read().decode("utf-8")) -except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError): - raise SystemExit(0) - -note = (result.get("response") or "").strip() -if note: - print("\n== Backstage Notes ==") - print(note) -PY -} - -status_systemd_units() { - local unit - - for unit in "$@"; do - if systemctl list-unit-files --no-legend "${unit}.service" 2>/dev/null | grep -q .; then - printf '%-28s %s\n' "${unit}" "$(systemctl is-active "${unit}" 2>/dev/null || true)" - else - printf '%-28s not-installed\n' "${unit}" - fi - done -} - -status_compose_stack() { - local name="$1" - local directory="$2" - - printf '\n-- Docker Compose: %s (%s)\n' "${name}" "${directory}" - if [[ ! -d "${directory}" ]]; then - printf 'missing directory\n' - return 0 - fi - if ! command -v docker >/dev/null 2>&1; then - printf 'docker not installed\n' - return 0 - fi - if ! sudo docker compose version >/dev/null 2>&1; then - printf 'docker compose unavailable\n' - return 0 - fi - - (cd "${directory}" && sudo docker compose ps) || true -} - -status_kubernetes() { - status_section "Kubernetes" - - if ! command -v kubectl >/dev/null 2>&1; then - printf 'kubectl not installed\n' - return 0 - fi - if [[ ! -s "${KUBECONFIG_PATH}" ]]; then - printf 'kubeconfig missing: %s\n' "${KUBECONFIG_PATH}" - return 0 - fi - if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1; then - printf 'API server not reachable through %s\n' "${KUBECONFIG_PATH}" - return 0 - fi - - status_run "nodes" kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide - status_kubernetes_problem_pods - status_run "traefik services" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik get svc,pods -o wide - status_run "website pods" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n website-production get pods -o wide -} - -status_kubernetes_problem_pods() { - local rows - - printf '\n-- pods not Running/Completed\n' - rows="$( - kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers -o wide | - awk '$4 != "Running" && $4 != "Completed" { print }' - )" - if [[ -n "${rows}" ]]; then - printf '%s\n' "${rows}" - else - printf 'none\n' - fi -} - -status_pimox_workers() { - local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" - local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" - local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" - local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" - local worker_count - local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}" - local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" - local index - local vmid - - status_section "Pimox Workers" - - worker_count="$(pimox_worker_count_effective)" - if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then - printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}" - return 0 - fi - if ((worker_count == 0)); then - printf 'worker count is 0\n' - return 0 - fi - - for ((index = 1; index <= worker_count; index++)); do - if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then - printf 'worker index %s skipped\n' "${index}" - continue - fi - vmid=$((worker_base_vmid + index - 1)) - printf 'VM %-6s ' "${vmid}" - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}'" 2>/dev/null || - printf 'unreachable or missing\n' - done -} - -status_rpi_services() { - local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" - local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}" - local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}" - local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}" - local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}" - local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}" - local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}" - local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}" - local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}" - - status_section "RPi Services" - ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" " -set +e -dns_query() { - server=\"\$1\" - port=\"\$2\" - name=\"\$3\" - python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY' -import random -import socket -import struct -import sys - -server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.') -query_id = random.randrange(0, 65536) -packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0) -for label in name.split('.'): - packet += bytes([len(label)]) + label.encode('ascii') -packet += b'\x00' + struct.pack('!HH', 1, 1) -sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) -sock.settimeout(5) -sock.sendto(packet, (server, port)) -data, _ = sock.recvfrom(512) -if len(data) < 12: - raise SystemExit('short DNS response') -response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12]) -rcode = flags & 0x000f -if response_id != query_id: - raise SystemExit('mismatched DNS response id') -if rcode != 0: - raise SystemExit(f'DNS rcode {rcode}') -if an < 1: - raise SystemExit('DNS response had no answers') -print(f'ok answers={an}') -PY -} -check() { - label=\"\$1\" - shift - printf '%-34s ' \"\$label\" - output=\"\$("\$@" 2>&1)\" - status=\$? - if [ \"\$status\" -eq 0 ]; then - printf 'ok' - if [ -n \"\$output\" ]; then - printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\" - fi - printf '\n' - else - printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\" - fi - return 0 -} -docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\" -printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\" -if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then - if mountpoint -q '${docker_nvme_root}'; then - printf '%-34s ok - nvme mount active\n' 'Docker root class' - else - printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class' - fi -elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then - printf '%-34s warn - fallback root active; expected ${docker_nvme_root}\n' 'Docker root class' -else - printf '%-34s warn - unexpected docker root\n' 'Docker root class' -fi -if [ -d '${install_dir}' ]; then - cd '${install_dir}' - sudo docker compose ps || true - unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\" - check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com - if [ -n \"\$unbound_ip\" ]; then - check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com - else - printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query' - fi - upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\" - if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then - printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config' - else - printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config' - fi - check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com - check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com - check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\" -else - echo 'RPi services install directory is missing: ${install_dir}' -fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}" -} - -status_report() { - local cascade_status=0 - local heal_mode="" - local status_json - local -a status_args=() - - require_debian_server "status" - - while (($# > 0)); do - case "$1" in - --heal-plan) - heal_mode="plan" - shift - ;; - --heal) - heal_mode="apply" - shift - ;; - *) - status_args+=("$1") - shift - ;; - esac - done - - if ! report_ui_parse_args "${status_args[@]}"; then - return 1 - fi - - if [[ -n "${heal_mode}" ]]; then - status_json="$(mktemp "${TMPDIR:-/tmp}/jeannie-status-json.XXXXXX")" - # shellcheck disable=SC2034 - REPORT_UI_JSON=true - # shellcheck disable=SC2034 - REPORT_UI_ONLY=problems - report_ui_begin - JEANNIE_REPORT_MODE=true status_cascade_report >/dev/null || cascade_status=$? - report_ui_render "Jeannie Status" >"${status_json}" - report_ui_cleanup - if [[ "${heal_mode}" == "plan" ]]; then - "${REPO_ROOT}/scripts/heal" plan --status-json "${status_json}" --ai - else - "${REPO_ROOT}/scripts/heal" apply --status-json "${status_json}" --ai - fi - rm -f "${status_json}" - return "${cascade_status}" - fi - - if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then - report_ui_begin - JEANNIE_REPORT_MODE=true status_cascade_report || cascade_status=$? - report_ui_render "Jeannie Status" - report_ui_cleanup - return "${cascade_status}" - fi - - status_cascade_report || cascade_status=$? - - if ! truthy "${LAB_STATUS_DETAILS:-false}" && [[ "${REPORT_UI_DETAILS}" != "true" ]]; then - return "${cascade_status}" - fi - - status_section "Host" - printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)" - printf 'date: %s\n' "$(date -Is)" - printf 'uptime: %s\n' "$(uptime -p 2>/dev/null || uptime)" - status_run "memory" free -h - status_run "disk" df -h / /data /data/openebs/local - - status_section "Systemd" - status_systemd_units ssh docker containerd kubelet tailscaled homelab-gitea-runner - - status_section "Docker" - if command -v docker >/dev/null 2>&1; then - status_run "docker containers" sudo docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' - else - printf 'docker not installed\n' - fi - status_compose_stack "gitea" "/data/homelab-gitea" - status_compose_stack "heimdall" "${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" - status_compose_stack "arr-stack" "${REPO_ROOT}/infra/arr-stack" - - status_kubernetes - status_pimox_workers - status_rpi_services - - status_section "Tailscale" - if command -v tailscale >/dev/null 2>&1; then - status_run "tailscale ip" tailscale ip -4 - status_run "tailscale peers" tailscale status - else - printf 'tailscale not installed\n' - fi - - status_section "HTTP" - status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" - status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" - status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" - status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" - status_http "heimdall local" "http://127.0.0.1:${LAB_HEIMDALL_HTTP_PORT:-8082}/" - status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" - status_http "arr radarr" "http://127.0.0.1:7878/" - status_http "arr sonarr" "http://127.0.0.1:8989/" - status_http "arr prowlarr" "http://127.0.0.1:9696/" - - return "${cascade_status}" -} - -check_edge_traefik_backend() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}" - - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ - "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${traefik_ip}:80/' | grep -Eq '^[234]'" -} - -check_edge_gitea_backend() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}" - local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" - - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ - "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${gitea_ts_ip}:${gitea_http_port}/' | grep -Eq '^[234]'" -} - -check_edge_heimdall_backend() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}}" - local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" - - ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ - "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${heimdall_host}:${heimdall_http_port}/' | grep -Eq '^[234]'" -} - -check_gitea_git_remote() { - git ls-remote gitea main >/dev/null -} - -check_gitea_ssh() { - local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}" - local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}" - - ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" 2>&1 | - grep -Eiq 'successfully authenticated|Hi |shell access is disabled' -} - -doctor_edge() { - local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" - local edge_user="${LAB_EDGE_USER:-ubuntu}" - local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}" - local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}" - local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" - local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${gitea_ts_ip}}" - local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" - local failures=0 - - require_debian_server "doctor-edge" - if ! report_ui_parse_args "$@"; then - return 1 - fi - - if [[ "${REPORT_UI_VERBOSE}" != "true" ]]; then - report_ui_begin - report_ui_check fail "Public URLs" "Website public" "./jeannie doctor-edge --verbose" "" "" status_http_ok "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" || failures=$((failures + 1)) - report_ui_check fail "Public URLs" "Gitea public" "./jeannie doctor-gitea" "" "" status_gitea_public_ok || failures=$((failures + 1)) - report_ui_check fail "Edge" "OCI edge SSH" "./jeannie preflight" "" "" check_edge_ssh || failures=$((failures + 1)) - report_ui_check fail "Backends" "Traefik from edge" "./jeannie doctor-cluster" "" "" check_edge_traefik_backend || failures=$((failures + 1)) - report_ui_check fail "Backends" "Gitea from edge" "./jeannie doctor-gitea" "" "" check_edge_gitea_backend || failures=$((failures + 1)) - report_ui_check fail "Backends" "Heimdall from edge" "./jeannie deploy-heimdall" "" "" check_edge_heimdall_backend || failures=$((failures + 1)) - report_ui_render "Doctor Edge" - report_ui_cleanup - return "${failures}" - fi - - status_section "Doctor Edge" - status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" - status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" - status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" - - printf '\n-- edge backend reachability\n' - if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " -set +e -printf '%-28s ' 'traefik backend' -curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable -printf '%-28s ' 'gitea backend' -curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable -printf '%-28s ' 'heimdall backend' -curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${heimdall_host}:${heimdall_http_port}/' || echo unreachable -printf '\n-- tailscale\n' -tailscale status || true -"; then - printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}" - fi - - cat <<'EOF' - -Next steps: -- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster. -- If only Gitea is unreachable, run ./jeannie doctor-gitea. -- If only Heimdall is unreachable, run ./jeannie deploy-heimdall. -- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge. -- Runbook: docs/runbooks/edge-failures.md -EOF - - backstage_brain_note "doctor-edge" </dev/null 2>&1; then - sudo crictl ps -a | head -50 || true - else - printf 'crictl not installed\n' - fi - - cat <<'EOF' - -Next steps: -- If API server is unreachable after a stop, run ./jeannie start-cluster. -- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node. -- If Pimox workers are stopped, start them with ./jeannie start-cluster. -- If kubelet versions drift, run ./jeannie doctor-versions. -- Runbook: docs/runbooks/cluster-stop-start-failures.md -EOF - - backstage_brain_note "doctor-cluster" <&2 <<'EOF' -nuke destroys Kubernetes state and Pimox worker VMs. -Rerun with LAB_CONFIRM_NUKE=homelab when that destructive action is intended. -EOF - exit 1 - fi - - echo "Brutally nuking the homelab infrastructure..." - cluster_worker_targets - - if ((${#CLUSTER_WORKER_TARGETS[@]} > 0)); then - mkdir -p "$(dirname "${worker_known_hosts_file}")" - touch "${worker_known_hosts_file}" - chmod 0600 "${worker_known_hosts_file}" - fi - - echo "--> Terminating local OpenTofu tasks..." - killall tofu terraform 2>/dev/null || true - - echo "--> Eviscerating local Kubernetes components..." - cleanup_node - sudo rm -f "${KUBECONFIG_PATH}" - - for target in "${CLUSTER_WORKER_TARGETS[@]}"; do - echo "--> Eviscerating remote Kubernetes components (${target})..." - target_host="${target#*@}" - target_host="${target_host%%:*}" - ssh-keygen -R "${target_host}" -f "${worker_known_hosts_file}" >/dev/null 2>&1 || true - if ! ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${worker_known_hosts_file}" "${target}" "bash -s" <<'EOF' -set -euo pipefail - -cleanup_calico_links() { - ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true - sudo ip link delete vxlan.calico 2>/dev/null || true - sudo ip link delete tunl0 2>/dev/null || true - sudo ip link delete cni0 2>/dev/null || true - sudo ip link delete kube-ipvs0 2>/dev/null || true - ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true -} - -cleanup_iptables() { - sudo iptables -F || true - sudo iptables -X || true - sudo iptables -t nat -F || true - sudo iptables -t nat -X || true - sudo iptables -t mangle -F || true - sudo iptables -t mangle -X || true - sudo iptables -t raw -F || true - sudo iptables -t raw -X || true - if command -v ipvsadm >/dev/null 2>&1; then - sudo ipvsadm --clear || true - fi -} - -cleanup_calico_runtime_files() { - local path - - for path in /run/calico /var/run/calico; do - if sudo test -e "${path}"; then - sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true - sudo rmdir "${path}" 2>/dev/null || true - fi - done -} - -restore_node_dns() { - sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf - if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then - sudo rm -f /etc/resolv.conf - sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf - fi - sudo systemctl restart systemd-resolved 2>/dev/null || true -} - -cleanup_mounts() { - if command -v findmnt >/dev/null 2>&1; then - local mount_root - while IFS= read -r mountpoint; do - sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true - done < <( - for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do - findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true - done | sort -ru - ) - fi - while IFS= read -r mountpoint; do - sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true - done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) - sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true -} - -sudo kubeadm reset --force || true -sudo systemctl stop kubelet 2>/dev/null || true -sudo systemctl stop containerd 2>/dev/null || true -sudo killall containerd-shim-runc-v2 2>/dev/null || true - -cleanup_mounts - -sudo rm -rf \ - /etc/kubernetes/ \ - /var/lib/etcd/ \ - /var/lib/kubelet/ \ - /var/lib/cni/ \ - /etc/cni/net.d \ - /run/flannel \ - /var/lib/calico \ - /var/log/calico \ - /var/lib/containerd/* \ - /run/containerd/* \ - /etc/containerd/certs.d \ - /etc/containerd/config.toml -cleanup_calico_runtime_files -sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam - -cleanup_iptables -cleanup_calico_links -restore_node_dns - -sudo mkdir -p /etc/containerd/certs.d -sudo systemctl reset-failed kubelet containerd 2>/dev/null || true -sudo systemctl start containerd 2>/dev/null || true -EOF - then - echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 - exit 1 - fi - done - - destroy_pimox_worker_vms - - docker buildx rm lab-builder 2>/dev/null || true - docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true - rm -f "${BUILDX_CONFIG}" || true - - echo "--> Backing up local OpenTofu tracking state files..." - backup_tofu_state - - echo "--> Deleting OpenTofu tracking state files..." - rm -rf "${REPO_ROOT}"/bootstrap/cluster/terraform.tfstate* - rm -f "${REPO_ROOT}"/bootstrap/cluster/.terraform.tfstate.lock.info - rm -rf "${REPO_ROOT}"/bootstrap/cluster/.terraform/ - rm -rf "${REPO_ROOT}"/bootstrap/platform/terraform.tfstate* - rm -f "${REPO_ROOT}"/bootstrap/platform/.terraform.tfstate.lock.info - rm -rf "${REPO_ROOT}"/bootstrap/platform/.terraform/ - rm -rf "${REPO_ROOT}"/bootstrap/apps/terraform.tfstate* - rm -f "${REPO_ROOT}"/bootstrap/apps/.terraform.tfstate.lock.info - rm -rf "${REPO_ROOT}"/bootstrap/apps/.terraform/ - rm -rf "${REPO_ROOT}"/bootstrap/edge/terraform.tfstate* - rm -f "${REPO_ROOT}"/bootstrap/edge/.terraform.tfstate.lock.info - rm -rf "${REPO_ROOT}"/bootstrap/edge/.terraform/ - rm -f "${REPO_ROOT}/.lab/pimox-workers.tsv" "${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json" - - echo "Destruction complete. Retained data under /data/openebs/local was left intact." -} - -ensure_sops_age_tools() { - local missing_packages=() - - if ! command -v age-keygen >/dev/null 2>&1; then - missing_packages+=(age) - fi - - if ((${#missing_packages[@]} > 0)); then - echo "Installing missing secret-management tools: ${missing_packages[*]}" - sudo apt-get update - sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" - fi - - # age must be present; it's small and apt-provided on Debian. - if ! command -v age-keygen >/dev/null 2>&1; then - echo "age-keygen is still unavailable after package installation." >&2 - exit 1 - fi - - # sops: not packaged on Debian trixie — fall back to the GitHub release binary. - if ! command -v sops >/dev/null 2>&1; then - echo "sops not found in the apt repos; installing from GitHub release..." - local sops_ver sops_url - sops_ver="$(curl -fsSL https://api.github.com/repos/getsops/sops/releases/latest | grep -oE '"tag_name":\s*"v[^"]+"' | head -1 | grep -oE 'v[0-9]+\.[0-9]+\.[0-9]+')" - if [[ -z "${sops_ver}" ]]; then - echo "Could not determine the latest SOPS release version." >&2 - exit 1 - fi - sops_url="https://github.com/getsops/sops/releases/download/${sops_ver}/sops-v${sops_ver#v}.linux.amd64" - curl -fsSL -o "/tmp/sops-${sops_ver}" "${sops_url}" && \ - sudo install -m 0755 "/tmp/sops-${sops_ver}" /usr/local/bin/sops && \ - rm -f "/tmp/sops-${sops_ver}" - fi - if ! command -v sops >/dev/null 2>&1; then - echo "sops is still unavailable after package installation." >&2 - exit 1 - fi -} - -sops_age_key_file() { - printf '%s\n' "${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}" -} - -sops_age_recipient() { - local key_file="$1" - - awk -F': ' '/^# public key:/ { print $2; exit }' "${key_file}" -} - -# Decrypt a SOPS-encrypted YAML of `KEY: value` pairs into the current env. -# Usage: sops_load_secrets -# Requires the age key (sops_age_key_file) to decrypt; fails loudly if missing. -sops_load_secrets() { - local file="${1:?sops_load_secrets: missing file}" - local key_file - local out - - if [[ ! -f "${file}" ]]; then - echo "Missing encrypted secret file: ${file} (run ./jeannie secrets-init then sops -e)." >&2 - return 1 - fi - if ! sops_available; then - echo "sops not installed. Run ./jeannie secrets-init first." >&2 +s not installed. Run ./jeannie secrets-init first." >&2 return 1 fi @@ -7732,132 +227,6 @@ sops_load_secrets() { fi } -sops_available() { - command -v sops >/dev/null 2>&1 -} - -secrets_init() { - local key_file - local key_dir - local recipient - local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" - local example_file="${REPO_ROOT}/.sops.yaml.example" - - require_debian_server "secrets-init" - ensure_sops_age_tools - - key_file="$(sops_age_key_file)" - key_dir="$(dirname "${key_file}")" - mkdir -p "${key_dir}" - chmod 700 "${key_dir}" - - if [[ ! -s "${key_file}" ]]; then - echo "Generating age identity at ${key_file}..." - age-keygen -o "${key_file}" - chmod 600 "${key_file}" - else - echo "Using existing age identity at ${key_file}." - chmod 600 "${key_file}" - fi - - recipient="$(sops_age_recipient "${key_file}")" - if [[ -z "${recipient}" ]]; then - echo "Could not read the public recipient from ${key_file}." >&2 - exit 1 - fi - - if [[ -e "${config_file}" ]]; then - if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then - echo "${config_file} still contains the placeholder age recipient." >&2 - exit 1 - fi - echo "SOPS config already exists: ${config_file}" - else - if [[ ! -s "${example_file}" ]]; then - echo "Missing ${example_file}" >&2 - exit 1 - fi - sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}" - echo "Wrote ${config_file} with Debian host age recipient ${recipient}." - fi - - echo "Private age key: ${key_file}" - echo "Public age recipient: ${recipient}" - echo "Review and commit ${config_file}; never commit ${key_file}." -} - -secrets_check_file() { - local file="$1" - local key_file="$2" - - case "${file}" in - *.json) - grep -q '"sops"' "${file}" || { - echo "${file} does not look SOPS-encrypted." >&2 - return 1 - } - ;; - *) - grep -q '^sops:' "${file}" || { - echo "${file} does not look SOPS-encrypted." >&2 - return 1 - } - ;; - esac - - if [[ -s "${key_file}" ]]; then - SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null - fi -} - -secrets_check() { - local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" - local key_file - local recipient="" - local failures=0 - local found_files=0 - local file - - key_file="$(sops_age_key_file)" - - if [[ ! -s "${config_file}" ]]; then - echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2 - failures=$((failures + 1)) - elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then - echo "${config_file} still contains the placeholder age recipient." >&2 - failures=$((failures + 1)) - fi - - if [[ -s "${key_file}" ]]; then - recipient="$(sops_age_recipient "${key_file}")" - if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then - echo "${config_file} does not include this host's age recipient ${recipient}." >&2 - failures=$((failures + 1)) - fi - else - echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting." - fi - - if command -v git >/dev/null 2>&1; then - while IFS= read -r file; do - [[ -n "${file}" ]] || continue - found_files=$((found_files + 1)) - secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1)) - done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json') - fi - - if ((failures > 0)); then - echo "Secret checks failed with ${failures} issue(s)." >&2 - exit 1 - fi - - if ((found_files == 0)); then - echo "SOPS config checks passed. No encrypted secret files are committed yet." - else - echo "SOPS config checks passed for ${found_files} encrypted secret file(s)." - fi -} - tailnet_policy_check() { "${REPO_ROOT}/scripts/validate-tailnet-policy" } @@ -7907,17 +276,7 @@ ai_check() { if backstage_brain_enabled; then if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}" - if python3 - "${endpoint}" "${model}" <<'PY' -import json -import sys -import urllib.request - -endpoint, model = sys.argv[1:3] -with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response: - tags = json.loads(response.read().decode("utf-8")) -models = {item.get("name", "") for item in tags.get("models", [])} -raise SystemExit(0 if model in models or f"{model}:latest" in models else 1) -PY + if python3 "${REPO_ROOT}/scripts/ai-model-check.py" "${endpoint}" "${model}" then printf '%-28s ok - %s\n' "ollama model" "${model}" else @@ -8024,666 +383,3 @@ Useful practice trigger: kubectl -n security-lab exec deploy/juice-shop -- sh -c 'id; uname -a' EOF } - -security_attack_path() { - require_debian_server "security-attack-path" - "${REPO_ROOT}/scripts/security-attack-path" "${@:2}" -} - -prompt_injection_lab() { - "${REPO_ROOT}/scripts/prompt-injection-lab" "${@:2}" -} - -incident_commander() { - "${REPO_ROOT}/scripts/incident-commander" "${@:2}" -} - -safety_case() { - "${REPO_ROOT}/scripts/safety-case" "${@:2}" -} - -agent_sandbox() { - "${REPO_ROOT}/scripts/agent-sandbox" "${@:2}" -} - -ai_scheduler() { - "${REPO_ROOT}/scripts/ai-scheduler" "${@:2}" -} - -ai_memory() { - "${REPO_ROOT}/scripts/ai-memory" "${@:2}" -} - -red_blue_loop() { - "${REPO_ROOT}/scripts/red-blue-loop" "${@:2}" -} - -promote() { - "${REPO_ROOT}/scripts/promote" "${@:2}" -} - -model_observe() { - "${REPO_ROOT}/scripts/model-observe" "${@:2}" -} - -heal() { - require_debian_server "heal" - "${REPO_ROOT}/scripts/heal" "${@:2}" -} - -validate_homelab() { - "${REPO_ROOT}/scripts/validate-homelab" -} - -access_audit() { - require_debian_server "access-audit" - "${REPO_ROOT}/scripts/access-audit" -} - -drill_restore() { - require_debian_server "drill-restore" - "${REPO_ROOT}/scripts/restore-drill" all -} - -drill_pihole_restore() { - "${REPO_ROOT}/scripts/restore-drill" pihole -} - -kubeconfig_readonly() { - require_debian_server "kubeconfig-readonly" - "${REPO_ROOT}/scripts/kubeconfig-readonly" -} - -capacity_report() { - require_debian_server "capacity" - "${REPO_ROOT}/scripts/capacity-report" "${@:2}" -} - -recover_plan() { - require_debian_server "recover-plan" - "${REPO_ROOT}/scripts/recover-plan" -} - -gitops_status() { - require_debian_server "gitops-status" - "${REPO_ROOT}/scripts/gitops-status" -} - -cert_check() { - "${REPO_ROOT}/scripts/cert-check" -} - -release_snapshot() { - require_debian_server "release-snapshot" - "${REPO_ROOT}/scripts/release-snapshot" -} - -backup_status() { - require_debian_server "backup-status" - "${REPO_ROOT}/scripts/backup-status" -} - -synthetic_checks() { - require_debian_server "synthetic-checks" - "${REPO_ROOT}/scripts/synthetic-checks" -} - -resource_budget() { - require_debian_server "resource-budget" - "${REPO_ROOT}/scripts/resource-budget" "${@:2}" -} - -capacity_advisor() { - require_debian_server "capacity-advisor" - "${REPO_ROOT}/scripts/capacity-advisor" "${@:2}" -} - -capacity_limits() { - require_debian_server "capacity-limits" - "${REPO_ROOT}/scripts/capacity-limits" "${@:2}" -} - -control_plane() { - require_debian_server "control-plane" - "${REPO_ROOT}/scripts/control-plane" "${@:2}" -} - -artifact_cache() { - require_debian_server "artifact-cache" - "${REPO_ROOT}/scripts/artifact-cache" "${@:2}" -} - -blockchain_devnet() { - require_debian_server "blockchain-devnet" - "${REPO_ROOT}/scripts/blockchain-devnet" "${@:2}" -} - -blockchain_test() { - "${REPO_ROOT}/scripts/blockchain-test" "${@:2}" -} - -blockchain_wallet() { - "${REPO_ROOT}/scripts/blockchain-wallet" "${@:2}" -} - -golden_ledger() { - "${REPO_ROOT}/scripts/golden-ledger" "${@:2}" -} - -route_inventory() { - require_debian_server "route-inventory" - "${REPO_ROOT}/scripts/route-inventory" -} - -explain() { - if [ "${2:-}" != "output" ]; then - require_debian_server "explain" - fi - "${REPO_ROOT}/scripts/explain" "${@:2}" -} - -recover_power() { - require_debian_server "recover-power" - record_change_journal "recover-power" "$@" - "${REPO_ROOT}/scripts/recover-power" "${@:2}" -} - -scorecard() { - require_debian_server "scorecard" - "${REPO_ROOT}/scripts/scorecard" -} - -grafana_dashboards() { - require_debian_server "grafana-dashboards" - "${REPO_ROOT}/scripts/grafana-dashboards" "${@:2}" -} - -workers_manage() { - require_debian_server "workers" - "${REPO_ROOT}/scripts/workers" "${@:2}" -} - -record_change_journal() { - local command_name="$1" - shift || true - - "${REPO_ROOT}/scripts/change-journal" append "${command_name}" "$*" || true -} - -change_journal() { - "${REPO_ROOT}/scripts/change-journal" "${@:2}" -} - -homelab_map() { - "${REPO_ROOT}/scripts/homelab-map" "${@:2}" -} - -print_usage() { - cat <<'EOF' -Usage: ./jeannie [args] - -Base, Inventory, And Planning - help | -h | --help Show this grouped help. - validate Run repo validation checks. - inventory-check Validate the canonical homelab inventory. - preflight Run non-mutating infrastructure preflight checks. - doctor-preapply Run deeper pre-apply safety checks. - plan [all|provisioning|cluster|platform|apps|edge] - Run OpenTofu plans without applying changes. - -Build And Bootstrap - up Deploy the full homelab pipeline. - deploy-gitea Deploy the Debian-hosted Gitea container. - deploy-heimdall Deploy the Debian-hosted Heimdall dashboard. - deploy-matrix Deploy the Matrix homeserver (Synapse + Element + Postgres). - deploy-omniroute Deploy the OmniRoute LLM routing gateway. - deploy-hermes Deploy the Hermes agent (git-install + venv + gateway service). - bootstrap-gitea-repo Ensure the Gitea repo and SSH key wiring exist. - rpi-services Deploy Pi-hole, Unbound, and Uptime Kuma on RPi4. - ollama-setup Install/configure Ollama on the Debian host. - artifact-cache {status|up|down|instructions} - Manage optional Debian artifact caches. - blockchain-devnet {status|up|down|logs|rpc} - Manage the local Ethereum Anvil devnet. - blockchain-test [forge args...] Run Foundry tests for labs/blockchain. - blockchain-wallet {instructions|new|address|sign-message} - Practice dev wallet and signing workflows. - golden-ledger {show|check} Show or validate Pimox golden image versions. - -Cluster Lifecycle - rebuild-cluster Recreate the cluster through the guarded path. - stop-cluster Stop Kubernetes and worker VMs without destroy. - start-cluster Start Kubernetes and desired worker VMs. - workers - Manage Pimox/Kubernetes workers. - move-prometheus-stack-workers Move monitoring workloads to worker nodes. - doctor-versions [--all|--details|--verbose|--json] - Check Kubernetes/container tooling versions. - -Operator View And Reports - status [--all|--details|--verbose|--json] Problems-first cascade from host to public URLs. - status --heal-plan Build a guarded self-heal plan from status. - status --heal Dry-run auto-eligible self-heal actions. - scorecard Compact pass/warn/fail operator scorecard. - capacity Compact capacity and placement report. - capacity-advisor Decide whether to add VMs or fix placement/resources. - capacity-limits [namespace] Recommend requests/limits YAML snippets. - recover-plan Print disaster recovery order and prerequisites. - recover-power [--dry-run] Run or preview post-outage recovery. - heal {plan|apply} Plan/apply guarded self-healing actions. - incident {list|replay|triage} Classify incident text and suggest read-only next steps. - safety-case [--since REF] [PATH...] Generate risk, blast-radius, test, and rollback case. - agent-sandbox {check|policy} Classify proposed commands against agent policy. - impact [--since REF] [PATH...] Explain affected lab areas before changes. - review-last-change [REF] Summarize changed files, impact, validation. - map [--dot] Print the homelab dependency map. - change-journal {list|path} Show risky-command journal entries. - release-snapshot Write a pre-change release snapshot. - -Observability And GitOps - grafana-dashboards {list|apply} List or apply repo-managed Grafana dashboards. - gitops-status Print focused Argo CD health and sync status. - platform Deploy platform stack only. - edge Deploy/check the OCI edge stack only. - promote {plan|validate|rollback APP} Run canary promotion gates and rollback plan. - cert-check Check public DNS, TLS, and edge URL health. - backup-status Check backup and restore-drill freshness. - synthetic-checks Run end-to-end service probes. - resource-budget Report Kubernetes resource request/limit gaps. - route-inventory Report routes and Uptime Kuma coverage. - control-plane {status|taint|untaint|pods} - Keep normal pods off the Debian control plane. - explain |output Explain warnings/errors and fix commands. - -Recovery And State - backup-gitea Back up the Debian-hosted Gitea data. - drill-restore Run all restore drills. - drill-gitea-restore Run the Gitea restore drill. - drill-pihole-restore Run the Pi-hole restore drill. - state-backup Back up local OpenTofu state files. - -Focused Doctors - doctor-edge [--all|--details|--verbose|--json] - Diagnose edge routing and public exposure. - doctor-gitea [--all|--details|--verbose|--json] - Diagnose Gitea local/public access. - doctor-rpi [--all|--details|--verbose|--json] - Diagnose RPi services and DNS. - doctor-cluster [--all|--details|--verbose|--json] - Diagnose Kubernetes cluster health. - -Access, Secrets, And Policy - access-audit Audit SSH, Gitea, Kubernetes, and Tailscale access. - kubeconfig-readonly Create/check read-only kubeconfig material. - secrets-init Initialize SOPS/age secret tooling. - secrets-check Validate repo-managed secret prerequisites. - tailnet-policy-check Validate Tailscale ACL policy as code. - fix-debian-docker-root Repair Debian Docker root placement. - -Apps And Services - apps Deploy application stack only. - website-translation-model Prepare website translation model support. - website-ollama-listen Configure website Ollama access. - install-gitea-runner [TOKEN] Install the Gitea Actions runner. - openwrt Deploy/check OpenWrt lab config. - -Security Learning - security-scan Run the security scan bundle. - security-prepare Prepare defensive security tools. - security-zap Run OWASP ZAP checks. - security-k8s Run Kubernetes security checks. - security-host Run host security checks. - security-trivy Run Trivy scans. - security-secrets Run secret leak checks. - security-nuclei Run Nuclei web checks. - security-web Run web security checks. - security-logs Review security-relevant logs. - security-runtime Check runtime security sensors. - security-attack-path Print prioritized attack-path findings. - prompt-injection-lab {list|run|show} Run local prompt-injection defense drills. - red-blue {plan|run|ledger} Run or plan safe red-team/blue-team loop. - -AI And Indexing - ai-index Build the local homelab RAG index. - ai-check Query/check the local AI index. - ask [--citations] QUESTION... Find relevant docs/runbooks/commands. - ai-evals {list|run|show} Run deterministic Jeannie/RAG eval cases. - ai-scheduler {simulate|recommend NAME} Simulate AI workload placement policy. - ai-memory {check} Check local RAG index provenance freshness. - model-observe {prompts|run|report} Track model behavior over stable prompts. - -Destructive - nuke Guarded cluster state destruction path. -EOF -} - -if [[ "${JEANNIE_LIBRARY_MODE:-false}" == "true" ]]; then - # shellcheck disable=SC2317 - return 0 2>/dev/null || exit 0 -fi - -case "${1:-}" in - __up-task) - up_task "${2:-}" - ;; - "" | help | -h | --help) - print_usage - ;; - up) - record_change_journal "up" "$@" - up - ;; - plan) - plan_homelab "${2:-all}" - ;; - rebuild-cluster) - record_change_journal "rebuild-cluster" "$@" - rebuild_cluster - ;; - stop-cluster) - record_change_journal "stop-cluster" "$@" - stop_cluster - ;; - start-cluster) - record_change_journal "start-cluster" "$@" - start_cluster - ;; - status) - status_report "${@:2}" - ;; - capacity) - capacity_report "$@" - ;; - capacity-advisor) - capacity_advisor "$@" - ;; - capacity-limits) - capacity_limits "$@" - ;; - recover-plan) - recover_plan - ;; - gitops-status) - gitops_status - ;; - platform) - record_change_journal "platform" "$@" - require_debian_server "platform" - jeannie_log_start "platform" - jeannie_step_plan 1 - run_step "Platform OpenTofu apply" platform_apply - echo "Log: ${JEANNIE_LOG_FILE}" - ;; - edge) - record_change_journal "edge" "$@" - edge_apply - ;; - promote) - promote "$@" - ;; - cert-check) - cert_check - ;; - release-snapshot) - record_change_journal "release-snapshot" "$@" - release_snapshot - ;; - backup-status) - backup_status - ;; - synthetic-checks) - synthetic_checks - ;; - resource-budget) - resource_budget "$@" - ;; - control-plane) - control_plane "$@" - ;; - artifact-cache) - artifact_cache "$@" - ;; - blockchain-devnet) - blockchain_devnet "$@" - ;; - blockchain-test) - blockchain_test "$@" - ;; - blockchain-wallet) - blockchain_wallet "$@" - ;; - golden-ledger) - golden_ledger "$@" - ;; - route-inventory) - route_inventory - ;; - explain) - explain "$@" - ;; - recover-power) - recover_power "$@" - ;; - heal) - heal "$@" - ;; - incident) - incident_commander "$@" - ;; - safety-case) - safety_case "$@" - ;; - agent-sandbox) - agent_sandbox "$@" - ;; - scorecard) - scorecard - ;; - grafana-dashboards) - grafana_dashboards "$@" - ;; - workers) - workers_manage "$@" - ;; - change-journal) - change_journal "$@" - ;; - map) - homelab_map "$@" - ;; - validate) - validate_homelab - ;; - access-audit) - access_audit - ;; - kubeconfig-readonly) - kubeconfig_readonly - ;; - apps) - record_change_journal "apps" "$@" - require_debian_server "apps" - jeannie_log_start "apps" - jeannie_step_plan 1 - run_step "Applications" apps - echo "Log: ${JEANNIE_LOG_FILE}" - ;; - website-translation-model) - website_translation_model - ;; - website-ollama-listen) - website_ollama_listen - ;; - ollama-setup) - ollama_setup - ;; - deploy-gitea) - record_change_journal "deploy-gitea" "$@" - deploy_gitea - ;; - deploy-heimdall) - record_change_journal "deploy-heimdall" "$@" - deploy_heimdall - ;; - deploy-matrix) - record_change_journal "deploy-matrix" "$@" - deploy_matrix - ;; - deploy-omniroute) - record_change_journal "deploy-omniroute" "$@" - deploy_omniroute - ;; - deploy-hermes) - record_change_journal "deploy-hermes" "$@" - deploy_hermes - ;; - rpi-services) - record_change_journal "rpi-services" "$@" - deploy_rpi_services - ;; - bootstrap-gitea-repo) - bootstrap_gitea_repo - ;; - backup-gitea) - backup_gitea - ;; - drill-restore) - drill_restore - ;; - drill-gitea-restore) - drill_gitea_restore - ;; - drill-pihole-restore) - drill_pihole_restore - ;; - install-gitea-runner) - install_gitea_runner "${2:-}" - ;; - move-prometheus-stack-workers) - move_prometheus_stack_workers - ;; - doctor-versions) - doctor_versions "${@:2}" - ;; - doctor-edge) - doctor_edge "${@:2}" - ;; - doctor-gitea) - doctor_gitea "${@:2}" - ;; - doctor-rpi) - doctor_rpi "${@:2}" - ;; - doctor-cluster) - doctor_cluster "${@:2}" - ;; - preflight) - homelab_preflight - ;; - doctor-preapply) - doctor_preapply - ;; - inventory-check) - inventory_check - ;; - state-backup) - backup_tofu_state - ;; - fix-debian-docker-root) - fix_debian_docker_root - ;; - secrets-init) - secrets_init - ;; - secrets-check) - secrets_check - ;; - tailnet-policy-check) - tailnet_policy_check - ;; - ai-index) - ai_index - ;; - ai-check) - ai_check - ;; - ask) - ask_homelab "$@" - ;; - ai-evals) - ai_evals "$@" - ;; - ai-scheduler) - ai_scheduler "$@" - ;; - ai-memory) - ai_memory "$@" - ;; - model-observe) - model_observe "$@" - ;; - impact) - impact "$@" - ;; - review-last-change) - review_last_change "$@" - ;; - security-scan) - security_scan - ;; - security-prepare) - security_prepare - ;; - security-zap) - security_zap - ;; - security-k8s) - security_k8s - ;; - security-host) - security_host - ;; - security-trivy) - security_trivy - ;; - security-secrets) - security_secrets - ;; - security-nuclei) - security_nuclei - ;; - security-web) - security_web - ;; - security-logs) - security_logs - ;; - security-runtime) - security_runtime - ;; - security-attack-path) - security_attack_path "$@" - ;; - prompt-injection-lab) - prompt_injection_lab "$@" - ;; - red-blue) - red_blue_loop "$@" - ;; - openwrt) - openwrt - ;; - nuke) - record_change_journal "nuke" "$@" - require_debian_server "nuke" - jeannie_log_start "nuke" - jeannie_step_plan 1 - run_step "Nuke homelab cluster state" nuke - echo "Log: ${JEANNIE_LOG_FILE}" - ;; - *) - printf 'Unknown command: %s\n\n' "$1" >&2 - print_usage >&2 - exit 1 - ;; -esac diff --git a/lib/jeannie/env.sh b/lib/jeannie/env.sh index 20d602b..45e4533 100644 --- a/lib/jeannie/env.sh +++ b/lib/jeannie/env.sh @@ -1,2 +1,313 @@ #!/usr/bin/env bash +load_homelab_inventory_defaults() { + local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}" + local rendered_defaults + + if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then + return 0 + fi + + rendered_defaults="$(python3 - "${inventory_file}" <<'PY' +import re +import json +import shlex +import sys + +inventory_file = sys.argv[1] +mapping = { + "domain.base": "LAB_DOMAIN", + "domain.public_url": "LAB_PUBLIC_URL", + "domain.gitea_url": "LAB_GITEA_ROOT_URL", + "network.lan_cidr": "LAB_LAN_CIDR", + "network.lan_ip_prefix": "LAB_LAN_IP_PREFIX", + "network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP", + "hosts.debian.user": "LAB_DEBIAN_USER", + "hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP", + "hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP", + "hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT", + "hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH", + "hosts.rpi4.user": "LAB_RPI_USER", + "hosts.rpi4.lan_ip": "LAB_RPI_HOST", + "hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP", + "hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT", + "hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT", + "hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT", + "hosts.opi5_pimox.user": "LAB_PIMOX_USER", + "hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST", + "hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE", + "hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE", + "hosts.oci_edge.user": "LAB_EDGE_USER", + "hosts.oci_edge.public_ip": "LAB_EDGE_HOST", + "hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR", + "services.gitea.http_port": "LAB_GITEA_HTTP_PORT", + "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", + "services.gitea.root_url": "LAB_GITEA_ROOT_URL", + "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", + "services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR", + "services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT", + "services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL", + "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", + "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS", + "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR", + "services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE", + "services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT", + "services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT", + "ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER", + "ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED", + "ai_gateway.url": "LAB_AI_GATEWAY_URL", + "ai_gateway.model": "LAB_AI_GATEWAY_MODEL", + "ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS", + "ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR", + "pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID", + "pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT", +} + +def flatten(document, prefix=""): + values = {} + if isinstance(document, dict): + for key, value in document.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict): + values.update(flatten(value, path)) + elif isinstance(value, list): + values[path] = ",".join(str(item) for item in value) + elif value is not None: + values[path] = str(value) + return values + + +def parse_simple_inventory(path): + stack = [] + values = {} + pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$") + + with open(path, encoding="utf-8") as handle: + for raw_line in handle: + if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "): + continue + match = pattern.match(raw_line.rstrip("\n")) + if not match: + continue + indent = len(match.group(1)) + key = match.group(2) + value = (match.group(3) or "").strip() + while stack and stack[-1][0] >= indent: + stack.pop() + current_path = ".".join([item[1] for item in stack] + [key]) + if value == "": + stack.append((indent, key)) + continue + if " #" in value: + value = value.split(" #", 1)[0].strip() + values[current_path] = value.strip("\"'") + return values + + +try: + import yaml +except ImportError: + values = parse_simple_inventory(inventory_file) +else: + with open(inventory_file, encoding="utf-8") as handle: + values = flatten(yaml.safe_load(handle) or {}) + +for path, env_name in mapping.items(): + value = values.get(path) + if value: + print(f": ${{{env_name}:={shlex.quote(value)}}}") + +debian_ip = values.get("hosts.debian.lan_ip") +gitea_http = values.get("services.gitea.http_port") +if debian_ip: + print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}") +if debian_ip and gitea_http: + print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}") +gitea_ts = values.get("hosts.debian.tailscale_ip") +if gitea_ts: + print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}") +subdomains = values.get("domain.subdomains") +if subdomains: + if isinstance(subdomains, str): + subdomain_values = [item.strip() for item in subdomains.split(",") if item.strip()] + else: + subdomain_values = list(subdomains) + if subdomain_values: + print(f": ${{LAB_ADDITIONAL_SERVER_NAMES_JSON:={shlex.quote(json.dumps(subdomain_values))}}}") +PY +)" + + if [[ -n "${rendered_defaults}" ]]; then + eval "${rendered_defaults}" + fi +} + +export_if_unset() { + local name="$1" + local value="$2" + + if [[ -z "${value}" ]]; then + return 0 + fi + if [[ -z "${!name:-}" ]]; then + printf -v "${name}" '%s' "${value}" + fi + # shellcheck disable=SC2163 + export "${name?}" +} + +export_homelab_inventory_tf_vars() { + export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}" + export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}" + export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}" + export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}" + export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}" + export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}" + export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}" + export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}" + export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}" + export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}" + export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}" + export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}" + export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}" + export_if_unset TF_VAR_additional_server_names "${LAB_ADDITIONAL_SERVER_NAMES_JSON:-}" + export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" + export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" + export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" + export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" + export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}" + export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" + export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}" + export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}" + export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}" +} + +sops_available() { + command -v sops >/dev/null 2>&1 +} + +secrets_init() { + local key_file + local key_dir + local recipient + local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" + local example_file="${REPO_ROOT}/.sops.yaml.example" + + require_debian_server "secrets-init" + ensure_sops_age_tools + + key_file="$(sops_age_key_file)" + key_dir="$(dirname "${key_file}")" + mkdir -p "${key_dir}" + chmod 700 "${key_dir}" + + if [[ ! -s "${key_file}" ]]; then + echo "Generating age identity at ${key_file}..." + age-keygen -o "${key_file}" + chmod 600 "${key_file}" + else + echo "Using existing age identity at ${key_file}." + chmod 600 "${key_file}" + fi + + recipient="$(sops_age_recipient "${key_file}")" + if [[ -z "${recipient}" ]]; then + echo "Could not read the public recipient from ${key_file}." >&2 + exit 1 + fi + + if [[ -e "${config_file}" ]]; then + if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then + echo "${config_file} still contains the placeholder age recipient." >&2 + exit 1 + fi + echo "SOPS config already exists: ${config_file}" + else + if [[ ! -s "${example_file}" ]]; then + echo "Missing ${example_file}" >&2 + exit 1 + fi + sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}" + echo "Wrote ${config_file} with Debian host age recipient ${recipient}." + fi + + echo "Private age key: ${key_file}" + echo "Public age recipient: ${recipient}" + echo "Review and commit ${config_file}; never commit ${key_file}." +} + +secrets_check_file() { + local file="$1" + local key_file="$2" + + case "${file}" in + *.json) + grep -q '"sops"' "${file}" || { + echo "${file} does not look SOPS-encrypted." >&2 + return 1 + } + ;; + *) + grep -q '^sops:' "${file}" || { + echo "${file} does not look SOPS-encrypted." >&2 + return 1 + } + ;; + esac + + if [[ -s "${key_file}" ]]; then + SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null + fi +} + +secrets_check() { + local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}" + local key_file + local recipient="" + local failures=0 + local found_files=0 + local file + + key_file="$(sops_age_key_file)" + + if [[ ! -s "${config_file}" ]]; then + echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2 + failures=$((failures + 1)) + elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then + echo "${config_file} still contains the placeholder age recipient." >&2 + failures=$((failures + 1)) + fi + + if [[ -s "${key_file}" ]]; then + recipient="$(sops_age_recipient "${key_file}")" + if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then + echo "${config_file} does not include this host's age recipient ${recipient}." >&2 + failures=$((failures + 1)) + fi + else + echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting." + fi + + if command -v git >/dev/null 2>&1; then + while IFS= read -r file; do + [[ -n "${file}" ]] || continue + found_files=$((found_files + 1)) + secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1)) + done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json') + fi + + if ((failures > 0)); then + echo "Secret checks failed with ${failures} issue(s)." >&2 + exit 1 + fi + + if ((found_files == 0)); then + echo "SOPS config checks passed. No encrypted secret files are committed yet." + else + echo "SOPS config checks passed for ${found_files} encrypted secret file(s)." + fi +} + diff --git a/lib/jeannie/toolchain.sh b/lib/jeannie/toolchain.sh index 20d602b..0a83004 100644 --- a/lib/jeannie/toolchain.sh +++ b/lib/jeannie/toolchain.sh @@ -1,2 +1,126 @@ #!/usr/bin/env bash +require_debian_server() { + local command_name="$1" + local os_id="" + + if [[ "$(uname -s)" != "Linux" ]]; then + echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2 + exit 1 + fi + + if [[ -r /etc/os-release ]]; then + os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)" + fi + + if [[ "${os_id}" != "debian" ]]; then + echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2 + exit 1 + fi +} + +tofu_state_has_resource() { + local stack="$1" + local resource_address="$2" + + tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1 +} + +helm_release_secret_exists() { + local namespace="$1" + local release_name="$2" + local secret_name + + secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \ + -l "owner=helm,name=${release_name}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)" + + [[ -n "${secret_name}" ]] +} + +kubernetes_resource_exists() { + local namespace="$1" + local resource_kind="$2" + local resource_name="$3" + + if [[ -n "${namespace}" ]]; then + kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 + return $? + fi + + kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1 +} + +adopt_tofu_helm_release() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local release_name="$4" + local has_state=false + local has_release_secret=false + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + has_state=true + fi + if helm_release_secret_exists "${namespace}" "${release_name}"; then + has_release_secret=true + fi + + if [[ "${has_state}" == "true" && "${has_release_secret}" == "false" ]]; then + echo "Removing stale Helm release state for ${namespace}/${release_name} from ${stack} (${resource_address}) because no Helm release secret exists..." + tofu -chdir="${REPO_ROOT}/${stack}" state rm "${resource_address}" + return 0 + fi + + if [[ "${has_state}" == "true" ]]; then + return 0 + fi + if [[ "${has_release_secret}" == "false" ]]; then + return 0 + fi + + echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}" +} + +adopt_tofu_kubernetes_resource() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local resource_kind="$4" + local resource_name="$5" + local import_id="$6" + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + return 0 + fi + if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then + return 0 + fi + + echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" +} + +adopt_tofu_kubernetes_manifest() { + local stack="$1" + local resource_address="$2" + local namespace="$3" + local kubectl_kind="$4" + local api_version="$5" + local manifest_kind="$6" + local resource_name="$7" + local import_id + + if tofu_state_has_resource "${stack}" "${resource_address}"; then + return 0 + fi + if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then + return 0 + fi + + import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}" + echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..." + tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}" +} + diff --git a/scripts/ai-model-check.py b/scripts/ai-model-check.py new file mode 100644 index 0000000..10fc4ec --- /dev/null +++ b/scripts/ai-model-check.py @@ -0,0 +1,12 @@ +import json +import sys +import urllib.request + +endpoint, model = sys.argv[1:3] +try: + with urllib.request.urlopen("{}/api/tags".format(endpoint.rstrip("/")), timeout=5) as response: + tags = json.loads(response.read().decode("utf-8")) + models = {item.get("name", "") for item in tags.get("models", [])} + sys.exit(0 if model in models or "{}:latest".format(model) in models else 1) +except Exception: + sys.exit(1)