5765 lines
208 KiB
Bash
5765 lines
208 KiB
Bash
#!/usr/bin/env bash
|
|
# shellcheck disable=SC2034,SC2043,SC2140
|
|
|
|
load_homelab_inventory_defaults() {
|
|
local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
|
|
local rendered_defaults
|
|
|
|
if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
rendered_defaults="$(python3 - "${inventory_file}" <<'PY'
|
|
import re
|
|
import json
|
|
import shlex
|
|
import sys
|
|
|
|
inventory_file = sys.argv[1]
|
|
mapping = {
|
|
"domain.base": "LAB_DOMAIN",
|
|
"domain.public_url": "LAB_PUBLIC_URL",
|
|
"domain.gitea_url": "LAB_GITEA_ROOT_URL",
|
|
"network.lan_cidr": "LAB_LAN_CIDR",
|
|
"network.lan_ip_prefix": "LAB_LAN_IP_PREFIX",
|
|
"network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP",
|
|
"hosts.debian.user": "LAB_DEBIAN_USER",
|
|
"hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP",
|
|
"hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP",
|
|
"hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT",
|
|
"hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH",
|
|
"hosts.rpi4.user": "LAB_RPI_USER",
|
|
"hosts.rpi4.lan_ip": "LAB_RPI_HOST",
|
|
"hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP",
|
|
"hosts.rpi4.docker_root": "LAB_RPI_DOCKER_ROOT",
|
|
"hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT",
|
|
"hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT",
|
|
"hosts.opi5_pimox.user": "LAB_PIMOX_USER",
|
|
"hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST",
|
|
"hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE",
|
|
"hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE",
|
|
"hosts.oci_edge.user": "LAB_EDGE_USER",
|
|
"hosts.oci_edge.public_ip": "LAB_EDGE_HOST",
|
|
"hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR",
|
|
"services.gitea.http_port": "LAB_GITEA_HTTP_PORT",
|
|
"services.gitea.ssh_port": "LAB_GITEA_SSH_PORT",
|
|
"services.gitea.root_url": "LAB_GITEA_ROOT_URL",
|
|
"services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL",
|
|
"services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR",
|
|
"services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT",
|
|
"services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL",
|
|
"services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT",
|
|
"services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS",
|
|
"services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR",
|
|
"services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE",
|
|
"services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT",
|
|
"services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT",
|
|
"ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER",
|
|
"ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED",
|
|
"ai_gateway.url": "LAB_AI_GATEWAY_URL",
|
|
"ai_gateway.model": "LAB_AI_GATEWAY_MODEL",
|
|
"ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS",
|
|
"ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR",
|
|
"pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID",
|
|
"pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT",
|
|
}
|
|
|
|
def flatten(document, prefix=""):
|
|
values = {}
|
|
if isinstance(document, dict):
|
|
for key, value in document.items():
|
|
path = f"{prefix}.{key}" if prefix else str(key)
|
|
if isinstance(value, dict):
|
|
values.update(flatten(value, path))
|
|
elif isinstance(value, list):
|
|
values[path] = ",".join(str(item) for item in value)
|
|
elif value is not None:
|
|
values[path] = str(value)
|
|
return values
|
|
|
|
|
|
def parse_simple_inventory(path):
|
|
stack = []
|
|
values = {}
|
|
pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$")
|
|
|
|
with open(path, encoding="utf-8") as handle:
|
|
for raw_line in handle:
|
|
if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "):
|
|
continue
|
|
match = pattern.match(raw_line.rstrip("\n"))
|
|
if not match:
|
|
continue
|
|
indent = len(match.group(1))
|
|
key = match.group(2)
|
|
value = (match.group(3) or "").strip()
|
|
while stack and stack[-1][0] >= indent:
|
|
stack.pop()
|
|
current_path = ".".join([item[1] for item in stack] + [key])
|
|
if value == "":
|
|
stack.append((indent, key))
|
|
continue
|
|
if " #" in value:
|
|
value = value.split(" #", 1)[0].strip()
|
|
values[current_path] = value.strip("\"'")
|
|
return values
|
|
|
|
|
|
try:
|
|
import yaml
|
|
except ImportError:
|
|
values = parse_simple_inventory(inventory_file)
|
|
else:
|
|
with open(inventory_file, encoding="utf-8") as handle:
|
|
values = flatten(yaml.safe_load(handle) or {})
|
|
|
|
for path, env_name in mapping.items():
|
|
value = values.get(path)
|
|
if value:
|
|
print(f": ${{{env_name}:={shlex.quote(value)}}}")
|
|
|
|
debian_ip = values.get("hosts.debian.lan_ip")
|
|
gitea_http = values.get("services.gitea.http_port")
|
|
if debian_ip:
|
|
print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}")
|
|
if debian_ip and gitea_http:
|
|
print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}")
|
|
gitea_ts = values.get("hosts.debian.tailscale_ip")
|
|
if gitea_ts:
|
|
print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}")
|
|
subdomains = values.get("domain.subdomains")
|
|
if subdomains:
|
|
if isinstance(subdomains, str):
|
|
subdomain_values = [item.strip() for item in subdomains.split(",") if item.strip()]
|
|
else:
|
|
subdomain_values = list(subdomains)
|
|
if subdomain_values:
|
|
print(f": ${{LAB_ADDITIONAL_SERVER_NAMES_JSON:={shlex.quote(json.dumps(subdomain_values))}}}")
|
|
PY
|
|
)"
|
|
|
|
if [[ -n "${rendered_defaults}" ]]; then
|
|
eval "${rendered_defaults}"
|
|
fi
|
|
}
|
|
|
|
export_if_unset() {
|
|
local name="$1"
|
|
local value="$2"
|
|
|
|
if [[ -z "${value}" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ -z "${!name:-}" ]]; then
|
|
printf -v "${name}" '%s' "${value}"
|
|
fi
|
|
# shellcheck disable=SC2163
|
|
export "${name?}"
|
|
}
|
|
|
|
export_homelab_inventory_tf_vars() {
|
|
export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}"
|
|
export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}"
|
|
export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}"
|
|
export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}"
|
|
export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}"
|
|
export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}"
|
|
export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}"
|
|
export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}"
|
|
export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}"
|
|
export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}"
|
|
export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}"
|
|
export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}"
|
|
export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}"
|
|
export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}"
|
|
export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}"
|
|
export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}"
|
|
export_if_unset TF_VAR_additional_server_names "${LAB_ADDITIONAL_SERVER_NAMES_JSON:-}"
|
|
export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}"
|
|
export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}"
|
|
export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}"
|
|
export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}"
|
|
export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}"
|
|
export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}"
|
|
export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}"
|
|
export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}"
|
|
export_if_unset TF_VAR_worker_tailscale_pod_egress_snat "${LAB_PIMOX_WORKER_TAILSCALE_POD_EGRESS_SNAT:-}"
|
|
}
|
|
|
|
require_debian_server() {
|
|
local command_name="$1"
|
|
local os_id=""
|
|
|
|
if [[ "$(uname -s)" != "Linux" ]]; then
|
|
echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -r /etc/os-release ]]; then
|
|
os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)"
|
|
fi
|
|
|
|
if [[ "${os_id}" != "debian" ]]; then
|
|
echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
tofu_state_has_resource() {
|
|
local stack="$1"
|
|
local resource_address="$2"
|
|
|
|
tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1
|
|
}
|
|
|
|
helm_release_secret_exists() {
|
|
local namespace="$1"
|
|
local release_name="$2"
|
|
local secret_name
|
|
|
|
secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \
|
|
-l "owner=helm,name=${release_name}" \
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
|
|
[[ -n "${secret_name}" ]]
|
|
}
|
|
|
|
kubernetes_resource_exists() {
|
|
local namespace="$1"
|
|
local resource_kind="$2"
|
|
local resource_name="$3"
|
|
|
|
if [[ -n "${namespace}" ]]; then
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
return $?
|
|
fi
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
}
|
|
|
|
adopt_tofu_helm_release() {
|
|
local stack="$1"
|
|
local resource_address="$2"
|
|
local namespace="$3"
|
|
local release_name="$4"
|
|
local has_state=false
|
|
local has_release_secret=false
|
|
|
|
if tofu_state_has_resource "${stack}" "${resource_address}"; then
|
|
has_state=true
|
|
fi
|
|
if helm_release_secret_exists "${namespace}" "${release_name}"; then
|
|
has_release_secret=true
|
|
fi
|
|
|
|
if [[ "${has_state}" == "true" && "${has_release_secret}" == "false" ]]; then
|
|
echo "Removing stale Helm release state for ${namespace}/${release_name} from ${stack} (${resource_address}) because no Helm release secret exists..."
|
|
tofu -chdir="${REPO_ROOT}/${stack}" state rm "${resource_address}"
|
|
return 0
|
|
fi
|
|
|
|
if [[ "${has_state}" == "true" ]]; then
|
|
return 0
|
|
fi
|
|
if [[ "${has_release_secret}" == "false" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..."
|
|
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}"
|
|
}
|
|
|
|
adopt_tofu_kubernetes_resource() {
|
|
local stack="$1"
|
|
local resource_address="$2"
|
|
local namespace="$3"
|
|
local resource_kind="$4"
|
|
local resource_name="$5"
|
|
local import_id="$6"
|
|
|
|
if tofu_state_has_resource "${stack}" "${resource_address}"; then
|
|
return 0
|
|
fi
|
|
if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..."
|
|
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
|
|
}
|
|
|
|
adopt_tofu_kubernetes_manifest() {
|
|
local stack="$1"
|
|
local resource_address="$2"
|
|
local namespace="$3"
|
|
local kubectl_kind="$4"
|
|
local api_version="$5"
|
|
local manifest_kind="$6"
|
|
local resource_name="$7"
|
|
local import_id
|
|
|
|
if tofu_state_has_resource "${stack}" "${resource_address}"; then
|
|
return 0
|
|
fi
|
|
if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then
|
|
return 0
|
|
fi
|
|
|
|
import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}"
|
|
echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..."
|
|
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
|
|
}
|
|
|
|
adopt_platform_existing_resources() {
|
|
local stack="bootstrap/platform"
|
|
|
|
adopt_tofu_helm_release "${stack}" "helm_release.calico_crds" "tigera-operator" "calico-crds"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.calico" "tigera-operator" "calico"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.openebs" "openebs" "openebs"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.metallb[\"enabled\"]" "metallb-system" "metallb"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.traefik[\"enabled\"]" "traefik" "traefik"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.argocd" "argocd" "argocd"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.kyverno" "kyverno" "kyverno"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.kyverno_policies" "kyverno" "kyverno-policies"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.loki" "monitoring" "loki"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.promtail" "monitoring" "promtail"
|
|
adopt_tofu_helm_release "${stack}" "helm_release.prometheus_stack" "monitoring" "prometheus-stack"
|
|
adopt_tofu_kubernetes_resource \
|
|
"${stack}" \
|
|
"kubernetes_storage_class_v1.openebs_hostpath_retain" \
|
|
"" \
|
|
"storageclass" \
|
|
"openebs-hostpath-retain" \
|
|
"openebs-hostpath-retain"
|
|
adopt_tofu_kubernetes_resource \
|
|
"${stack}" \
|
|
"kubernetes_namespace_v1.monitoring" \
|
|
"" \
|
|
"namespace" \
|
|
"monitoring" \
|
|
"monitoring"
|
|
}
|
|
|
|
adopt_apps_existing_resources() {
|
|
local stack="bootstrap/apps"
|
|
local namespace="${TF_VAR_argocd_namespace:-argocd}"
|
|
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["container-registry"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"container-registry"
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["website-production"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"website-production"
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["demos-static"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"demos-static"
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["heimdall"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"heimdall"
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["n8n"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"n8n"
|
|
adopt_tofu_kubernetes_manifest \
|
|
"${stack}" \
|
|
'kubernetes_manifest.argocd_application["supply-chain-policy"]' \
|
|
"${namespace}" \
|
|
"applications.argoproj.io" \
|
|
"argoproj.io/v1alpha1" \
|
|
"Application" \
|
|
"supply-chain-policy"
|
|
}
|
|
|
|
ensure_homelab_node_labels() {
|
|
local control_plane_node="${LAB_CONTROL_PLANE_NODE_NAME:-debian}"
|
|
local raspberry_node="${LAB_RASPBERRY_NODE_NAME:-raspberry}"
|
|
local prometheus_selector="homelab.dev/node-role=app,homelab.dev/storage=ssd"
|
|
local node
|
|
local target_nodes
|
|
|
|
echo "Applying homelab labels to existing Kubernetes nodes..."
|
|
while IFS= read -r node; do
|
|
[[ -n "${node}" ]] || continue
|
|
|
|
if [[ "${node}" == "${control_plane_node}" ]]; then
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
homelab.dev/node-role=control-plane \
|
|
homelab.dev/storage=local \
|
|
homelab.dev/workload-class=control-plane \
|
|
--overwrite
|
|
continue
|
|
fi
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
node-role.kubernetes.io/worker=worker \
|
|
--overwrite
|
|
|
|
if [[ "${node}" == pimox-worker-* ]]; then
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
homelab.dev/node-role=app \
|
|
homelab.dev/storage=ssd \
|
|
homelab.dev/workload-class=platform \
|
|
--overwrite
|
|
elif [[ "${node}" == "${raspberry_node}" ]]; then
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
homelab.dev/node-role=edge-app \
|
|
homelab.dev/storage=local \
|
|
homelab.dev/workload-class=edge \
|
|
--overwrite
|
|
fi
|
|
done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
|
|
target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)"
|
|
if [[ -z "${target_nodes}" ]]; then
|
|
echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
delete_prometheus_stack_storage() {
|
|
local namespace="${1:-monitoring}"
|
|
local pattern='(^|-)prometheus-stack-(prometheus|alertmanager|grafana)(-|$)|^prometheus-prometheus-stack|^alertmanager-prometheus-stack|^storage-prometheus-stack-grafana'
|
|
local pvc_names
|
|
local pv_names
|
|
|
|
pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null |
|
|
awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')"
|
|
pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \
|
|
-o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null |
|
|
awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')"
|
|
|
|
if [[ -n "${pvc_names}" ]]; then
|
|
echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..."
|
|
printf '%s\n' "${pvc_names}" |
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s
|
|
fi
|
|
|
|
if [[ -n "${pv_names}" ]]; then
|
|
echo "Deleting old prometheus-stack retained PV objects..."
|
|
printf '%s\n' "${pv_names}" |
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false
|
|
fi
|
|
}
|
|
|
|
run_tofu_stack() {
|
|
local stack="$1"
|
|
local auto_approve="${LAB_AUTO_APPROVE:-true}"
|
|
local -a apply_args=()
|
|
|
|
if [[ "${stack}" == "bootstrap/cluster" ]]; then
|
|
ensure_cluster_worker_var_file
|
|
fi
|
|
if [[ "${stack}" == "bootstrap/edge" ]]; then
|
|
ensure_edge_tailscale_routes
|
|
ensure_edge_haproxy_stats_password
|
|
fi
|
|
|
|
if truthy "${auto_approve}"; then
|
|
apply_args+=("-auto-approve")
|
|
elif ! disabled_value "${auto_approve}"; then
|
|
echo "LAB_AUTO_APPROVE must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
|
|
apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
|
|
fi
|
|
|
|
tofu -chdir="${REPO_ROOT}/${stack}" init
|
|
ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
|
|
if [[ "${stack}" == "bootstrap/platform" ]]; then
|
|
adopt_platform_existing_resources
|
|
fi
|
|
if [[ "${stack}" == "bootstrap/apps" ]]; then
|
|
adopt_apps_existing_resources
|
|
fi
|
|
ensure_kubernetes_api_ready_for_tofu_stack "${stack}"
|
|
tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}"
|
|
}
|
|
|
|
run_tofu_plan_stack() {
|
|
local stack="$1"
|
|
local -a plan_args=()
|
|
|
|
if [[ "${stack}" == "bootstrap/cluster" ]]; then
|
|
ensure_cluster_worker_var_file
|
|
fi
|
|
|
|
if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
|
|
plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
|
|
fi
|
|
|
|
tofu -chdir="${REPO_ROOT}/${stack}" init
|
|
tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}"
|
|
}
|
|
|
|
backup_tofu_state() {
|
|
local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}"
|
|
local timestamp
|
|
local archive
|
|
local -a paths=()
|
|
local path
|
|
|
|
timestamp="$(date +%Y%m%d-%H%M%S)"
|
|
archive="${backup_dir}/tofu-state-${timestamp}.tgz"
|
|
mkdir -p "${backup_dir}"
|
|
|
|
for path in \
|
|
bootstrap/provisioning/terraform.tfstate \
|
|
bootstrap/provisioning/terraform.tfstate.backup \
|
|
bootstrap/cluster/terraform.tfstate \
|
|
bootstrap/cluster/terraform.tfstate.backup \
|
|
bootstrap/platform/terraform.tfstate \
|
|
bootstrap/platform/terraform.tfstate.backup \
|
|
bootstrap/apps/terraform.tfstate \
|
|
bootstrap/apps/terraform.tfstate.backup \
|
|
bootstrap/edge/terraform.tfstate \
|
|
bootstrap/edge/terraform.tfstate.backup \
|
|
.lab/cluster-workers.auto.tfvars.json \
|
|
.lab/pimox-workers.tsv \
|
|
.lab/manual-workers.tsv; do
|
|
if [[ -e "${REPO_ROOT}/${path}" ]]; then
|
|
paths+=("${path}")
|
|
fi
|
|
done
|
|
|
|
if ((${#paths[@]} == 0)); then
|
|
echo "No local OpenTofu state files found to back up."
|
|
return 0
|
|
fi
|
|
|
|
tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}"
|
|
chmod 0600 "${archive}"
|
|
echo "Backed up local OpenTofu state to ${archive}."
|
|
}
|
|
|
|
move_prometheus_stack_workers() {
|
|
local stack="bootstrap/platform"
|
|
local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}"
|
|
local auto_approve="${LAB_AUTO_APPROVE:-true}"
|
|
local -a approve_args=()
|
|
|
|
require_debian_server "move-prometheus-stack-workers"
|
|
|
|
if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then
|
|
cat >&2 <<'EOF'
|
|
move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data.
|
|
Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended.
|
|
EOF
|
|
exit 1
|
|
fi
|
|
|
|
if truthy "${auto_approve}"; then
|
|
approve_args+=("-auto-approve")
|
|
elif ! disabled_value "${auto_approve}"; then
|
|
echo "LAB_AUTO_APPROVE must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
|
|
export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"
|
|
|
|
echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted."
|
|
ensure_homelab_node_labels
|
|
tofu -chdir="${REPO_ROOT}/${stack}" init
|
|
adopt_platform_existing_resources
|
|
tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}"
|
|
delete_prometheus_stack_storage "${namespace}"
|
|
tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}"
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
|
|
}
|
|
|
|
doctor_versions_report() {
|
|
local api_version
|
|
local api_minor
|
|
local containerd_major_count
|
|
local kubelet_minor_count
|
|
local node_rows
|
|
local status=0
|
|
|
|
require_debian_server "doctor-versions"
|
|
ensure_python3
|
|
|
|
api_version="$(
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
|
|
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
|
|
)"
|
|
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
|
|
node_rows="$(
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
|
|
)"
|
|
|
|
if [[ -z "${node_rows}" ]]; then
|
|
echo "No Kubernetes nodes found." >&2
|
|
return 1
|
|
fi
|
|
|
|
printf 'API server: %s\n\n' "${api_version}"
|
|
printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME"
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' '{
|
|
version = $2
|
|
gsub(/^v/, "", version)
|
|
split(version, parts, ".")
|
|
printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3
|
|
}'
|
|
|
|
kubelet_minor_count="$(
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
|
|
sort -u |
|
|
wc -l |
|
|
tr -d ' '
|
|
)"
|
|
containerd_major_count="$(
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
|
|
sort -u |
|
|
wc -l |
|
|
tr -d ' '
|
|
)"
|
|
|
|
if [[ "${kubelet_minor_count}" != "1" ]]; then
|
|
echo
|
|
echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2
|
|
status=1
|
|
fi
|
|
|
|
if printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo
|
|
echo "At least one kubelet does not match the API server minor (${api_minor})." >&2
|
|
status=1
|
|
fi
|
|
|
|
if [[ "${containerd_major_count}" -gt 1 ]]; then
|
|
echo
|
|
echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2
|
|
status=1
|
|
fi
|
|
|
|
if [[ "${status}" -eq 0 ]]; then
|
|
echo
|
|
echo "Node Kubernetes and containerd versions are aligned."
|
|
fi
|
|
|
|
return "${status}"
|
|
}
|
|
|
|
doctor_versions_compact() {
|
|
local api_version
|
|
local api_minor
|
|
local node_rows
|
|
local kubelet_minor_count
|
|
local containerd_major_count
|
|
local mismatch_count
|
|
local failures=0
|
|
|
|
ensure_python3
|
|
report_ui_begin
|
|
|
|
if ! status_kubernetes_api_ok; then
|
|
report_ui_emit fail "Kubernetes" "API" "not reachable" "kubectl readyz failed" "./jeannie start-cluster"
|
|
report_ui_render "Doctor Versions"
|
|
report_ui_cleanup
|
|
return 1
|
|
fi
|
|
|
|
api_version="$(
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
|
|
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
|
|
)"
|
|
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
|
|
node_rows="$(
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
|
|
)"
|
|
|
|
if [[ -z "${node_rows}" ]]; then
|
|
report_ui_emit fail "Kubernetes" "Nodes" "none found" "kubectl returned no node rows" "./jeannie doctor-cluster"
|
|
report_ui_render "Doctor Versions"
|
|
report_ui_cleanup
|
|
return 1
|
|
fi
|
|
|
|
kubelet_minor_count="$(
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
|
|
sort -u |
|
|
wc -l |
|
|
tr -d ' '
|
|
)"
|
|
mismatch_count="$(
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) count++ } END { print count + 0 }'
|
|
)"
|
|
containerd_major_count="$(
|
|
printf '%s\n' "${node_rows}" |
|
|
awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
|
|
sort -u |
|
|
wc -l |
|
|
tr -d ' '
|
|
)"
|
|
|
|
report_ui_emit ok "Kubernetes" "API version" "${api_version}"
|
|
if [[ "${kubelet_minor_count}" != "1" || "${mismatch_count}" != "0" ]]; then
|
|
report_ui_emit fail "Kubernetes" "Kubelet versions" "${mismatch_count} node(s) differ from ${api_minor}" "unique kubelet minors=${kubelet_minor_count}" "./jeannie doctor-versions --verbose"
|
|
failures=$((failures + 1))
|
|
else
|
|
report_ui_emit ok "Kubernetes" "Kubelet versions" "aligned to ${api_minor}"
|
|
fi
|
|
if [[ "${containerd_major_count}" -gt 1 ]]; then
|
|
report_ui_emit warn "Runtime" "Containerd versions" "mixed major versions" "unique majors=${containerd_major_count}" "./jeannie doctor-versions --verbose"
|
|
else
|
|
report_ui_emit ok "Runtime" "Containerd versions" "single major version"
|
|
fi
|
|
|
|
report_ui_render "Doctor Versions"
|
|
report_ui_cleanup
|
|
return "${failures}"
|
|
}
|
|
|
|
doctor_versions() {
|
|
require_debian_server "doctor-versions"
|
|
if ! report_ui_parse_args "$@"; then
|
|
return 1
|
|
fi
|
|
if [[ "${REPORT_UI_VERBOSE}" == "true" ]]; then
|
|
doctor_versions_report
|
|
exit "$?"
|
|
fi
|
|
doctor_versions_compact
|
|
exit "$?"
|
|
}
|
|
|
|
worker_index_is_skipped() {
|
|
local index="$1"
|
|
local skip_indexes="$2"
|
|
local skip_index
|
|
|
|
skip_indexes="${skip_indexes//,/ }"
|
|
for skip_index in ${skip_indexes}; do
|
|
[[ -z "${skip_index}" ]] && continue
|
|
if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2
|
|
exit 1
|
|
fi
|
|
if ((skip_index == index)); then
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
ensure_python3() {
|
|
if command -v python3 >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends python3
|
|
}
|
|
|
|
ensure_go_toolchain() {
|
|
require_debian_server "go toolchain"
|
|
|
|
if command -v go >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Installing Go toolchain for Jeannie core helpers..."
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends golang-go
|
|
|
|
if ! command -v go >/dev/null 2>&1; then
|
|
echo "go is still unavailable after installing golang-go." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
detect_route_interface() {
|
|
local target="$1"
|
|
|
|
ip route get "${target}" 2>/dev/null | awk '
|
|
{
|
|
for (i = 1; i <= NF; i++) {
|
|
if ($i == "dev") {
|
|
print $(i + 1)
|
|
exit
|
|
}
|
|
}
|
|
}
|
|
'
|
|
}
|
|
|
|
pimox_ssh() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
|
|
shift 3
|
|
ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@"
|
|
}
|
|
|
|
pimox_guest_ipv4() {
|
|
local guest_json
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local vmid="$4"
|
|
local ip_prefix="$5"
|
|
local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
|
|
|
|
guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)"
|
|
if [[ -z "${guest_json}" ]]; then
|
|
return 1
|
|
fi
|
|
|
|
GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
prefix = sys.argv[1]
|
|
try:
|
|
interfaces = json.loads(os.environ.get("GUEST_JSON", ""))
|
|
except Exception:
|
|
sys.exit(1)
|
|
|
|
for iface in interfaces or []:
|
|
for address in iface.get("ip-addresses") or []:
|
|
if address.get("ip-address-type") != "ipv4":
|
|
continue
|
|
ip = address.get("ip-address", "")
|
|
if not ip or ip.startswith(("127.", "169.254.")):
|
|
continue
|
|
if prefix and not ip.startswith(prefix):
|
|
continue
|
|
print(ip)
|
|
sys.exit(0)
|
|
sys.exit(1)
|
|
PY
|
|
}
|
|
|
|
pimox_worker_vm_debug() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local vmid="$4"
|
|
local qm_bin="$5"
|
|
local script
|
|
local encoded_script
|
|
|
|
script="$(cat <<'EOF'
|
|
set +e
|
|
echo "-- addresses --"
|
|
ip -br addr
|
|
echo "-- routes --"
|
|
ip route
|
|
echo "-- units --"
|
|
systemctl is-active ssh sshd 2>/dev/null || true
|
|
echo "-- listeners --"
|
|
ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true
|
|
echo "-- sshd config test --"
|
|
sudo sshd -t 2>&1 || true
|
|
echo "-- recent ssh logs --"
|
|
journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
|
|
EOF
|
|
)"
|
|
encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
|
|
pimox_ssh "${host}" "${user}" "${key_path}" "set +e
|
|
echo 'Pimox VM ${vmid} status:'
|
|
sudo '${qm_bin}' status '${vmid}'
|
|
echo 'Pimox VM ${vmid} config summary:'
|
|
sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
|
|
echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
|
|
sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true
|
|
echo 'Pimox VM ${vmid} guest SSH diagnostics:'
|
|
sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\" || true" >&2 || true
|
|
}
|
|
|
|
pimox_worker_guest_agent_recovery_hint() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local vmid="$3"
|
|
local qm_bin="$4"
|
|
|
|
cat >&2 <<EOF
|
|
|
|
QEMU guest agent did not become available for Pimox VM ${vmid}.
|
|
Fast checks:
|
|
ssh ${user}@${host} 'sudo ${qm_bin} status ${vmid}; sudo ${qm_bin} config ${vmid}; sudo ${qm_bin} guest cmd ${vmid} network-get-interfaces'
|
|
|
|
Recovery options:
|
|
1. If the clone is still booting slowly, rerun the Pimox stage with a longer agent wait:
|
|
LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS=600 ./jeannie up
|
|
2. If the same VM still reports "QEMU guest agent is not running", recreate the worker clone:
|
|
LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
|
|
3. If a recreated worker still has no guest agent, rebuild the template and worker:
|
|
LAB_RPI_SERVICES_DEPLOY=false LAB_PIMOX_TEMPLATE_REPLACE_EXISTING=true LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./jeannie up
|
|
EOF
|
|
}
|
|
|
|
pimox_guest_exec_exitcode() {
|
|
local guest_exec_json="$1"
|
|
|
|
GUEST_EXEC_JSON="${guest_exec_json}" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
try:
|
|
document = json.loads(os.environ.get("GUEST_EXEC_JSON", ""))
|
|
except Exception:
|
|
sys.exit(2)
|
|
|
|
try:
|
|
print(int(document.get("exitcode", 1)))
|
|
except Exception:
|
|
sys.exit(2)
|
|
PY
|
|
}
|
|
|
|
pimox_worker_guest_ssh_repair() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local vmid="$4"
|
|
local qm_bin="$5"
|
|
local script
|
|
local encoded_script
|
|
local repair_output
|
|
local repair_exitcode
|
|
|
|
script="$(cat <<'EOF'
|
|
set -eu
|
|
ssh_unit=""
|
|
if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
|
|
ssh_unit=ssh.service
|
|
elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then
|
|
ssh_unit=sshd.service
|
|
else
|
|
echo "Neither ssh.service nor sshd.service exists in the guest" >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo ssh-keygen -A
|
|
sudo install -d -m 0755 /run/sshd
|
|
sudo mkdir -p /etc/ssh/sshd_config.d
|
|
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
|
|
Port 22
|
|
ListenAddress 0.0.0.0
|
|
PubkeyAuthentication yes
|
|
PasswordAuthentication no
|
|
KbdInteractiveAuthentication no
|
|
PermitRootLogin no
|
|
UsePAM yes
|
|
SSHD_CONFIG
|
|
|
|
sudo sshd -t
|
|
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
|
|
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
|
|
sudo systemctl enable "$ssh_unit" >/dev/null
|
|
sudo systemctl restart "$ssh_unit"
|
|
sleep 2
|
|
sudo systemctl is-active "$ssh_unit"
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -ltnp 2>/dev/null || ss -ltn
|
|
ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
|
|
fi
|
|
EOF
|
|
)"
|
|
encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
|
|
if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
|
|
echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2
|
|
printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2
|
|
return 1
|
|
fi
|
|
|
|
printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2
|
|
if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then
|
|
return 0
|
|
fi
|
|
echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2
|
|
if [[ -n "${repair_exitcode:-}" ]]; then
|
|
echo "Guest exit code: ${repair_exitcode}" >&2
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
wait_for_pimox_guest_ssh() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local vmid="$4"
|
|
local guest_user="$5"
|
|
local guest_key_path="$6"
|
|
local ip_prefix="$7"
|
|
local timeout_seconds="$8"
|
|
local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
|
|
local deadline
|
|
local elapsed
|
|
local guest_ip
|
|
local ip_filter_description
|
|
local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
|
|
local last_guest_ip=""
|
|
local last_known_hosts_ip=""
|
|
local last_ssh_output=""
|
|
local last_host_probe_output=""
|
|
local host_probe_failures=0
|
|
local require_host_probe="${LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE:-false}"
|
|
local next_log
|
|
local ssh_deadline=0
|
|
local ssh_output
|
|
local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}"
|
|
local ssh_refused_since=0
|
|
local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}"
|
|
local ssh_repair_attempted=false
|
|
|
|
ip_filter_description="matching prefix ${ip_prefix}"
|
|
if [[ -z "${ip_prefix}" ]]; then
|
|
ip_filter_description="that is not loopback or link-local"
|
|
fi
|
|
if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then
|
|
echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
return 1
|
|
fi
|
|
if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then
|
|
echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
return 1
|
|
fi
|
|
if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
|
|
echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
|
|
return 1
|
|
fi
|
|
mkdir -p "$(dirname "${known_hosts_file}")"
|
|
touch "${known_hosts_file}"
|
|
chmod 0600 "${known_hosts_file}"
|
|
|
|
deadline=$((SECONDS + timeout_seconds))
|
|
next_log="${SECONDS}"
|
|
while ((SECONDS < deadline)); do
|
|
guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)"
|
|
if [[ -n "${guest_ip}" ]]; then
|
|
if ((ssh_deadline == 0)); then
|
|
ssh_deadline=$((SECONDS + ssh_timeout_seconds))
|
|
elif ((SECONDS >= ssh_deadline)); then
|
|
break
|
|
fi
|
|
last_guest_ip="${guest_ip}"
|
|
if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then
|
|
ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true
|
|
last_known_hosts_ip="${guest_ip}"
|
|
fi
|
|
if truthy "${require_host_probe}"; then
|
|
if last_host_probe_output="$(ping -c 1 -W 2 "${guest_ip}" 2>&1)"; then
|
|
host_probe_failures=0
|
|
else
|
|
host_probe_failures=$((host_probe_failures + 1))
|
|
last_ssh_output="Debian host cannot ping ${guest_ip}; latest probe: ${last_host_probe_output}"
|
|
if ((host_probe_failures >= 3)); then
|
|
echo "Worker VM ${vmid} reported guest IP ${guest_ip}, but the Debian host cannot reach it on the LAN." >&2
|
|
echo "Latest host probe: ${last_host_probe_output}" >&2
|
|
echo "Fast checks:" >&2
|
|
echo " ssh ${user}@${host} 'ip -br link show; bridge link; sudo qm config ${vmid}; sudo qm guest cmd ${vmid} network-get-interfaces'" >&2
|
|
echo " arping -c 3 ${guest_ip} || ping -c 3 ${guest_ip}" >&2
|
|
echo " Check bridge ${LAB_PIMOX_BRIDGE:-vmbr0}, switch/VLAN, duplicate IP, and firewall rules between Debian and Pimox workers." >&2
|
|
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
|
|
return 1
|
|
fi
|
|
sleep 10
|
|
continue
|
|
fi
|
|
fi
|
|
if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then
|
|
printf '%s\n' "${guest_ip}"
|
|
return 0
|
|
fi
|
|
last_ssh_output="${ssh_output}"
|
|
if [[ "${ssh_output}" == *"Connection refused"* ]]; then
|
|
if ((ssh_refused_since == 0)); then
|
|
ssh_refused_since="${SECONDS}"
|
|
fi
|
|
if [[ "${ssh_repair_attempted}" == "false" ]]; then
|
|
echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2
|
|
pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true
|
|
ssh_repair_attempted=true
|
|
elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then
|
|
echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2
|
|
echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2
|
|
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
|
|
return 1
|
|
fi
|
|
else
|
|
ssh_refused_since=0
|
|
fi
|
|
fi
|
|
|
|
if ((SECONDS >= next_log)); then
|
|
elapsed=$((timeout_seconds - (deadline - SECONDS)))
|
|
if [[ -n "${last_guest_ip}" ]]; then
|
|
echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2
|
|
if [[ -n "${last_ssh_output}" ]]; then
|
|
echo "Last SSH failure: ${last_ssh_output}" >&2
|
|
fi
|
|
else
|
|
echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2
|
|
fi
|
|
next_log=$((SECONDS + 60))
|
|
fi
|
|
sleep 10
|
|
done
|
|
|
|
if [[ -n "${last_guest_ip}" ]]; then
|
|
echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2
|
|
if [[ -n "${last_ssh_output}" ]]; then
|
|
echo "Last SSH failure: ${last_ssh_output}" >&2
|
|
fi
|
|
if [[ -n "${last_host_probe_output}" ]]; then
|
|
echo "Last Debian-to-worker probe: ${last_host_probe_output}" >&2
|
|
fi
|
|
else
|
|
echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2
|
|
fi
|
|
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
|
|
|
|
return 1
|
|
}
|
|
|
|
pimox_generated_mac() {
|
|
local vmid="$1"
|
|
|
|
printf '02:68:10:%02x:%02x:%02x\n' \
|
|
$(((vmid >> 16) & 255)) \
|
|
$(((vmid >> 8) & 255)) \
|
|
$((vmid & 255))
|
|
}
|
|
|
|
pimox_worker_static_ip() {
|
|
local index="$1"
|
|
local static_ips="$2"
|
|
local current_index=1
|
|
local ip
|
|
|
|
static_ips="${static_ips//,/ }"
|
|
for ip in ${static_ips}; do
|
|
if ((current_index == index)); then
|
|
printf '%s\n' "${ip}"
|
|
return 0
|
|
fi
|
|
current_index=$((current_index + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
pimox_worker_net0_config() {
|
|
local vmid="$1"
|
|
local bridge="$2"
|
|
local mac_mode="${LAB_PIMOX_WORKER_MAC_MODE:-deterministic}"
|
|
local mac
|
|
|
|
case "${mac_mode}" in
|
|
auto)
|
|
printf 'virtio,bridge=%s\n' "${bridge}"
|
|
;;
|
|
deterministic)
|
|
mac="$(pimox_generated_mac "${vmid}")"
|
|
printf 'virtio=%s,bridge=%s\n' "${mac}" "${bridge}"
|
|
;;
|
|
*)
|
|
echo "LAB_PIMOX_WORKER_MAC_MODE must be 'auto' or 'deterministic'." >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
validate_ipv4_cidr_or_host() {
|
|
local value="$1"
|
|
|
|
[[ "${value}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}(/[0-9]{1,2})?$ ]]
|
|
}
|
|
|
|
configure_pimox_worker_guest_network() {
|
|
local pimox_host="$1"
|
|
local pimox_user="$2"
|
|
local pimox_key="$3"
|
|
local vmid="$4"
|
|
local qm_bin="$5"
|
|
local worker_ip="$6"
|
|
local gateway="$7"
|
|
local dns_servers="$8"
|
|
local interface_name="$9"
|
|
local timeout_seconds="${LAB_PIMOX_GUEST_AGENT_CONFIG_TIMEOUT_SECONDS:-300}"
|
|
local deadline
|
|
local script
|
|
local encoded_script
|
|
local network_output
|
|
local network_exitcode
|
|
local worker_address
|
|
local lan_probe_host="${LAB_PIMOX_WORKER_LAN_PROBE_HOST:-${LAB_DEBIAN_LAN_IP:-}}"
|
|
local require_lan_probe="${LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE:-false}"
|
|
local lan_probe_block=""
|
|
|
|
if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
|
|
echo "Invalid LAB_PIMOX_WORKER_STATIC_IPS entry '${worker_ip}' for VM ${vmid}." >&2
|
|
return 1
|
|
fi
|
|
if [[ "${worker_ip}" == */* && "${worker_ip}" != */24 ]]; then
|
|
echo "LAB_PIMOX_WORKER_STATIC_IPS currently supports host IPv4 values or /24 CIDRs, got '${worker_ip}'." >&2
|
|
return 1
|
|
fi
|
|
if [[ "${worker_ip}" != */* ]]; then
|
|
worker_ip="${worker_ip}/24"
|
|
fi
|
|
worker_address="${worker_ip%%/*}"
|
|
if ! [[ "${gateway}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
|
|
echo "Invalid LAB_PIMOX_WORKER_GATEWAY '${gateway}'." >&2
|
|
return 1
|
|
fi
|
|
if ! [[ "${dns_servers}" =~ ^[0-9.[:space:]]+$ ]]; then
|
|
echo "Invalid LAB_PIMOX_WORKER_DNS_SERVERS '${dns_servers}'." >&2
|
|
return 1
|
|
fi
|
|
if ! [[ "${interface_name}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
|
|
echo "Invalid LAB_PIMOX_WORKER_INTERFACE '${interface_name}'." >&2
|
|
return 1
|
|
fi
|
|
if [[ -n "${lan_probe_host}" ]] && ! [[ "${lan_probe_host}" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then
|
|
echo "Invalid LAB_PIMOX_WORKER_LAN_PROBE_HOST '${lan_probe_host}'." >&2
|
|
return 1
|
|
fi
|
|
if ! truthy "${require_lan_probe}" && ! disabled_value "${require_lan_probe}"; then
|
|
echo "LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE must be true or false." >&2
|
|
return 1
|
|
fi
|
|
|
|
if truthy "${require_lan_probe}"; then
|
|
lan_probe_block="$(cat <<EOF
|
|
echo "guest-check: ping gateway ${gateway}"
|
|
ping -c 2 -W 2 '${gateway}'
|
|
EOF
|
|
)"
|
|
if [[ -n "${lan_probe_host}" ]]; then
|
|
lan_probe_block+="
|
|
echo \"guest-check: ping Debian/probe host ${lan_probe_host}\"
|
|
ping -c 2 -W 2 '${lan_probe_host}'"
|
|
fi
|
|
else
|
|
lan_probe_block="echo 'guest-check: LAN probe disabled by LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=false'"
|
|
fi
|
|
|
|
script="$(cat <<EOF
|
|
set -eu
|
|
cat >/tmp/homelab-interfaces <<'INTERFACES'
|
|
auto lo
|
|
iface lo inet loopback
|
|
|
|
auto ${interface_name}
|
|
iface ${interface_name} inet static
|
|
address ${worker_address}
|
|
netmask 255.255.255.0
|
|
gateway ${gateway}
|
|
dns-nameservers ${dns_servers}
|
|
INTERFACES
|
|
sudo cp /tmp/homelab-interfaces /etc/network/interfaces
|
|
sudo mkdir -p /etc/cloud/cloud.cfg.d
|
|
printf '%s\n' 'network: {config: disabled}' | sudo tee /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg >/dev/null
|
|
for unit in systemd-networkd systemd-networkd.socket NetworkManager dhcpcd; do
|
|
sudo systemctl disable --now "\$unit" >/dev/null 2>&1 || true
|
|
done
|
|
sudo pkill -x dhclient >/dev/null 2>&1 || true
|
|
sudo pkill -x dhcpcd >/dev/null 2>&1 || true
|
|
sudo systemctl enable networking >/dev/null 2>&1 || true
|
|
sudo ifdown --force ${interface_name} 2>/dev/null || true
|
|
sudo ip addr flush dev ${interface_name} scope global || true
|
|
sudo ip route flush dev ${interface_name} || true
|
|
sudo ip link set ${interface_name} up
|
|
sudo ip addr add ${worker_ip} dev ${interface_name}
|
|
sudo ip route replace default via ${gateway} dev ${interface_name} src ${worker_address}
|
|
printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null
|
|
sudo ifup --force ${interface_name} 2>/dev/null || true
|
|
sudo ssh-keygen -A
|
|
sudo install -d -m 0755 /run/sshd
|
|
sudo mkdir -p /etc/ssh/sshd_config.d
|
|
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
|
|
Port 22
|
|
ListenAddress 0.0.0.0
|
|
PubkeyAuthentication yes
|
|
PasswordAuthentication no
|
|
KbdInteractiveAuthentication no
|
|
PermitRootLogin no
|
|
UsePAM yes
|
|
SSHD_CONFIG
|
|
sudo sshd -t
|
|
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
|
|
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
|
|
ssh_unit=ssh.service
|
|
if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
|
|
ssh_unit=sshd.service
|
|
fi
|
|
sudo systemctl enable "\$ssh_unit" >/dev/null
|
|
sudo systemctl restart "\$ssh_unit"
|
|
ip -br addr show ${interface_name}
|
|
ip route
|
|
ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
|
|
${lan_probe_block}
|
|
echo 'guest-check: ssh service'
|
|
sudo systemctl is-active "\$ssh_unit"
|
|
if command -v ss >/dev/null 2>&1; then
|
|
ss -ltnp 2>/dev/null || ss -ltn
|
|
ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
|
|
fi
|
|
EOF
|
|
)"
|
|
encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
|
|
|
|
deadline=$((SECONDS + timeout_seconds))
|
|
while ((SECONDS < deadline)); do
|
|
if network_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
|
|
printf '%s\n' "${network_output}" | sed 's/^/ guest-network: /'
|
|
if network_exitcode="$(pimox_guest_exec_exitcode "${network_output}")" && [[ "${network_exitcode}" == "0" ]]; then
|
|
return 0
|
|
fi
|
|
echo "Static guest network validation failed inside Pimox VM ${vmid}." >&2
|
|
if [[ -n "${network_exitcode:-}" ]]; then
|
|
echo "Guest exit code: ${network_exitcode}" >&2
|
|
fi
|
|
return 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
echo "Timed out configuring static guest network for Pimox VM ${vmid} through qemu-guest-agent." >&2
|
|
if [[ -n "${network_output:-}" ]]; then
|
|
printf '%s\n' "${network_output}" | sed 's/^/ /' >&2
|
|
fi
|
|
pimox_worker_guest_agent_recovery_hint "${pimox_host}" "${pimox_user}" "${vmid}" "${qm_bin}"
|
|
return 1
|
|
}
|
|
|
|
configure_pimox_worker_forwarding() {
|
|
local pimox_host="$1"
|
|
local pimox_user="$2"
|
|
local pimox_key="$3"
|
|
local worker_ip="$4"
|
|
local enabled="${LAB_PIMOX_WORKER_CONFIGURE_FORWARDING:-true}"
|
|
local backup_dir="${LAB_PIMOX_WORKER_IPTABLES_BACKUP_DIR:-~/iptables-backups}"
|
|
local worker_address
|
|
|
|
if disabled_value "${enabled}"; then
|
|
return 0
|
|
fi
|
|
if ! truthy "${enabled}"; then
|
|
echo "LAB_PIMOX_WORKER_CONFIGURE_FORWARDING must be true or false." >&2
|
|
return 1
|
|
fi
|
|
if ! validate_ipv4_cidr_or_host "${worker_ip}"; then
|
|
echo "Invalid Pimox worker forwarding IP '${worker_ip}'." >&2
|
|
return 1
|
|
fi
|
|
worker_address="${worker_ip%%/*}"
|
|
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\"
|
|
if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then
|
|
iptables_bin=/usr/sbin/iptables
|
|
fi
|
|
if [ -z \"\$iptables_bin\" ]; then
|
|
if command -v apt-get >/dev/null 2>&1; then
|
|
echo 'iptables not found on Pimox host; installing iptables for worker forwarding rules.'
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends iptables
|
|
fi
|
|
fi
|
|
iptables_bin=\"\$(command -v iptables 2>/dev/null || true)\"
|
|
if [ -z \"\$iptables_bin\" ] && [ -x /usr/sbin/iptables ]; then
|
|
iptables_bin=/usr/sbin/iptables
|
|
fi
|
|
if [ -z \"\$iptables_bin\" ]; then
|
|
echo 'iptables not found on Pimox host and could not be installed; cannot configure worker forwarding rules.' >&2
|
|
exit 1
|
|
fi
|
|
iptables_wait_seconds='${LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS:-60}'
|
|
case \"\$iptables_wait_seconds\" in
|
|
''|*[!0-9]*)
|
|
echo \"LAB_PIMOX_WORKER_IPTABLES_WAIT_SECONDS must be a non-negative integer, got '\$iptables_wait_seconds'.\" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
iptables_wait_args=
|
|
if \"\$iptables_bin\" -h 2>&1 | grep -q -- ' -w'; then
|
|
iptables_wait_args=\"-w \$iptables_wait_seconds\"
|
|
else
|
|
echo 'Pimox host iptables does not support -w; proceeding without xtables lock wait.'
|
|
fi
|
|
iptables_cmd() {
|
|
sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\"
|
|
}
|
|
iptables_save_bin=\"\$(command -v iptables-save 2>/dev/null || true)\"
|
|
if [ -z \"\$iptables_save_bin\" ] && [ -x /usr/sbin/iptables-save ]; then
|
|
iptables_save_bin=/usr/sbin/iptables-save
|
|
fi
|
|
backup_dir='${backup_dir}'
|
|
case \"\$backup_dir\" in
|
|
'~'|'~/') backup_dir=\"\$HOME\" ;;
|
|
'~/'*) backup_dir=\"\$HOME/\${backup_dir#~/}\" ;;
|
|
esac
|
|
mkdir -p \"\$backup_dir\"
|
|
backup_file=\"\$backup_dir/iptables-before-jeannie-workers-\$(date +%Y%m%d-%H%M%S).rules\"
|
|
if [ -n \"\$iptables_save_bin\" ]; then
|
|
sudo \"\$iptables_save_bin\" >\"\$backup_file\"
|
|
echo \"Saved Pimox iptables backup: \$backup_file\"
|
|
fi
|
|
sudo mkdir -p /etc/sysctl.d
|
|
printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-pimox-worker-forwarding.conf >/dev/null
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
chain=FORWARD
|
|
if iptables_cmd -S DOCKER-USER >/dev/null 2>&1; then
|
|
chain=DOCKER-USER
|
|
fi
|
|
if ! iptables_cmd -C \"\$chain\" -s '${worker_address}/32' -j ACCEPT 2>/dev/null; then
|
|
iptables_cmd -I \"\$chain\" 1 -s '${worker_address}/32' -j ACCEPT
|
|
fi
|
|
if ! iptables_cmd -C \"\$chain\" -d '${worker_address}/32' -j ACCEPT 2>/dev/null; then
|
|
iptables_cmd -I \"\$chain\" 1 -d '${worker_address}/32' -j ACCEPT
|
|
fi
|
|
echo \"Pimox worker forwarding rules active in \$chain for ${worker_address}\""
|
|
}
|
|
|
|
iptables_cmd() {
|
|
sudo \"\$iptables_bin\" \$iptables_wait_args \"\$@\"
|
|
}
|
|
|
|
cpuset_cpu_count() {
|
|
local cpuset="$1"
|
|
local count=0
|
|
local part
|
|
local start
|
|
local end
|
|
local -a parts
|
|
|
|
IFS=',' read -r -a parts <<<"${cpuset}"
|
|
for part in "${parts[@]}"; do
|
|
if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
|
|
start="${BASH_REMATCH[1]}"
|
|
end="${BASH_REMATCH[2]}"
|
|
if ((end < start)); then
|
|
return 1
|
|
fi
|
|
count=$((count + end - start + 1))
|
|
elif [[ "${part}" =~ ^[0-9]+$ ]]; then
|
|
count=$((count + 1))
|
|
else
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
printf '%s\n' "${count}"
|
|
}
|
|
|
|
pimox_worker_cpu_affinity() {
|
|
local index="$1"
|
|
local affinities="$2"
|
|
local worker_cores="$3"
|
|
local affinity
|
|
local affinity_index=1
|
|
local cpu_count
|
|
|
|
for affinity in ${affinities}; do
|
|
if ((affinity_index == index)); then
|
|
if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then
|
|
echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2
|
|
exit 1
|
|
fi
|
|
if ((cpu_count != worker_cores)); then
|
|
echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2
|
|
exit 1
|
|
fi
|
|
printf '%s\n' "${affinity}"
|
|
return 0
|
|
fi
|
|
affinity_index=$((affinity_index + 1))
|
|
done
|
|
|
|
echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2
|
|
exit 1
|
|
}
|
|
|
|
pimox_start_vm_with_cpuset() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local qm_bin="$4"
|
|
local vmid="$5"
|
|
local cpuset="$6"
|
|
|
|
if [[ -z "${cpuset}" ]]; then
|
|
pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'"
|
|
return
|
|
fi
|
|
if ! cpuset_cpu_count "${cpuset}" >/dev/null; then
|
|
echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated
|
|
# command under taskset so PSCI brings secondary guest CPUs online reliably.
|
|
pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
|
|
vmid='${vmid}'
|
|
qm_bin='${qm_bin}'
|
|
cpuset='${cpuset}'
|
|
|
|
if ! command -v taskset >/dev/null 2>&1; then
|
|
echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2
|
|
exit 1
|
|
fi
|
|
if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then
|
|
echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2
|
|
exit 1
|
|
fi
|
|
|
|
start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\")
|
|
cleanup() {
|
|
rm -f \"\$start_script\"
|
|
}
|
|
trap cleanup 0
|
|
|
|
sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\"
|
|
if ! awk -v vmid=\"\$vmid\" '
|
|
{
|
|
for (i = 1; i <= NF; i++) {
|
|
if (\$i == "/usr/bin/kvm" || \$i ~ /(^|/)qemu-system-aarch64$/) kvm = 1
|
|
if (\$i == "-id" && (i + 1) <= NF && \$(i + 1) == vmid) id = 1
|
|
}
|
|
}
|
|
END { exit (kvm && id) ? 0 : 1 }
|
|
' \"\$start_script\"; then
|
|
echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2
|
|
sed 's/^/ showcmd: /' \"\$start_script\" >&2
|
|
exit 1
|
|
fi
|
|
smp_count=\$(awk '{ for (i = 1; i <= NF; i++) if (\$i == "-smp") count++ } END { print count + 0 }' \"\$start_script\")
|
|
if [ \"\$smp_count\" -ne 1 ]; then
|
|
echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo taskset -c \"\$cpuset\" bash \"\$start_script\""
|
|
}
|
|
|
|
pimox_shutdown_vm_gracefully() {
|
|
local host="$1"
|
|
local user="$2"
|
|
local key_path="$3"
|
|
local qm_bin="$4"
|
|
local vmid="$5"
|
|
|
|
pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
|
|
sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
|
|
elapsed=0
|
|
while [ \"\$elapsed\" -lt 300 ]; do
|
|
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
|
|
exit 0
|
|
fi
|
|
sleep 5
|
|
elapsed=\$((elapsed + 5))
|
|
done
|
|
echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2
|
|
exit 1"
|
|
}
|
|
|
|
pimox_guest_cpu_count() {
|
|
local guest_ip="$1"
|
|
local guest_user="$2"
|
|
local guest_key_path="$3"
|
|
local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
|
|
|
|
ssh -i "${guest_key_path}" \
|
|
-o BatchMode=yes \
|
|
-o ConnectTimeout=10 \
|
|
-o IdentitiesOnly=yes \
|
|
-o StrictHostKeyChecking=accept-new \
|
|
-o UserKnownHostsFile="${known_hosts_file}" \
|
|
"${guest_user}@${guest_ip}" \
|
|
'nproc --all'
|
|
}
|
|
|
|
ensure_pimox_worker_node() {
|
|
local index="$1"
|
|
local spec_file="$2"
|
|
local pimox_host="$3"
|
|
local pimox_user="$4"
|
|
local pimox_key="$5"
|
|
local template_vmid="$6"
|
|
local bridge="$7"
|
|
local worker_base_vmid="$8"
|
|
local worker_name_prefix="$9"
|
|
local worker_node_prefix="${10}"
|
|
local worker_key_prefix="${11}"
|
|
local worker_cores="${12}"
|
|
local worker_memory="${13}"
|
|
local worker_user="${14}"
|
|
local worker_key_path="${15}"
|
|
local ip_prefix="${16}"
|
|
local timeout_seconds="${17}"
|
|
local qm_bin="${18}"
|
|
local worker_storage="${19}"
|
|
local worker_replace_existing="${20}"
|
|
local worker_cpu_affinity="${21}"
|
|
local worker_kvm_cpuset="${22}"
|
|
local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
|
|
local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
|
|
local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}"
|
|
local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}"
|
|
local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}"
|
|
local padded
|
|
local vmid
|
|
local worker_key
|
|
local worker_name
|
|
local node_name
|
|
local net0_config
|
|
local guest_ip
|
|
local guest_cpu_count
|
|
local static_ip
|
|
|
|
printf -v padded '%02d' "${index}"
|
|
vmid=$((worker_base_vmid + index - 1))
|
|
worker_key="${worker_key_prefix}${padded}"
|
|
worker_name="${worker_name_prefix}-${padded}"
|
|
node_name="${worker_node_prefix}-${padded}"
|
|
if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
|
|
exit 1
|
|
fi
|
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
|
|
echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2
|
|
exit 1
|
|
fi
|
|
if ! truthy "${worker_replace_existing}" &&
|
|
! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then
|
|
echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}."
|
|
worker_replace_existing=true
|
|
fi
|
|
if truthy "${worker_replace_existing}"; then
|
|
echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..."
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true
|
|
elapsed=0
|
|
while [ \"\$elapsed\" -lt 300 ]; do
|
|
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
elapsed=\$((elapsed + 5))
|
|
done
|
|
sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
|
|
else
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
was_running=false
|
|
had_legacy_args=false
|
|
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then
|
|
was_running=true
|
|
fi
|
|
if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
|
|
had_legacy_args=true
|
|
fi
|
|
sudo '${qm_bin}' set '${vmid}' \
|
|
--agent enabled=1 \
|
|
--bios ovmf \
|
|
--boot 'order=scsi0;net0' \
|
|
--cores '${worker_cores}' \
|
|
--memory '${worker_memory}' \
|
|
--net0 '${net0_config}' \
|
|
--numa 0 \
|
|
--ostype l26 \
|
|
--scsihw virtio-scsi-pci \
|
|
--sockets 1 \
|
|
--vga virtio
|
|
if [ \"\$had_legacy_args\" = true ]; then
|
|
sudo '${qm_bin}' set '${vmid}' --delete args
|
|
fi
|
|
if [ -n '${worker_cpu_affinity}' ]; then
|
|
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
|
|
case \"\$affinity_output\" in
|
|
*'Unknown option: affinity'*)
|
|
echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
|
|
;;
|
|
*)
|
|
printf '%s\n' \"\$affinity_output\" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
fi
|
|
if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then
|
|
echo 'Restarting VM ${vmid} to apply removal of legacy qm args.'
|
|
sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
|
|
elapsed=0
|
|
while [ \"\$elapsed\" -lt 300 ]; do
|
|
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
elapsed=\$((elapsed + 5))
|
|
done
|
|
if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
|
|
echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2
|
|
exit 1
|
|
fi
|
|
fi"
|
|
fi
|
|
fi
|
|
|
|
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
if ! ip link show '${bridge}' >/dev/null 2>&1; then
|
|
echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2
|
|
exit 1
|
|
fi
|
|
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
|
|
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
|
|
pvesm_cmd=/usr/sbin/pvesm
|
|
fi
|
|
if [ -z \"\$pvesm_cmd\" ]; then
|
|
echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2
|
|
exit 1
|
|
fi
|
|
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2
|
|
exit 1
|
|
fi
|
|
sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}'
|
|
sudo '${qm_bin}' set '${vmid}' \
|
|
--agent enabled=1 \
|
|
--bios ovmf \
|
|
--boot 'order=scsi0;net0' \
|
|
--cores '${worker_cores}' \
|
|
--memory '${worker_memory}' \
|
|
--net0 '${net0_config}' \
|
|
--numa 0 \
|
|
--ostype l26 \
|
|
--scsihw virtio-scsi-pci \
|
|
--sockets 1 \
|
|
--vga virtio
|
|
if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
|
|
sudo '${qm_bin}' set '${vmid}' --delete args
|
|
fi
|
|
if [ -n '${worker_cpu_affinity}' ]; then
|
|
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
|
|
case \"\$affinity_output\" in
|
|
*'Unknown option: affinity'*)
|
|
echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
|
|
;;
|
|
*)
|
|
printf '%s\n' \"\$affinity_output\" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
fi
|
|
sudo '${qm_bin}' set '${vmid}' --onboot 1"
|
|
fi
|
|
|
|
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then
|
|
pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
|
|
elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then
|
|
echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2
|
|
exit 1
|
|
fi
|
|
|
|
case "${worker_network_mode}" in
|
|
static)
|
|
if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then
|
|
echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2
|
|
exit 1
|
|
fi
|
|
echo "Configuring Pimox worker VM ${vmid} static network ${static_ip} via qemu-guest-agent..."
|
|
configure_pimox_worker_forwarding \
|
|
"${pimox_host}" \
|
|
"${pimox_user}" \
|
|
"${pimox_key}" \
|
|
"${static_ip}"
|
|
if ! configure_pimox_worker_guest_network \
|
|
"${pimox_host}" \
|
|
"${pimox_user}" \
|
|
"${pimox_key}" \
|
|
"${vmid}" \
|
|
"${qm_bin}" \
|
|
"${static_ip}" \
|
|
"${worker_gateway}" \
|
|
"${worker_dns_servers}" \
|
|
"${worker_interface}"; then
|
|
pimox_worker_vm_debug "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${qm_bin}"
|
|
exit 1
|
|
fi
|
|
;;
|
|
dhcp)
|
|
;;
|
|
*)
|
|
echo "LAB_PIMOX_WORKER_NETWORK_MODE must be 'static' or 'dhcp'." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
|
|
echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2
|
|
exit 1
|
|
fi
|
|
|
|
guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
|
|
if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
|
|
echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2
|
|
pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}"
|
|
pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
|
|
if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
|
|
echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2
|
|
exit 1
|
|
fi
|
|
guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
|
|
fi
|
|
if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
|
|
echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2
|
|
echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
|
|
}
|
|
|
|
write_cluster_worker_var_file() {
|
|
local var_file="$1"
|
|
local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}'
|
|
local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}"
|
|
local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}"
|
|
shift
|
|
|
|
LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \
|
|
LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \
|
|
LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \
|
|
LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \
|
|
LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \
|
|
LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \
|
|
LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \
|
|
LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \
|
|
python3 - "${var_file}" "$@" <<'PY'
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
var_file = sys.argv[1]
|
|
spec_files = sys.argv[2:]
|
|
nodes = {}
|
|
node_labels = {}
|
|
|
|
try:
|
|
raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"])
|
|
manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"])
|
|
pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"])
|
|
except json.JSONDecodeError as exc:
|
|
raise SystemExit(f"Invalid node label JSON: {exc}") from exc
|
|
|
|
if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}:
|
|
nodes["raspberrypi"] = {
|
|
"host": os.environ["LAB_RASPBERRY_HOST"],
|
|
"user": os.environ["LAB_RASPBERRY_USER"],
|
|
"node_name": os.environ["LAB_RASPBERRY_NODE_NAME"],
|
|
"ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"],
|
|
}
|
|
node_labels["raspberrypi"] = raspberry_labels
|
|
|
|
for spec_file in spec_files:
|
|
path = Path(spec_file)
|
|
if not path.exists() or path.stat().st_size == 0:
|
|
continue
|
|
labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels
|
|
with path.open(encoding="utf-8") as handle:
|
|
for line_number, line in enumerate(handle, start=1):
|
|
line = line.rstrip("\n")
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
try:
|
|
key, host, user, node_name, ssh_key_path = line.split("\t")
|
|
except ValueError as exc:
|
|
raise SystemExit(
|
|
f"{spec_file}:{line_number}: expected tab-separated "
|
|
"key, host, user, node_name, ssh_key_path"
|
|
) from exc
|
|
nodes[key] = {
|
|
"host": host,
|
|
"user": user,
|
|
"node_name": node_name,
|
|
"ssh_key_path": ssh_key_path,
|
|
}
|
|
node_labels[key] = labels
|
|
|
|
with open(var_file, "w", encoding="utf-8") as handle:
|
|
json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2)
|
|
handle.write("\n")
|
|
PY
|
|
}
|
|
|
|
prepare_cluster_worker_var_file() {
|
|
local include_raspberry_default="$1"
|
|
local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv"
|
|
local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
|
|
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
|
|
local include_manual="${LAB_INCLUDE_MANUAL_WORKERS:-false}"
|
|
local -a spec_files=()
|
|
|
|
export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}"
|
|
|
|
mkdir -p "${REPO_ROOT}/.lab"
|
|
ensure_static_pimox_worker_spec_file "${pimox_spec_file}"
|
|
if truthy "${include_manual}"; then
|
|
spec_files+=("${manual_spec_file}")
|
|
elif ! disabled_value "${include_manual}"; then
|
|
echo "LAB_INCLUDE_MANUAL_WORKERS must be true or false." >&2
|
|
return 1
|
|
fi
|
|
spec_files+=("${pimox_spec_file}")
|
|
write_cluster_worker_var_file "${var_file}" "${spec_files[@]}"
|
|
export LAB_CLUSTER_VAR_FILE="${var_file}"
|
|
}
|
|
|
|
ensure_static_pimox_worker_spec_file() {
|
|
local spec_file="$1"
|
|
local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
|
|
local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
|
|
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
|
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
|
|
local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
|
|
local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
|
|
local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
|
|
local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
|
|
local index
|
|
local padded
|
|
local worker_key
|
|
local node_name
|
|
local static_ip
|
|
|
|
if [[ -s "${spec_file}" || "${worker_network_mode}" != "static" ]]; then
|
|
return 0
|
|
fi
|
|
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2
|
|
return 1
|
|
fi
|
|
if ((worker_count == 0)); then
|
|
return 0
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${spec_file}")"
|
|
: >"${spec_file}"
|
|
for ((index = 1; index <= worker_count; index++)); do
|
|
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
|
continue
|
|
fi
|
|
if ! static_ip="$(pimox_worker_static_ip "${index}" "${worker_static_ips}")"; then
|
|
echo "LAB_PIMOX_WORKER_NETWORK_MODE=static requires LAB_PIMOX_WORKER_STATIC_IPS to include an IP for worker index ${index}." >&2
|
|
return 1
|
|
fi
|
|
printf -v padded '%02d' "${index}"
|
|
worker_key="${worker_key_prefix}${padded}"
|
|
node_name="${worker_node_prefix}-${padded}"
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${static_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
|
|
done
|
|
}
|
|
|
|
cluster_worker_var_file_has_workers() {
|
|
local var_file="$1"
|
|
|
|
[[ -s "${var_file}" ]] || return 1
|
|
python3 - "${var_file}" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
sys.exit(0 if document.get("worker_nodes") else 1)
|
|
PY
|
|
}
|
|
|
|
preflight_check() {
|
|
local description="$1"
|
|
local output
|
|
shift
|
|
|
|
printf '%s %s... ' '-->' "${description}"
|
|
if output="$("$@" 2>&1)"; then
|
|
printf 'ok\n'
|
|
return 0
|
|
fi
|
|
|
|
printf 'failed\n'
|
|
if [[ -n "${output}" ]]; then
|
|
printf '%s\n' "${output}" | sed 's/^/ /' >&2
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
preflight_warn() {
|
|
local description="$1"
|
|
local output
|
|
shift
|
|
|
|
printf '%s %s... ' '-->' "${description}"
|
|
if output="$("$@" 2>&1)"; then
|
|
printf 'ok\n'
|
|
return 0
|
|
fi
|
|
|
|
printf 'warning\n'
|
|
if [[ -n "${output}" ]]; then
|
|
printf '%s\n' "${output}" | sed 's/^/ /' >&2
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
check_gitea_reachable() {
|
|
local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}"
|
|
local output
|
|
|
|
if output="$(curl -fsS --max-time 10 "${gitea_url}" 2>&1 >/dev/null)"; then
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "${output}" >&2
|
|
cat >&2 <<'EOF'
|
|
Gitea local HTTP failed. Run ./jeannie doctor-gitea --verbose.
|
|
If the local Gitea container is down or unhealthy, run ./jeannie deploy-gitea.
|
|
EOF
|
|
return 1
|
|
}
|
|
|
|
check_debian_docker_root() {
|
|
local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
|
|
local actual_root
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "docker command was not found on Debian host" >&2
|
|
return 1
|
|
fi
|
|
actual_root="$(sudo docker info --format '{{.DockerRootDir}}')"
|
|
if [[ "${actual_root}" != "${expected_root}" ]]; then
|
|
echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
fix_debian_docker_root() {
|
|
local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
|
|
local daemon_file="/etc/docker/daemon.json"
|
|
local current_root=""
|
|
|
|
require_debian_server "fix-debian-docker-root"
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
echo "docker command was not found on Debian host" >&2
|
|
exit 1
|
|
fi
|
|
|
|
current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
|
|
if [[ "${current_root}" == "${expected_root}" ]]; then
|
|
echo "Debian Docker root is already ${expected_root}."
|
|
return 0
|
|
fi
|
|
|
|
echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..."
|
|
sudo mkdir -p "${expected_root}" /etc/docker
|
|
|
|
echo "Stopping Docker services..."
|
|
sudo systemctl stop docker 2>/dev/null || true
|
|
sudo systemctl stop containerd 2>/dev/null || true
|
|
|
|
if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then
|
|
echo "Copying existing Docker data to ${expected_root}..."
|
|
sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/"
|
|
fi
|
|
|
|
sudo python3 - "${daemon_file}" "${expected_root}" <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
path, expected_root = sys.argv[1:3]
|
|
document = {}
|
|
if os.path.exists(path) and os.path.getsize(path) > 0:
|
|
with open(path, encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
document["data-root"] = expected_root
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
json.dump(document, handle, indent=2, sort_keys=True)
|
|
handle.write("\n")
|
|
PY
|
|
|
|
echo "Starting Docker services..."
|
|
sudo systemctl start containerd
|
|
sudo systemctl start docker
|
|
|
|
check_debian_docker_root
|
|
echo "Debian Docker root is now ${expected_root}."
|
|
}
|
|
|
|
check_debian_tailscale_ip() {
|
|
local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}"
|
|
|
|
if [[ -z "${expected_ip}" ]]; then
|
|
return 0
|
|
fi
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo "tailscale command was not found on Debian host" >&2
|
|
return 1
|
|
fi
|
|
if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then
|
|
echo "Debian Tailscale IP does not include ${expected_ip}" >&2
|
|
tailscale ip -4 >&2 || true
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_edge_ssh() {
|
|
local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
|
|
local edge_user="${LAB_EDGE_USER:-ubuntu}"
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true"
|
|
}
|
|
|
|
check_rpi_ssh() {
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true"
|
|
}
|
|
|
|
check_rpi_docker_root_state() {
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
|
|
local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
|
|
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
|
|
actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
|
|
if [ \"\$actual_root\" = '${nvme_root}' ]; then
|
|
mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; }
|
|
exit 0
|
|
fi
|
|
if [ \"\$actual_root\" = '${fallback_root}' ]; then
|
|
echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2
|
|
exit 0
|
|
fi
|
|
echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2
|
|
exit 1"
|
|
}
|
|
|
|
check_rpi_tailscale_ip() {
|
|
local expected_ip="${LAB_RPI_TAILSCALE_IP:-}"
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
|
|
if [[ -z "${expected_ip}" ]]; then
|
|
return 0
|
|
fi
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'"
|
|
}
|
|
|
|
check_pimox_storage() {
|
|
local mode="${LAB_PIMOX_PIPELINE:-true}"
|
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
|
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
|
|
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
|
|
pvesm_cmd=/usr/sbin/pvesm
|
|
fi
|
|
if [ -z \"\$pvesm_cmd\" ]; then
|
|
echo 'pvesm was not found' >&2
|
|
exit 1
|
|
fi
|
|
sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'"
|
|
}
|
|
|
|
check_local_disk_free_gib() {
|
|
local path="$1"
|
|
local min_gib="$2"
|
|
local available_kib
|
|
local min_kib
|
|
|
|
available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')"
|
|
min_kib=$((min_gib * 1024 * 1024))
|
|
if ((available_kib < min_kib)); then
|
|
echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_systemd_active() {
|
|
local unit="$1"
|
|
|
|
systemctl is-active --quiet "${unit}"
|
|
}
|
|
|
|
check_rpi_docker_writable() {
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
|
|
root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
|
|
sudo mkdir -p \"\$root/.homelab-check\"
|
|
sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\"
|
|
sudo rm -f \"\$root/.homelab-check/write-test\""
|
|
}
|
|
|
|
check_edge_disk_free() {
|
|
local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}"
|
|
local edge_user="${LAB_EDGE_USER:-ubuntu}"
|
|
local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}"
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu
|
|
available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\"
|
|
min_kib=$((min_gib * 1024 * 1024))
|
|
if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then
|
|
echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2
|
|
exit 1
|
|
fi"
|
|
}
|
|
|
|
check_pihole_dns_query() {
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}"
|
|
local output
|
|
|
|
if command -v dig >/dev/null 2>&1; then
|
|
output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || {
|
|
printf '%s\n' "${output}" >&2
|
|
echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2
|
|
return 1
|
|
}
|
|
if [[ -z "${output}" ]]; then
|
|
echo "Pi-hole returned no A records for ${query_name}" >&2
|
|
return 1
|
|
fi
|
|
return 0
|
|
fi
|
|
if command -v nslookup >/dev/null 2>&1; then
|
|
nslookup "${query_name}" "${rpi_host}" >/dev/null
|
|
return $?
|
|
fi
|
|
|
|
echo "dig or nslookup is required to test Pi-hole DNS directly." >&2
|
|
return 1
|
|
}
|
|
|
|
homelab_preflight() {
|
|
local phase="${1:-full}"
|
|
local failures=0
|
|
|
|
require_debian_server "preflight"
|
|
|
|
if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then
|
|
echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}."
|
|
return 0
|
|
fi
|
|
|
|
case "${phase}" in
|
|
early | full)
|
|
;;
|
|
*)
|
|
echo "Unsupported preflight phase '${phase}'." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..."
|
|
|
|
preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1))
|
|
preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1))
|
|
preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1))
|
|
if [[ "${phase}" == "full" ]]; then
|
|
preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1))
|
|
preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1))
|
|
fi
|
|
preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip
|
|
preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
|
|
preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" check_edge_ssh || failures=$((failures + 1))
|
|
|
|
if ((failures > 0)); then
|
|
echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "${phase^} preflight checks passed."
|
|
}
|
|
|
|
doctor_preapply() {
|
|
local failures=0
|
|
|
|
require_debian_server "doctor-preapply"
|
|
|
|
echo "Running pre-apply readiness checks..."
|
|
preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1))
|
|
preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1))
|
|
preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1))
|
|
preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1))
|
|
preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1))
|
|
preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
|
|
preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1))
|
|
preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1))
|
|
|
|
if ((failures > 0)); then
|
|
echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Pre-apply doctor checks passed."
|
|
}
|
|
|
|
inventory_check() {
|
|
"${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
|
|
}
|
|
|
|
run_pimox_pipeline() {
|
|
local mode="${LAB_PIMOX_PIPELINE:-true}"
|
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
|
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
|
local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}"
|
|
local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}"
|
|
local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}"
|
|
local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}"
|
|
local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}"
|
|
local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}"
|
|
local provisioning_interface
|
|
local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
|
|
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
|
|
local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}"
|
|
local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
|
|
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
|
|
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
|
|
local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}"
|
|
local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}"
|
|
local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}"
|
|
local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}"
|
|
local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}"
|
|
local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}"
|
|
local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}"
|
|
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
|
|
local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
|
|
local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
|
|
local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}"
|
|
local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}"
|
|
local worker_wait_timeout_seconds="${LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS:-600}"
|
|
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
|
|
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
|
|
local index
|
|
local readiness_output
|
|
local readiness_status
|
|
local worker_cpu_affinity
|
|
local cpuset_cpu_total
|
|
local cpuset_name
|
|
local cpuset_value
|
|
local required_cores
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then
|
|
mode="true"
|
|
fi
|
|
|
|
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2
|
|
exit 1
|
|
fi
|
|
if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then
|
|
echo "Pimox template and worker CPU and memory values must be greater than zero." >&2
|
|
exit 1
|
|
fi
|
|
for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do
|
|
cpuset_value="${!cpuset_name}"
|
|
if [[ -z "${cpuset_value}" ]]; then
|
|
continue
|
|
fi
|
|
if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then
|
|
echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2
|
|
exit 1
|
|
fi
|
|
required_cores="${template_cores}"
|
|
if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then
|
|
required_cores="${worker_cores}"
|
|
fi
|
|
if ((cpuset_cpu_total < required_cores)); then
|
|
echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then
|
|
echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${worker_wait_timeout_seconds}" =~ ^[0-9]+$ && "${worker_wait_timeout_seconds}" -gt 0 ]]; then
|
|
echo "LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then
|
|
echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
|
|
echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2
|
|
exit 1
|
|
fi
|
|
if [[ "${worker_storage}" == "local" ]]; then
|
|
echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2
|
|
exit 1
|
|
fi
|
|
|
|
set +e
|
|
readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
|
|
if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then
|
|
echo 'qm was not found in PATH and ${qm_bin} is not executable'
|
|
exit 1
|
|
fi
|
|
if ! ip link show '${bridge}' >/dev/null 2>&1; then
|
|
echo 'bridge ${bridge} was not found'
|
|
exit 1
|
|
fi
|
|
if ! sudo -n true >/dev/null 2>&1; then
|
|
echo 'passwordless sudo is not available for ${pimox_user}'
|
|
exit 1
|
|
fi
|
|
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
|
|
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
|
|
pvesm_cmd=/usr/sbin/pvesm
|
|
fi
|
|
if [ -z \"\$pvesm_cmd\" ]; then
|
|
echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}'
|
|
exit 1
|
|
fi
|
|
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo 'Pimox worker storage ${worker_storage} was not found or is not active'
|
|
exit 1
|
|
fi" 2>&1)"
|
|
readiness_status=$?
|
|
set -e
|
|
if ((readiness_status != 0)); then
|
|
if [[ "${mode}" == "auto" ]]; then
|
|
echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready."
|
|
return 0
|
|
fi
|
|
echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
ensure_python3
|
|
provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}"
|
|
if [[ -z "${provisioning_interface}" ]]; then
|
|
echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2
|
|
exit 1
|
|
fi
|
|
|
|
export TF_VAR_provisioning_interface="${provisioning_interface}"
|
|
export TF_VAR_pimox_host="${pimox_host}"
|
|
export TF_VAR_pimox_user="${pimox_user}"
|
|
export TF_VAR_pimox_ssh_key_path="${pimox_key}"
|
|
export TF_VAR_pimox_qm_bin="${qm_bin}"
|
|
export TF_VAR_pimox_template_bridge="${bridge}"
|
|
export TF_VAR_pimox_template_vmid="${template_vmid}"
|
|
export TF_VAR_pimox_template_name="${template_name}"
|
|
export TF_VAR_pimox_template_cores="${template_cores}"
|
|
export TF_VAR_pimox_template_memory="${template_memory}"
|
|
export TF_VAR_pimox_template_replace_existing="${template_replace_existing}"
|
|
export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}"
|
|
export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}"
|
|
export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}"
|
|
export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}"
|
|
export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}"
|
|
export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}"
|
|
|
|
echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..."
|
|
run_tofu_stack "bootstrap/provisioning"
|
|
|
|
if ((worker_count == 0)); then
|
|
return 0
|
|
fi
|
|
|
|
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then
|
|
echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2
|
|
exit 1
|
|
fi
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1"
|
|
echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage."
|
|
|
|
mkdir -p "${REPO_ROOT}/.lab"
|
|
: >"${spec_file}"
|
|
for ((index = 1; index <= worker_count; index++)); do
|
|
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
|
|
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
|
|
continue
|
|
fi
|
|
|
|
worker_cpu_affinity=""
|
|
if [[ -n "${worker_cpu_affinities}" ]]; then
|
|
worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")"
|
|
fi
|
|
ensure_pimox_worker_node \
|
|
"${index}" \
|
|
"${spec_file}" \
|
|
"${pimox_host}" \
|
|
"${pimox_user}" \
|
|
"${pimox_key}" \
|
|
"${template_vmid}" \
|
|
"${bridge}" \
|
|
"${worker_base_vmid}" \
|
|
"${worker_name_prefix}" \
|
|
"${worker_node_prefix}" \
|
|
"${worker_key_prefix}" \
|
|
"${worker_cores}" \
|
|
"${worker_memory}" \
|
|
"${worker_user}" \
|
|
"${worker_key_path}" \
|
|
"${ip_prefix}" \
|
|
"${worker_wait_timeout_seconds}" \
|
|
"${qm_bin}" \
|
|
"${worker_storage}" \
|
|
"${worker_replace_existing}" \
|
|
"${worker_cpu_affinity}" \
|
|
"${worker_kvm_cpuset}"
|
|
done
|
|
|
|
write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}"
|
|
export LAB_CLUSTER_VAR_FILE="${var_file}"
|
|
}
|
|
|
|
run_openwrt_pipeline() {
|
|
local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}"
|
|
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
|
|
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
|
|
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
|
|
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
|
|
local vmid="${LAB_OPENWRT_VMID:-9100}"
|
|
local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}"
|
|
local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}"
|
|
local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}"
|
|
local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}"
|
|
local cores="${LAB_OPENWRT_CORES:-2}"
|
|
local memory="${LAB_OPENWRT_MEMORY:-512}"
|
|
local version="${LAB_OPENWRT_VERSION:-24.10.6}"
|
|
local image_url="${LAB_OPENWRT_IMAGE_URL:-}"
|
|
local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}"
|
|
local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}"
|
|
local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}"
|
|
local start_vm="${LAB_OPENWRT_START:-false}"
|
|
local onboot="${LAB_OPENWRT_ONBOOT:-false}"
|
|
local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}"
|
|
local root_key_b64=""
|
|
local lan_dhcp_ignore="1"
|
|
local start_vm_flag="false"
|
|
local onboot_flag="0"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
if ! truthy "${mode}"; then
|
|
echo "LAB_OPENWRT_VM must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${image_url}" ]]; then
|
|
image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz"
|
|
fi
|
|
|
|
if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2
|
|
exit 1
|
|
fi
|
|
for value_name in storage wan_bridge lan_bridge vm_name; do
|
|
local value="${!value_name}"
|
|
if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
|
|
echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ "${storage}" == "local" ]]; then
|
|
echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then
|
|
echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2
|
|
exit 1
|
|
fi
|
|
if truthy "${lan_dhcp_enabled}"; then
|
|
lan_dhcp_ignore="0"
|
|
fi
|
|
if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then
|
|
echo "LAB_OPENWRT_START must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
if truthy "${start_vm}"; then
|
|
start_vm_flag="true"
|
|
fi
|
|
if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then
|
|
echo "LAB_OPENWRT_ONBOOT must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
if truthy "${onboot}"; then
|
|
onboot_flag="1"
|
|
fi
|
|
if [[ -r "${root_key_path}" ]]; then
|
|
root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')"
|
|
fi
|
|
|
|
echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..."
|
|
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <<EOF
|
|
set -euo pipefail
|
|
|
|
vmid="${vmid}"
|
|
vm_name="${vm_name}"
|
|
storage="${storage}"
|
|
wan_bridge="${wan_bridge}"
|
|
lan_bridge="${lan_bridge}"
|
|
cores="${cores}"
|
|
memory="${memory}"
|
|
image_url="${image_url}"
|
|
lan_ip="${lan_ip}"
|
|
lan_netmask="${lan_netmask}"
|
|
lan_dhcp_ignore="${lan_dhcp_ignore}"
|
|
start_vm="${start_vm_flag}"
|
|
onboot="${onboot_flag}"
|
|
root_key_b64="${root_key_b64}"
|
|
qm_cmd="${qm_bin}"
|
|
|
|
if [ ! -x "\$qm_cmd" ]; then
|
|
qm_cmd="\$(command -v qm 2>/dev/null || true)"
|
|
fi
|
|
if [ -z "\$qm_cmd" ]; then
|
|
echo "qm is not installed on this Pimox host" >&2
|
|
exit 1
|
|
fi
|
|
|
|
pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)"
|
|
if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then
|
|
pvesm_cmd=/usr/sbin/pvesm
|
|
fi
|
|
if [ -z "\$pvesm_cmd" ]; then
|
|
echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if ! sudo -n true >/dev/null 2>&1; then
|
|
echo "passwordless sudo is required for OpenWrt VM automation" >&2
|
|
exit 1
|
|
fi
|
|
if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then
|
|
echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2
|
|
exit 1
|
|
fi
|
|
if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then
|
|
echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2
|
|
exit 1
|
|
fi
|
|
if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo "Pimox storage \$storage was not found." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then
|
|
if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then
|
|
echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2
|
|
exit 1
|
|
fi
|
|
sudo "\$qm_cmd" set "\$vmid" \\
|
|
--net0 "virtio,bridge=\$wan_bridge" \\
|
|
--net1 "virtio,bridge=\$lan_bridge" \\
|
|
--cores "\$cores" \\
|
|
--memory "\$memory" \\
|
|
--onboot "\$onboot"
|
|
if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then
|
|
sudo "\$qm_cmd" start "\$vmid"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
for required_cmd in curl gzip losetup mount umount awk sed; do
|
|
if ! command -v "\$required_cmd" >/dev/null 2>&1; then
|
|
echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)"
|
|
mnt_dir="\$tmp_dir/root"
|
|
loopdev=""
|
|
cleanup() {
|
|
if mountpoint -q "\$mnt_dir" 2>/dev/null; then
|
|
sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true
|
|
fi
|
|
if [ -n "\$loopdev" ]; then
|
|
sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true
|
|
fi
|
|
rm -rf "\$tmp_dir"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
mkdir -p "\$mnt_dir"
|
|
curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz"
|
|
gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img"
|
|
|
|
loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")"
|
|
root_part="\${loopdev}p2"
|
|
if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then
|
|
root_part="\${loopdev}p2"
|
|
fi
|
|
if [ ! -b "\$root_part" ]; then
|
|
echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo mount "\$root_part" "\$mnt_dir"
|
|
sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh"
|
|
|
|
cat >"\$tmp_dir/network" <<NETWORK
|
|
config interface 'loopback'
|
|
option device 'lo'
|
|
option proto 'static'
|
|
option ipaddr '127.0.0.1'
|
|
option netmask '255.0.0.0'
|
|
|
|
config globals 'globals'
|
|
option ula_prefix 'fd00:68:50::/48'
|
|
|
|
config interface 'wan'
|
|
option device 'eth0'
|
|
option proto 'dhcp'
|
|
|
|
config interface 'lan'
|
|
option device 'eth1'
|
|
option proto 'static'
|
|
option ipaddr '\$lan_ip'
|
|
option netmask '\$lan_netmask'
|
|
option ip6assign '60'
|
|
NETWORK
|
|
|
|
cat >"\$tmp_dir/dhcp" <<DHCP
|
|
config dnsmasq
|
|
option domainneeded '1'
|
|
option boguspriv '1'
|
|
option filterwin2k '0'
|
|
option localise_queries '1'
|
|
option rebind_protection '1'
|
|
option rebind_localhost '1'
|
|
option local '/lan/'
|
|
option domain 'lan'
|
|
option expandhosts '1'
|
|
option cachesize '1000'
|
|
option authoritative '1'
|
|
option readethers '1'
|
|
option leasefile '/tmp/dhcp.leases'
|
|
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
|
|
|
|
config dhcp 'lan'
|
|
option interface 'lan'
|
|
option start '100'
|
|
option limit '150'
|
|
option leasetime '12h'
|
|
option ignore '\$lan_dhcp_ignore'
|
|
|
|
config dhcp 'wan'
|
|
option interface 'wan'
|
|
option ignore '1'
|
|
DHCP
|
|
|
|
cat >"\$tmp_dir/firewall" <<'FIREWALL'
|
|
config defaults
|
|
option input 'REJECT'
|
|
option output 'ACCEPT'
|
|
option forward 'REJECT'
|
|
option synflood_protect '1'
|
|
|
|
config zone
|
|
option name 'lan'
|
|
list network 'lan'
|
|
option input 'ACCEPT'
|
|
option output 'ACCEPT'
|
|
option forward 'ACCEPT'
|
|
|
|
config zone
|
|
option name 'wan'
|
|
list network 'wan'
|
|
option input 'REJECT'
|
|
option output 'ACCEPT'
|
|
option forward 'REJECT'
|
|
option masq '1'
|
|
option mtu_fix '1'
|
|
|
|
config forwarding
|
|
option src 'lan'
|
|
option dest 'wan'
|
|
|
|
config rule
|
|
option name 'Allow-DHCP-Renew'
|
|
option src 'wan'
|
|
option proto 'udp'
|
|
option dest_port '68'
|
|
option target 'ACCEPT'
|
|
option family 'ipv4'
|
|
|
|
config rule
|
|
option name 'Allow-Ping'
|
|
option src 'wan'
|
|
option proto 'icmp'
|
|
option icmp_type 'echo-request'
|
|
option family 'ipv4'
|
|
option target 'ACCEPT'
|
|
FIREWALL
|
|
|
|
cat >"\$tmp_dir/system" <<SYSTEM
|
|
config system
|
|
option hostname '\$vm_name'
|
|
option timezone 'UTC'
|
|
option ttylogin '0'
|
|
option log_size '64'
|
|
option urandom_seed '0'
|
|
SYSTEM
|
|
|
|
sudo cp "\$tmp_dir/network" "\$mnt_dir/etc/config/network"
|
|
sudo cp "\$tmp_dir/dhcp" "\$mnt_dir/etc/config/dhcp"
|
|
sudo cp "\$tmp_dir/firewall" "\$mnt_dir/etc/config/firewall"
|
|
sudo cp "\$tmp_dir/system" "\$mnt_dir/etc/config/system"
|
|
if [ -n "\$root_key_b64" ]; then
|
|
printf '%s' "\$root_key_b64" | base64 -d >"\$tmp_dir/authorized_keys"
|
|
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys"
|
|
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
|
|
sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
|
|
fi
|
|
sync
|
|
sudo umount "\$mnt_dir"
|
|
sudo losetup -d "\$loopdev"
|
|
loopdev=""
|
|
|
|
sudo "\$qm_cmd" create "\$vmid" \\
|
|
--name "\$vm_name" \\
|
|
--bios ovmf \\
|
|
--cores "\$cores" \\
|
|
--memory "\$memory" \\
|
|
--net0 "virtio,bridge=\$wan_bridge" \\
|
|
--net1 "virtio,bridge=\$lan_bridge" \\
|
|
--numa 0 \\
|
|
--ostype l26 \\
|
|
--scsihw virtio-scsi-pci \\
|
|
--sockets 1 \\
|
|
--vga virtio \\
|
|
--onboot "\$onboot"
|
|
|
|
sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0"
|
|
sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null
|
|
disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')"
|
|
if [ -z "\$disk_volume" ]; then
|
|
echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2
|
|
exit 1
|
|
fi
|
|
sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume"
|
|
sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0"
|
|
|
|
if [ "\$start_vm" = "true" ]; then
|
|
sudo "\$qm_cmd" start "\$vmid"
|
|
fi
|
|
EOF
|
|
}
|
|
|
|
openwrt() {
|
|
require_debian_server "openwrt"
|
|
|
|
LAB_OPENWRT_VM=true run_openwrt_pipeline
|
|
}
|
|
|
|
cleanup_calico_links() {
|
|
ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true
|
|
sudo ip link delete vxlan.calico 2>/dev/null || true
|
|
sudo ip link delete tunl0 2>/dev/null || true
|
|
sudo ip link delete cni0 2>/dev/null || true
|
|
sudo ip link delete kube-ipvs0 2>/dev/null || true
|
|
ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true
|
|
}
|
|
|
|
cleanup_iptables() {
|
|
sudo iptables -F || true
|
|
sudo iptables -X || true
|
|
sudo iptables -t nat -F || true
|
|
sudo iptables -t nat -X || true
|
|
sudo iptables -t mangle -F || true
|
|
sudo iptables -t mangle -X || true
|
|
sudo iptables -t raw -F || true
|
|
sudo iptables -t raw -X || true
|
|
if command -v ipvsadm >/dev/null 2>&1; then
|
|
sudo ipvsadm --clear || true
|
|
fi
|
|
}
|
|
|
|
cleanup_calico_runtime_files() {
|
|
local path
|
|
|
|
for path in /run/calico /var/run/calico; do
|
|
if sudo test -e "${path}"; then
|
|
sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true
|
|
sudo rmdir "${path}" 2>/dev/null || true
|
|
fi
|
|
done
|
|
}
|
|
|
|
restore_node_dns() {
|
|
sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf
|
|
if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then
|
|
sudo rm -f /etc/resolv.conf
|
|
sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf
|
|
fi
|
|
sudo systemctl restart systemd-resolved 2>/dev/null || true
|
|
}
|
|
|
|
cleanup_mounts() {
|
|
if command -v findmnt >/dev/null 2>&1; then
|
|
local mount_root
|
|
while IFS= read -r mountpoint; do
|
|
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
|
|
done < <(
|
|
for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do
|
|
findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true
|
|
done | sort -ru
|
|
)
|
|
fi
|
|
while IFS= read -r mountpoint; do
|
|
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
|
|
done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true)
|
|
sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true
|
|
}
|
|
|
|
cleanup_node() {
|
|
sudo kubeadm reset --force || true
|
|
sudo systemctl stop kubelet 2>/dev/null || true
|
|
sudo systemctl stop containerd 2>/dev/null || true
|
|
sudo killall containerd-shim-runc-v2 2>/dev/null || true
|
|
|
|
cleanup_mounts
|
|
|
|
sudo rm -rf \
|
|
/etc/kubernetes/ \
|
|
/var/lib/etcd/ \
|
|
/var/lib/kubelet/ \
|
|
/var/lib/cni/ \
|
|
/etc/cni/net.d \
|
|
/run/flannel \
|
|
/var/lib/calico \
|
|
/var/log/calico \
|
|
/var/lib/containerd/* \
|
|
/run/containerd/* \
|
|
/etc/containerd/certs.d \
|
|
/etc/containerd/config.toml
|
|
cleanup_calico_runtime_files
|
|
sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam
|
|
|
|
cleanup_iptables
|
|
cleanup_calico_links
|
|
restore_node_dns
|
|
|
|
sudo mkdir -p /etc/containerd/certs.d
|
|
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
|
|
sudo systemctl start containerd 2>/dev/null || true
|
|
}
|
|
|
|
image_ref_tag() {
|
|
local image_ref="$1"
|
|
local tag
|
|
|
|
tag="${image_ref##*:}"
|
|
if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then
|
|
echo "Image reference ${image_ref} must include an immutable tag." >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\n' "${tag}"
|
|
}
|
|
|
|
website_image_tag() {
|
|
local source_hash="$1"
|
|
|
|
printf 'src-%s\n' "${source_hash:0:12}"
|
|
}
|
|
|
|
apps_registry_endpoint() {
|
|
if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then
|
|
printf '%s\n' "${TF_VAR_registry_endpoint}"
|
|
return 0
|
|
fi
|
|
|
|
demos_registry_endpoint
|
|
}
|
|
|
|
demos_registry_endpoint() {
|
|
local image
|
|
|
|
image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")"
|
|
if [[ -z "${image}" || "${image}" != */* ]]; then
|
|
echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2
|
|
exit 1
|
|
fi
|
|
|
|
printf '%s\n' "${image%%/*}"
|
|
}
|
|
|
|
website_source_hash() {
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
|
|
)
|
|
}
|
|
|
|
demos_source_hash() {
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
|
|
)
|
|
}
|
|
|
|
registry_image_exists() {
|
|
local registry_endpoint="$1"
|
|
local repository="$2"
|
|
local tag="$3"
|
|
local accept_header
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
return 1
|
|
fi
|
|
|
|
accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json"
|
|
curl -fsS \
|
|
-H "Accept: ${accept_header}" \
|
|
"http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null
|
|
}
|
|
|
|
image_state_value() {
|
|
local state_file="$1"
|
|
local key="$2"
|
|
|
|
awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true
|
|
}
|
|
|
|
website_image_is_current() {
|
|
local state_file="$1"
|
|
local source_hash="$2"
|
|
local platforms="$3"
|
|
local image_ref="$4"
|
|
local registry_endpoint="$5"
|
|
local image_tag
|
|
local saved_hash
|
|
local saved_platforms
|
|
local saved_image
|
|
|
|
[[ -f "${state_file}" ]] || return 1
|
|
|
|
saved_hash="$(image_state_value "${state_file}" source_hash)"
|
|
saved_platforms="$(image_state_value "${state_file}" platforms)"
|
|
saved_image="$(image_state_value "${state_file}" image)"
|
|
|
|
[[ "${saved_hash}" == "${source_hash}" ]] || return 1
|
|
[[ "${saved_platforms}" == "${platforms}" ]] || return 1
|
|
[[ "${saved_image}" == "${image_ref}" ]] || return 1
|
|
|
|
image_tag="$(image_ref_tag "${image_ref}")"
|
|
registry_image_exists "${registry_endpoint}" php-website "${image_tag}"
|
|
}
|
|
|
|
ensure_website_image_tag_not_reused() {
|
|
local state_file="$1"
|
|
local source_hash="$2"
|
|
local image_ref="$3"
|
|
local saved_hash
|
|
local saved_image
|
|
|
|
[[ -f "${state_file}" ]] || return 0
|
|
|
|
saved_hash="$(image_state_value "${state_file}" source_hash)"
|
|
saved_image="$(image_state_value "${state_file}" image)"
|
|
|
|
if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then
|
|
echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2
|
|
echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
demos_image_is_current() {
|
|
local state_file="$1"
|
|
local source_hash="$2"
|
|
local platforms="$3"
|
|
local image_ref="$4"
|
|
local registry_endpoint="$5"
|
|
local saved_hash
|
|
local saved_platforms
|
|
local saved_image
|
|
|
|
[[ -f "${state_file}" ]] || return 1
|
|
|
|
saved_hash="$(image_state_value "${state_file}" source_hash)"
|
|
saved_platforms="$(image_state_value "${state_file}" platforms)"
|
|
saved_image="$(image_state_value "${state_file}" image)"
|
|
|
|
[[ "${saved_hash}" == "${source_hash}" ]] || return 1
|
|
[[ "${saved_platforms}" == "${platforms}" ]] || return 1
|
|
[[ "${saved_image}" == "${image_ref}" ]] || return 1
|
|
|
|
registry_image_exists "${registry_endpoint}" demos-static latest
|
|
}
|
|
|
|
write_website_image_state() {
|
|
local state_file="$1"
|
|
local source_hash="$2"
|
|
local platforms="$3"
|
|
local image_ref="$4"
|
|
|
|
mkdir -p "$(dirname "${state_file}")"
|
|
{
|
|
printf 'source_hash=%s\n' "${source_hash}"
|
|
printf 'platforms=%s\n' "${platforms}"
|
|
printf 'image=%s\n' "${image_ref}"
|
|
} > "${state_file}"
|
|
}
|
|
|
|
write_demos_image_state() {
|
|
local state_file="$1"
|
|
local source_hash="$2"
|
|
local platforms="$3"
|
|
local image_ref="$4"
|
|
|
|
mkdir -p "$(dirname "${state_file}")"
|
|
{
|
|
printf 'source_hash=%s\n' "${source_hash}"
|
|
printf 'platforms=%s\n' "${platforms}"
|
|
printf 'image=%s\n' "${image_ref}"
|
|
} > "${state_file}"
|
|
}
|
|
|
|
path_available_mb() {
|
|
local path="$1"
|
|
|
|
while [[ ! -e "${path}" && "${path}" != "/" ]]; do
|
|
path="$(dirname "${path}")"
|
|
done
|
|
|
|
df -Pm "${path}" | awk 'NR == 2 {print $4}'
|
|
}
|
|
|
|
docker_root_dir() {
|
|
docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n'
|
|
}
|
|
|
|
prune_unused_docker_build_data() {
|
|
docker buildx rm lab-builder 2>/dev/null || true
|
|
docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true
|
|
docker builder prune -af 2>/dev/null || true
|
|
docker system prune -af 2>/dev/null || true
|
|
}
|
|
|
|
ensure_docker_build_space() {
|
|
local docker_root
|
|
local free_mb
|
|
local min_free_mb
|
|
|
|
min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}"
|
|
docker_root="$(docker_root_dir)"
|
|
free_mb="$(path_available_mb "${docker_root}")"
|
|
|
|
if (( free_mb >= min_free_mb )); then
|
|
return 0
|
|
fi
|
|
|
|
echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..."
|
|
prune_unused_docker_build_data
|
|
free_mb="$(path_available_mb "${docker_root}")"
|
|
|
|
if (( free_mb < min_free_mb )); then
|
|
echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2
|
|
echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2
|
|
echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
prepare_buildx_builder() {
|
|
local registry_endpoint="$1"
|
|
|
|
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
|
|
|
cat <<EOF > "${BUILDX_CONFIG}"
|
|
[registry."${registry_endpoint}"]
|
|
http = true
|
|
insecure = true
|
|
[registry."127.0.0.1:30500"]
|
|
http = true
|
|
insecure = true
|
|
[registry."localhost:30500"]
|
|
http = true
|
|
insecure = true
|
|
EOF
|
|
|
|
docker buildx rm lab-builder 2>/dev/null || true
|
|
docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use
|
|
docker buildx inspect --bootstrap
|
|
}
|
|
|
|
ensure_cosign_available() {
|
|
local arch
|
|
local cosign_url
|
|
local target_dir
|
|
local target_path
|
|
local tmp_path
|
|
|
|
if [[ -n "${COSIGN_BIN}" ]]; then
|
|
if [[ -x "${COSIGN_BIN}" ]]; then
|
|
return 0
|
|
fi
|
|
echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if command -v cosign >/dev/null 2>&1; then
|
|
COSIGN_BIN="$(command -v cosign)"
|
|
return 0
|
|
fi
|
|
|
|
case "$(uname -m)" in
|
|
x86_64 | amd64)
|
|
arch="amd64"
|
|
;;
|
|
aarch64 | arm64)
|
|
arch="arm64"
|
|
;;
|
|
*)
|
|
echo "Unsupported Cosign install architecture: $(uname -m)" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
target_dir="${HOMELAB_STATE_DIR}/bin"
|
|
target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}"
|
|
if [[ -x "${target_path}" ]]; then
|
|
COSIGN_BIN="${target_path}"
|
|
return 0
|
|
fi
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "${target_dir}"
|
|
tmp_path="${target_path}.tmp"
|
|
cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"
|
|
echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..."
|
|
curl -fsSL "${cosign_url}" -o "${tmp_path}"
|
|
chmod 0755 "${tmp_path}"
|
|
mv "${tmp_path}" "${target_path}"
|
|
COSIGN_BIN="${target_path}"
|
|
}
|
|
|
|
ensure_cosign_password() {
|
|
if [[ -n "${COSIGN_PASSWORD:-}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then
|
|
COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
|
|
export COSIGN_PASSWORD
|
|
return 0
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")"
|
|
if command -v openssl >/dev/null 2>&1; then
|
|
openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}"
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}"
|
|
import secrets
|
|
|
|
print(secrets.token_hex(32))
|
|
PY
|
|
else
|
|
echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2
|
|
exit 1
|
|
fi
|
|
chmod 0600 "${COSIGN_PASSWORD_FILE}"
|
|
COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
|
|
export COSIGN_PASSWORD
|
|
}
|
|
|
|
run_cosign() {
|
|
ensure_cosign_available
|
|
ensure_cosign_password
|
|
|
|
COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@"
|
|
}
|
|
|
|
ensure_cosign_keypair() {
|
|
ensure_cosign_available
|
|
ensure_cosign_password
|
|
|
|
if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
|
|
echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$(dirname "${COSIGN_KEY_PATH}")"
|
|
echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..."
|
|
COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}"
|
|
chmod 0600 "${COSIGN_KEY_PATH}"
|
|
chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}"
|
|
}
|
|
|
|
publish_cosign_public_key_config_map() {
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" create namespace kyverno --dry-run=client -o yaml |
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f -
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \
|
|
--from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \
|
|
--dry-run=client -o yaml |
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" apply -f -
|
|
}
|
|
|
|
cosign_registry_flags() {
|
|
local registry_endpoint="$1"
|
|
local image_ref="$2"
|
|
|
|
if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then
|
|
printf '%s\n' "--allow-http-registry"
|
|
fi
|
|
}
|
|
|
|
image_supply_chain_metadata_exists() {
|
|
local image_ref="$1"
|
|
local registry_endpoint="$2"
|
|
local digest_ref
|
|
local -a registry_flags=()
|
|
|
|
if ! digest_ref="$(image_digest_ref "${image_ref}")"; then
|
|
return 1
|
|
fi
|
|
|
|
mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")
|
|
|
|
run_cosign verify \
|
|
--key "${COSIGN_PUBLIC_KEY_PATH}" \
|
|
--insecure-ignore-tlog=true \
|
|
"${registry_flags[@]}" \
|
|
"${digest_ref}" >/dev/null 2>&1 &&
|
|
run_cosign verify-attestation \
|
|
--key "${COSIGN_PUBLIC_KEY_PATH}" \
|
|
--type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
|
|
--insecure-ignore-tlog=true \
|
|
"${registry_flags[@]}" \
|
|
"${digest_ref}" >/dev/null 2>&1
|
|
}
|
|
|
|
write_image_sbom_predicate() {
|
|
local image_ref="$1"
|
|
local output_file="$2"
|
|
local source_hash="$3"
|
|
|
|
docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}"
|
|
python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY'
|
|
from datetime import datetime, timezone
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
path = sys.argv[1]
|
|
image_ref = sys.argv[2]
|
|
source_hash = sys.argv[3]
|
|
|
|
try:
|
|
with open(path, encoding="utf-8") as handle:
|
|
document = json.load(handle)
|
|
except json.JSONDecodeError:
|
|
document = None
|
|
|
|
if not isinstance(document, dict):
|
|
safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-")
|
|
document = {
|
|
"spdxVersion": "SPDX-2.3",
|
|
"dataLicense": "CC0-1.0",
|
|
"SPDXID": "SPDXRef-DOCUMENT",
|
|
"name": f"homelab image SBOM for {image_ref}",
|
|
"documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}",
|
|
"creationInfo": {
|
|
"created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"creators": ["Tool: jeannie"],
|
|
},
|
|
"packages": [
|
|
{
|
|
"name": image_ref,
|
|
"SPDXID": "SPDXRef-Image",
|
|
"downloadLocation": "NOASSERTION",
|
|
"filesAnalyzed": False,
|
|
"licenseConcluded": "NOASSERTION",
|
|
"licenseDeclared": "NOASSERTION",
|
|
"copyrightText": "NOASSERTION",
|
|
}
|
|
],
|
|
"relationships": [
|
|
{
|
|
"spdxElementId": "SPDXRef-DOCUMENT",
|
|
"relationshipType": "DESCRIBES",
|
|
"relatedSpdxElement": "SPDXRef-Image",
|
|
}
|
|
],
|
|
}
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
json.dump(document, handle, sort_keys=True)
|
|
handle.write("\n")
|
|
|
|
if not isinstance(document, dict):
|
|
raise SystemExit("image SBOM predicate is not a JSON object")
|
|
if document.get("SPDXID") != "SPDXRef-DOCUMENT":
|
|
raise SystemExit("image SBOM predicate is not an SPDX document")
|
|
if not document.get("packages"):
|
|
raise SystemExit("image SBOM predicate does not include packages")
|
|
PY
|
|
}
|
|
|
|
image_digest_ref() {
|
|
local image_ref="$1"
|
|
local digest
|
|
local repository
|
|
|
|
if [[ "${image_ref}" == *@sha256:* ]]; then
|
|
printf '%s\n' "${image_ref}"
|
|
return 0
|
|
fi
|
|
|
|
digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')"
|
|
if [[ -z "${digest}" || "${digest}" == "<no value>" ]]; then
|
|
echo "Unable to resolve image digest for ${image_ref}." >&2
|
|
return 1
|
|
fi
|
|
|
|
if [[ "${image_ref##*/}" == *:* ]]; then
|
|
repository="${image_ref%:*}"
|
|
else
|
|
repository="${image_ref}"
|
|
fi
|
|
printf '%s@%s\n' "${repository}" "${digest}"
|
|
}
|
|
|
|
publish_image_supply_chain_metadata() {
|
|
local image_ref="$1"
|
|
local registry_endpoint="$2"
|
|
local source_hash="$3"
|
|
local digest_ref
|
|
local sbom_file
|
|
local -a registry_flags=()
|
|
|
|
if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
|
|
echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM."
|
|
return 0
|
|
fi
|
|
|
|
digest_ref="$(image_digest_ref "${image_ref}")"
|
|
mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")
|
|
sbom_file="$(mktemp)"
|
|
write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}"
|
|
|
|
echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..."
|
|
run_cosign sign \
|
|
--yes \
|
|
--key "${COSIGN_KEY_PATH}" \
|
|
--tlog-upload=false \
|
|
-a "homelab.dev/source-hash=${source_hash}" \
|
|
"${registry_flags[@]}" \
|
|
"${digest_ref}"
|
|
run_cosign attest \
|
|
--yes \
|
|
--key "${COSIGN_KEY_PATH}" \
|
|
--predicate "${sbom_file}" \
|
|
--type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
|
|
--tlog-upload=false \
|
|
"${registry_flags[@]}" \
|
|
"${digest_ref}"
|
|
rm -f "${sbom_file}"
|
|
|
|
if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
|
|
echo "Cosign verification failed after signing ${digest_ref}." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
dump_argocd_debug() {
|
|
local app="$1"
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" -o yaml || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd describe application "${app}" || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get pods -o wide || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs deployment/argocd-repo-server --tail=120 || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true
|
|
}
|
|
|
|
dump_namespace_debug() {
|
|
local namespace="$1"
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get all -o wide || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o wide || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe pods || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
|
|
}
|
|
|
|
wait_for_namespace() {
|
|
local namespace="$1"
|
|
local app="$2"
|
|
local timeout_seconds="$3"
|
|
local elapsed=0
|
|
|
|
until kubectl --kubeconfig "${KUBECONFIG_PATH}" get namespace "${namespace}" >/dev/null 2>&1; do
|
|
if ((elapsed >= timeout_seconds)); then
|
|
echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2
|
|
dump_argocd_debug "${app}"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|
|
|
|
wait_for_namespaced_resource() {
|
|
local namespace="$1"
|
|
local kind="$2"
|
|
local name="$3"
|
|
local app="$4"
|
|
local timeout_seconds="$5"
|
|
local elapsed=0
|
|
|
|
until kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do
|
|
if ((elapsed >= timeout_seconds)); then
|
|
echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2
|
|
dump_argocd_debug "${app}"
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|
|
|
|
wait_for_cluster_resource() {
|
|
local kind="$1"
|
|
local name="$2"
|
|
local app="$3"
|
|
local timeout_seconds="$4"
|
|
local elapsed=0
|
|
local api_resource
|
|
|
|
api_resource="${kind%%/*}"
|
|
if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get crd "${api_resource}" >/dev/null 2>&1; then
|
|
echo "Required CRD ${api_resource} is not installed; cannot wait for ${kind}/${name} from Argo CD app ${app}." >&2
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n argocd get application "${app}" \
|
|
-o jsonpath='sync={.status.sync.status} health={.status.health.status} message={.status.operationState.message}{"\n"}' 2>/dev/null || true
|
|
dump_argocd_debug "${app}"
|
|
exit 1
|
|
fi
|
|
|
|
until kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${kind}/${name}" >/dev/null 2>&1; do
|
|
if ((elapsed >= timeout_seconds)); then
|
|
echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2
|
|
dump_argocd_debug "${app}"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|
|
|
|
wait_for_deployment_ready() {
|
|
local namespace="$1"
|
|
local deployment="$2"
|
|
local app="$3"
|
|
local timeout_seconds="$4"
|
|
local desired_replicas
|
|
local ready_replicas
|
|
local elapsed=0
|
|
|
|
desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
|
|
desired_replicas="${desired_replicas:-1}"
|
|
|
|
until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \
|
|
(( ${ready_replicas:-0} >= desired_replicas )); do
|
|
if ((elapsed >= timeout_seconds)); then
|
|
echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2
|
|
dump_argocd_debug "${app}"
|
|
dump_namespace_debug "${namespace}"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
}
|
|
|
|
deploy_gitea() {
|
|
local mode="${LAB_GITEA_DEPLOY:-true}"
|
|
local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}"
|
|
local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}"
|
|
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}"
|
|
local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}"
|
|
local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
|
|
local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
|
|
local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}}"
|
|
local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}"
|
|
local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
|
|
local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml"
|
|
local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}"
|
|
|
|
require_debian_server "deploy-gitea"
|
|
|
|
if disabled_value "${mode}"; then
|
|
install_gitea_backup_timer
|
|
return 0
|
|
fi
|
|
|
|
if [[ ! -s "${compose_file}" ]]; then
|
|
echo "Missing ${compose_file}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..."
|
|
|
|
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea"
|
|
scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml"
|
|
|
|
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu
|
|
install_dir='${install_dir}'
|
|
install_docker='${install_docker}'
|
|
|
|
install_missing_packages() {
|
|
missing_packages=''
|
|
for package in \"\$@\"; do
|
|
if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then
|
|
missing_packages=\"\$missing_packages \$package\"
|
|
fi
|
|
done
|
|
if [ -n \"\$missing_packages\" ]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \$missing_packages
|
|
fi
|
|
}
|
|
|
|
install_missing_packages ca-certificates curl iptables
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
if [ \"\$install_docker\" = 'true' ]; then
|
|
curl -fsSL https://get.docker.com | sudo sh
|
|
else
|
|
echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if ! sudo docker compose version >/dev/null 2>&1; then
|
|
install_missing_packages docker-compose-plugin
|
|
fi
|
|
|
|
repair_docker_iptables() {
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...'
|
|
sudo systemctl restart docker
|
|
sleep 3
|
|
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
echo 'Docker NAT chain is still missing after restarting Docker.' >&2
|
|
sudo iptables -t nat -S >&2 || true
|
|
sudo systemctl status docker --no-pager -l >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
repair_docker_iptables
|
|
|
|
sudo mkdir -p \"\$install_dir/data\"
|
|
sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\"
|
|
sudo chown -R 1000:1000 \"\$install_dir/data\"
|
|
sudo tee \"\$install_dir/.env\" >/dev/null <<ENV_EOT
|
|
GITEA_IMAGE=${image}
|
|
GITEA_CONTAINER_NAME=${container_name}
|
|
GITEA_HTTP_PORT=${http_port}
|
|
GITEA_SSH_PORT=${ssh_port}
|
|
GITEA_DOMAIN=${domain}
|
|
GITEA_ROOT_URL=${root_url}
|
|
GITEA_UID=1000
|
|
GITEA_GID=1000
|
|
ENV_EOT
|
|
|
|
cd \"\$install_dir\"
|
|
sudo docker compose pull
|
|
sudo docker compose up -d --remove-orphans
|
|
sudo docker compose ps
|
|
"
|
|
|
|
install_gitea_backup_timer
|
|
}
|
|
|
|
install_missing_debian_packages() {
|
|
local missing_packages=()
|
|
local package
|
|
|
|
for package in "$@"; do
|
|
if ! dpkg-query -W -f='${Status}' "${package}" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages+=("${package}")
|
|
fi
|
|
done
|
|
|
|
if ((${#missing_packages[@]})); then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
|
|
fi
|
|
}
|
|
|
|
ensure_local_docker_compose() {
|
|
local command_name="$1"
|
|
local install_docker="$2"
|
|
|
|
install_missing_debian_packages ca-certificates curl iptables
|
|
|
|
if ! command -v docker >/dev/null 2>&1; then
|
|
if [[ "${install_docker}" == "true" ]]; then
|
|
curl -fsSL https://get.docker.com | sudo sh
|
|
else
|
|
echo "Docker is not installed on the Debian host. Install Docker through host bootstrap first, or rerun with LAB_${command_name}_INSTALL_DOCKER=true to allow get.docker.com installation." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if ! sudo docker compose version >/dev/null 2>&1; then
|
|
install_missing_debian_packages docker-compose-plugin
|
|
fi
|
|
}
|
|
|
|
repair_local_docker_iptables() {
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Docker NAT chain is missing on the Debian host; restarting Docker once to restore iptables state..."
|
|
sudo systemctl restart docker
|
|
sleep 3
|
|
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
return 0
|
|
fi
|
|
|
|
echo "Docker NAT chain is still missing after restarting Docker." >&2
|
|
sudo iptables -t nat -S >&2 || true
|
|
sudo systemctl status docker --no-pager -l >&2 || true
|
|
exit 1
|
|
}
|
|
|
|
deploy_heimdall() {
|
|
local mode="${LAB_HEIMDALL_DEPLOY:-true}"
|
|
local install_dir="${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}"
|
|
local data_dir="${LAB_HEIMDALL_DATA_DIR:-${install_dir}/data}"
|
|
local config_dir="${LAB_HEIMDALL_CONFIG_DIR:-${data_dir}/config}"
|
|
local http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}"
|
|
local bind_ip="${LAB_HEIMDALL_BIND_IP:-0.0.0.0}"
|
|
local public_url="${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:-lab2025.duckdns.org}/}"
|
|
local image="${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347}"
|
|
local container_name="${HEIMDALL_CONTAINER_NAME:-homelab-heimdall}"
|
|
local seeder_container_name="${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder}"
|
|
local network="${HEIMDALL_NETWORK:-homelab-heimdall}"
|
|
local install_docker="${LAB_HEIMDALL_INSTALL_DOCKER:-false}"
|
|
local source_dir="${REPO_ROOT}/infra/heimdall"
|
|
local elapsed=0
|
|
local value_name
|
|
local value
|
|
|
|
require_debian_server "deploy-heimdall"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
for value_name in install_dir data_dir config_dir http_port bind_ip public_url image container_name seeder_container_name network install_docker; do
|
|
value="${!value_name}"
|
|
if [[ "${value}" == *$'\n'* ]]; then
|
|
echo "${value_name} cannot contain a newline." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/links.json" || ! -s "${source_dir}/seed-heimdall.py" ]]; then
|
|
echo "Missing Heimdall source files in ${source_dir}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Deploying external Heimdall on the Debian host at ${install_dir}..."
|
|
|
|
ensure_local_docker_compose "HEIMDALL" "${install_docker}"
|
|
repair_local_docker_iptables
|
|
|
|
sudo mkdir -p "${install_dir}" "${config_dir}"
|
|
sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
|
|
sudo cp "${source_dir}/links.json" "${install_dir}/links.json"
|
|
sudo cp "${source_dir}/seed-heimdall.py" "${install_dir}/seed-heimdall.py"
|
|
sudo chown -R 1000:1000 "${data_dir}"
|
|
sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
|
|
HEIMDALL_IMAGE=${image}
|
|
HEIMDALL_CONTAINER_NAME=${container_name}
|
|
HEIMDALL_SEEDER_CONTAINER_NAME=${seeder_container_name}
|
|
HEIMDALL_HTTP_PORT=${http_port}
|
|
HEIMDALL_BIND_IP=${bind_ip}
|
|
HEIMDALL_PUBLIC_URL=${public_url%/}
|
|
HEIMDALL_CONFIG_DIR=${config_dir}
|
|
HEIMDALL_UID=1000
|
|
HEIMDALL_GID=1000
|
|
HEIMDALL_NETWORK=${network}
|
|
TZ=America/Mexico_City
|
|
ENV_EOT
|
|
|
|
(
|
|
cd "${install_dir}"
|
|
sudo docker compose pull
|
|
sudo docker compose up -d --remove-orphans
|
|
sudo docker compose ps
|
|
)
|
|
|
|
until curl -fsS --connect-timeout 5 "http://127.0.0.1:${http_port}/" >/dev/null; do
|
|
if ((elapsed >= 180)); then
|
|
echo "Timed out waiting for Heimdall on http://127.0.0.1:${http_port}/" >&2
|
|
(cd "${install_dir}" && sudo docker compose ps && sudo docker compose logs --tail=120) >&2 || true
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
elapsed=$((elapsed + 5))
|
|
done
|
|
|
|
echo "Heimdall is reachable at http://127.0.0.1:${http_port}/ and ${public_url%/}/."
|
|
}
|
|
|
|
deploy_matrix() {
|
|
local mode="${LAB_MATRIX_DEPLOY:-true}"
|
|
local install_dir="${LAB_MATRIX_INSTALL_DIR:-/home/jv/matrix-server}"
|
|
local image=""
|
|
local container_name=""
|
|
local http_port=""
|
|
local ssh_port=""
|
|
local domain=""
|
|
local install_docker="false"
|
|
local source_dir="${REPO_ROOT}/infra/matrix-server"
|
|
local value_name
|
|
local value
|
|
|
|
require_debian_server "deploy-matrix"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
for value_name in install_dir; do
|
|
value="${!value_name}"
|
|
if [[ "${value}" == *$'\n'* ]]; then
|
|
echo "${value_name} cannot contain a newline." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/homeserver.yaml.tpl" || ! -s "${source_dir}/bootstrap.sh" ]]; then
|
|
echo "Missing Matrix source files in ${source_dir}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Deploying Matrix stack on the Debian host at ${install_dir}..."
|
|
|
|
ensure_local_docker_compose "MATRIX" "${install_docker}"
|
|
repair_local_docker_iptables
|
|
|
|
sudo mkdir -p "${install_dir}"
|
|
sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
|
|
sudo cp "${source_dir}/homeserver.yaml.tpl" "${install_dir}/homeserver.yaml.tpl"
|
|
sudo cp "${source_dir}/log_config.yaml" "${install_dir}/log_config.yaml"
|
|
sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json" 2>/dev/null || {
|
|
sudo mkdir -p "${install_dir}/element-config"
|
|
sudo cp "${source_dir}/element-config/config.json" "${install_dir}/element-config/config.json"
|
|
}
|
|
sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh"
|
|
sudo chmod +x "${install_dir}/bootstrap.sh"
|
|
|
|
# Secrets come from the SOPS-encrypted repo file, decrypted at deploy time.
|
|
# (Env vars override the file, for manual/ops overrides.)
|
|
if ! sops_load_secrets "${REPO_ROOT}/infra/matrix-server/matrix.secret.yaml"; then
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${MATRIX_POSTGRES_PASSWORD:-}" || -z "${MATRIX_REGISTRATION_SHARED_SECRET:-}" ]]; then
|
|
echo "MATRIX_POSTGRES_PASSWORD and MATRIX_REGISTRATION_SHARED_SECRET missing after decrypt." >&2
|
|
exit 1
|
|
fi
|
|
|
|
local matrix_server_name="${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}"
|
|
local matrix_synapse_port="${LAB_MATRIX_SYNAPSE_PORT:-8008}"
|
|
local matrix_element_port="${LAB_MATRIX_ELEMENT_PORT:-8081}"
|
|
local postgres_db="${LAB_MATRIX_POSTGRES_DB:-synapse}"
|
|
local postgres_user="${LAB_MATRIX_POSTGRES_USER:-synapse}"
|
|
|
|
sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
|
|
MATRIX_SERVER_NAME=${matrix_server_name}
|
|
MATRIX_SYNAPSE_PORT=${matrix_synapse_port}
|
|
MATRIX_ELEMENT_PORT=${matrix_element_port}
|
|
MATRIX_SYNAPSE_CONTAINER_NAME=${LAB_MATRIX_SYNAPSE_CONTAINER_NAME:-matrix-synapse}
|
|
MATRIX_POSTGRES_CONTAINER_NAME=${LAB_MATRIX_POSTGRES_CONTAINER_NAME:-matrix-postgres}
|
|
MATRIX_ELEMENT_CONTAINER_NAME=${LAB_MATRIX_ELEMENT_CONTAINER_NAME:-matrix-element}
|
|
MATRIX_POSTGRES_DB=${postgres_db}
|
|
MATRIX_POSTGRES_USER=${postgres_user}
|
|
MATRIX_POSTGRES_PASSWORD=${MATRIX_POSTGRES_PASSWORD}
|
|
MATRIX_REGISTRATION_SHARED_SECRET=${MATRIX_REGISTRATION_SHARED_SECRET}
|
|
MATRIX_MACAROON_SECRET_KEY=${MATRIX_MACAROON_SECRET_KEY:-${MATRIX_REGISTRATION_SHARED_SECRET}}
|
|
MATRIX_FORM_SECRET=${MATRIX_FORM_SECRET:-${MATRIX_REGISTRATION_SHARED_SECRET}}
|
|
ENV_EOT
|
|
|
|
# Pull + render + chown + up. bootstrap.sh is the single source of truth.
|
|
# sudo the docker compose commands; data dirs are chown'd inside bootstrap.
|
|
(
|
|
cd "${install_dir}"
|
|
sudo docker compose --env-file "${install_dir}/.env" -f "${install_dir}/docker-compose.yml" pull
|
|
sudo MATRIX_ENV_FILE="${install_dir}/.env" bash "${install_dir}/bootstrap.sh"
|
|
)
|
|
|
|
echo "Matrix stack is reachable at http://127.0.0.1:${matrix_synapse_port}/ and Element at http://127.0.0.1:${matrix_element_port}/."
|
|
}
|
|
|
|
deploy_omniroute() {
|
|
local mode="${LAB_OMNIROUTE_DEPLOY:-true}"
|
|
local install_dir="${LAB_OMNIROUTE_INSTALL_DIR:-/home/jv/homelab-omniroute}"
|
|
local install_docker="false"
|
|
local source_dir="${REPO_ROOT}/infra/omniroute"
|
|
|
|
require_debian_server "deploy-omniroute"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/bootstrap.sh" ]]; then
|
|
echo "Missing OmniRoute source files in ${source_dir}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Deploying OmniRoute LLM gateway on the Debian host at ${install_dir}..."
|
|
|
|
ensure_local_docker_compose "OMNIROUTE" "${install_docker}"
|
|
repair_local_docker_iptables
|
|
|
|
sudo mkdir -p "${install_dir}"
|
|
sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml"
|
|
sudo cp "${source_dir}/bootstrap.sh" "${install_dir}/bootstrap.sh"
|
|
sudo chmod +x "${install_dir}/bootstrap.sh"
|
|
|
|
sudo tee "${install_dir}/.env" >/dev/null <<ENV_EOT
|
|
OMNIROUTE_IMAGE=${LAB_OMNIROUTE_IMAGE:-diegosouzapw/omniroute:latest}
|
|
OMNIROUTE_CONTAINER_NAME=${LAB_OMNIROUTE_CONTAINER_NAME:-omniroute}
|
|
OMNIROUTE_PORT=${LAB_OMNIROUTE_PORT:-20128}
|
|
ENV_EOT
|
|
|
|
(
|
|
cd "${install_dir}"
|
|
sudo docker compose pull
|
|
sudo bash "${install_dir}/bootstrap.sh"
|
|
)
|
|
|
|
echo "OmniRoute is reachable at http://127.0.0.1:${LAB_OMNIROUTE_PORT:-20128}."
|
|
}
|
|
|
|
deploy_hermes() {
|
|
local mode="${LAB_HERMES_DEPLOY:-true}"
|
|
local hermes_home="${LAB_HERMES_HOME:-/home/jv/.hermes}"
|
|
local install_dir="${LAB_HERMES_INSTALL_DIR:-/home/jv/.hermes/hermes-agent}"
|
|
local uv_bin="${LAB_HERMES_UV_BIN:-/home/jv/.hermes/bin/uv}"
|
|
local install_docker="${LAB_HERMES_INSTALL_DOCKER:-false}"
|
|
local value_name
|
|
local value
|
|
|
|
require_debian_server "deploy-hermes"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
for value_name in hermes_home install_dir uv_bin; do
|
|
value="${!value_name}"
|
|
if [[ "${value}" == *$'\n'* ]]; then
|
|
echo "${value_name} cannot contain a newline." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "Deploying Hermes agent on the Debian host at ${install_dir}..."
|
|
|
|
ensure_local_docker_compose "HERMES" "${install_docker}"
|
|
|
|
# Install uv if missing.
|
|
if [[ ! -x "${uv_bin}" ]]; then
|
|
mkdir -p "$(dirname "${uv_bin}")"
|
|
echo "Installing uv..."
|
|
curl -LsSf https://astral.sh/uv/install.sh | bash
|
|
# uv installs to ~/.local/bin; copy/alias into HERMES bin if that path differs.
|
|
if command -v uv >/dev/null 2>&1 && [[ ! -x "${uv_bin}" ]]; then
|
|
cp "$(command -v uv)" "${uv_bin}"
|
|
fi
|
|
fi
|
|
|
|
# Install Hermes via its official installer (git method, managed venv layout).
|
|
if [[ ! -x "${hermes_home}/hermes-agent/venv/bin/python" ]]; then
|
|
echo "Installing Hermes agent..."
|
|
curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash
|
|
fi
|
|
|
|
# Ensure the Matrix platform dependencies (E2EE) are present in the venv.
|
|
if [[ -x "${uv_bin}" && -f "${install_dir}/pyproject.toml" ]]; then
|
|
cd "${install_dir}"
|
|
"${uv_bin}" pip install 'mautrix[encryption]' -p venv/bin/python
|
|
fi
|
|
|
|
# Secret env for the gateway (.env under HERMES_HOME, mode 600, never committed).
|
|
# Pull the hermes secrets from the SOPS-encrypted repo file; env vars override.
|
|
if ! sops_load_secrets "${REPO_ROOT}/infra/hermes.secret.yaml"; then
|
|
echo "WARNING: could not load hermes secrets; MATRIX_PASSWORD may be blank." >&2
|
|
fi
|
|
|
|
local secrets_file="${hermes_home}/.env"
|
|
sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "${hermes_home}"
|
|
if [[ -f "${secrets_file}" ]]; then
|
|
sudo cp "${secrets_file}" "${secrets_file}.bak-pre-jeannie"
|
|
fi
|
|
sudo tee "${secrets_file}" >/dev/null <<ENV_EOT
|
|
OPENAI_BASE_URL=${HERMES_OPENAI_BASE_URL:-http://localhost:${LAB_OMNIROUTE_PORT:-20128}/v1}
|
|
HERMES_CUSTOM_LOCALHOST_${LAB_OMNIROUTE_PORT:-20128}_API_KEY=${HERMES_OMNIROUTE_API_KEY:-}
|
|
MATRIX_HOMESERVER=${LAB_MATRIX_SERVER_NAME:-matrix.lab2025.duckdns.org}
|
|
MATRIX_USER_ID=${HERMES_MATRIX_USER_ID:-@hermes:matrix.lab2025.duckdns.org}
|
|
MATRIX_PASSWORD=${MATRIX_BOT_PASSWORD:-}
|
|
MATRIX_ALLOWED_USERS=${HERMES_MATRIX_ALLOWED_USERS:-@jv:matrix.lab2025.duckdns.org}
|
|
MATRIX_E2EE_MODE=required
|
|
ENV_EOT
|
|
sudo chmod 600 "${secrets_file}"
|
|
|
|
# Create the Matrix bot account (idempotent) if the homeserver is up.
|
|
if command -v docker >/dev/null 2>&1 && docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^matrix-synapse$'; then
|
|
if [[ -n "${MATRIX_BOT_PASSWORD:-}" ]]; then
|
|
if ! docker exec matrix-synapse register_new_matrix_user -c /data/homeserver.yaml \
|
|
-u hermes -p "${MATRIX_BOT_PASSWORD}" --no-admin >/dev/null 2>&1; then
|
|
echo "Hermes bot account already exists (expected on re-run) or auto-registration skipped." >&2
|
|
else
|
|
echo "Hermes Matrix bot account ready."
|
|
fi
|
|
else
|
|
echo "MATRIX_BOT_PASSWORD not set; skipping Hermes bot account registration (needed only on fresh install)." >&2
|
|
fi
|
|
fi
|
|
|
|
# Install/start the gateway systemd user service (Hermes manages the unit).
|
|
"${hermes_home}/hermes-agent/venv/bin/python" "${hermes_home}/hermes-agent/hermes" gateway install
|
|
systemctl --user daemon-reload
|
|
systemctl --user enable --now hermes-gateway 2>/dev/null || true
|
|
|
|
echo "Hermes agent deployed. Run 'hermes' to configure the model, or check gateway logs at ${hermes_home}/logs/gateway.log."
|
|
}
|
|
|
|
deploy_rpi_services() {
|
|
local mode="${LAB_RPI_SERVICES_DEPLOY:-true}"
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
|
|
local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}"
|
|
local docker_nvme_root="${LAB_RPI_DOCKER_ROOT:-${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}}"
|
|
local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
|
|
local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}"
|
|
local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}"
|
|
local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}"
|
|
local source_dir="${REPO_ROOT}/infra/rpi-services"
|
|
local value_name
|
|
local value
|
|
|
|
require_debian_server "rpi-services"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
if [[ ! -d "${source_dir}" ]]; then
|
|
echo "Missing ${source_dir}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do
|
|
value="${!value_name}"
|
|
if [[ "${value}" == *"'"* ]]; then
|
|
echo "${value_name} cannot contain a single quote." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "Deploying RPi services on ${rpi_user}@${rpi_host}..."
|
|
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services"
|
|
scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
|
"${source_dir}/docker-compose.yml" \
|
|
"${source_dir}/adlists.txt" \
|
|
"${source_dir}/local-dns-records.txt" \
|
|
"${source_dir}/cname-records.txt" \
|
|
"${source_dir}/static-dhcp-hosts.txt" \
|
|
"${source_dir}/uptime-kuma-monitors.json" \
|
|
"${source_dir}/seed-uptime-kuma.js" \
|
|
"${source_dir}/bootstrap.sh" \
|
|
"${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/"
|
|
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
|
|
LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \
|
|
LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \
|
|
LAB_RPI_DOCKER_ROOT='${docker_nvme_root}' \
|
|
LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \
|
|
LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \
|
|
LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \
|
|
LAB_RPI_INSTALL_DOCKER='${install_docker}' \
|
|
PIHOLE_WEBPASSWORD='${pihole_webpassword}' \
|
|
bash /tmp/homelab-rpi-services/bootstrap.sh"
|
|
}
|
|
|
|
gitea_bootstrap_password() {
|
|
if command -v openssl >/dev/null 2>&1; then
|
|
openssl rand -hex 32
|
|
return 0
|
|
fi
|
|
|
|
python3 - <<'PY'
|
|
import secrets
|
|
|
|
print(secrets.token_hex(32))
|
|
PY
|
|
}
|
|
|
|
gitea_api_base_url() {
|
|
local gitea_host="$1"
|
|
local http_port="$2"
|
|
local candidate
|
|
local api_base_override="${LAB_GITEA_API_BASE_URL:-}"
|
|
|
|
if [[ -n "${api_base_override}" ]]; then
|
|
printf '%s\n' "${api_base_override%/}"
|
|
return 0
|
|
fi
|
|
|
|
for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do
|
|
if curl -fsS "${candidate}/version" >/dev/null 2>&1; then
|
|
printf '%s\n' "${candidate}"
|
|
return 0
|
|
fi
|
|
done
|
|
|
|
echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2
|
|
exit 1
|
|
}
|
|
|
|
gitea_repo_exists() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local owner="$4"
|
|
local repo_name="$5"
|
|
local status
|
|
|
|
status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")"
|
|
case "${status}" in
|
|
200)
|
|
return 0
|
|
;;
|
|
404)
|
|
return 1
|
|
;;
|
|
401 | 403)
|
|
echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
gitea_branch_exists() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local owner="$4"
|
|
local repo_name="$5"
|
|
local branch="$6"
|
|
local status
|
|
|
|
status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")"
|
|
case "${status}" in
|
|
200)
|
|
return 0
|
|
;;
|
|
404)
|
|
return 1
|
|
;;
|
|
401 | 403)
|
|
echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2
|
|
exit 1
|
|
;;
|
|
*)
|
|
echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
create_gitea_repo() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local repo_name="$4"
|
|
local default_branch="$5"
|
|
local payload
|
|
|
|
payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
repo_name, default_branch = sys.argv[1:3]
|
|
print(json.dumps({
|
|
"name": repo_name,
|
|
"private": False,
|
|
"auto_init": False,
|
|
"default_branch": default_branch,
|
|
"description": "Homelab infrastructure configuration",
|
|
}))
|
|
PY
|
|
)"
|
|
|
|
curl -fsS \
|
|
-u "${auth_user}:${auth_password}" \
|
|
-H "Content-Type: application/json" \
|
|
-X POST \
|
|
-d "${payload}" \
|
|
"${api_base}/user/repos" >/dev/null
|
|
}
|
|
|
|
gitea_public_key_registered() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local owner="$4"
|
|
local repo_name="$5"
|
|
local public_key_path="$6"
|
|
local repo_keys
|
|
local user_keys
|
|
|
|
user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")"
|
|
repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")"
|
|
|
|
GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
|
|
GITEA_USER_KEYS="${user_keys}" \
|
|
GITEA_REPO_KEYS="${repo_keys}" \
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
public_key = os.environ["GITEA_PUBLIC_KEY"].strip()
|
|
for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"):
|
|
for key in json.loads(os.environ[env_name]) or []:
|
|
if key.get("key", "").strip() == public_key:
|
|
sys.exit(0)
|
|
sys.exit(1)
|
|
PY
|
|
}
|
|
|
|
create_gitea_repo_deploy_key() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local owner="$4"
|
|
local repo_name="$5"
|
|
local title="$6"
|
|
local public_key_path="$7"
|
|
local read_only="$8"
|
|
local payload
|
|
|
|
payload="$(
|
|
GITEA_DEPLOY_KEY_TITLE="${title}" \
|
|
GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
|
|
GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \
|
|
python3 - <<'PY'
|
|
import json
|
|
import os
|
|
|
|
print(json.dumps({
|
|
"title": os.environ["GITEA_DEPLOY_KEY_TITLE"],
|
|
"key": os.environ["GITEA_PUBLIC_KEY"].strip(),
|
|
"read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true",
|
|
}))
|
|
PY
|
|
)"
|
|
|
|
curl -fsS \
|
|
-u "${auth_user}:${auth_password}" \
|
|
-H "Content-Type: application/json" \
|
|
-X POST \
|
|
-d "${payload}" \
|
|
"${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null
|
|
}
|
|
|
|
ensure_gitea_repo_ssh_access() {
|
|
local api_base="$1"
|
|
local auth_user="$2"
|
|
local auth_password="$3"
|
|
local owner="$4"
|
|
local repo_name="$5"
|
|
local ssh_host="$6"
|
|
local ssh_port="$7"
|
|
local key_path="$8"
|
|
local key_title="$9"
|
|
local key_read_only="${10}"
|
|
local key_dir
|
|
local known_hosts
|
|
local public_key_path
|
|
local read_only_json="false"
|
|
local ssh_repo_url
|
|
|
|
if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then
|
|
echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2
|
|
exit 1
|
|
fi
|
|
|
|
key_dir="$(dirname "${key_path}")"
|
|
public_key_path="${key_path}.pub"
|
|
mkdir -p "${key_dir}"
|
|
chmod 0700 "${key_dir}"
|
|
|
|
if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then
|
|
ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null
|
|
elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then
|
|
ssh-keygen -y -f "${key_path}" >"${public_key_path}"
|
|
elif [[ ! -s "${key_path}" ]]; then
|
|
echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2
|
|
exit 1
|
|
fi
|
|
|
|
chmod 0600 "${key_path}"
|
|
chmod 0644 "${public_key_path}"
|
|
|
|
if truthy "${key_read_only}"; then
|
|
read_only_json="true"
|
|
fi
|
|
|
|
if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then
|
|
echo "Gitea already has Debian host SSH key ${public_key_path}."
|
|
else
|
|
create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}"
|
|
echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}."
|
|
fi
|
|
|
|
known_hosts="${HOME}/.ssh/known_hosts"
|
|
touch "${known_hosts}"
|
|
chmod 0644 "${known_hosts}"
|
|
if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then
|
|
ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null
|
|
fi
|
|
|
|
ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git"
|
|
git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null ||
|
|
git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}"
|
|
git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"
|
|
git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null
|
|
echo "Gitea SSH remote: ${ssh_repo_url}"
|
|
}
|
|
|
|
bootstrap_gitea_repo() {
|
|
local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}"
|
|
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
|
|
local gitea_user="${LAB_GITEA_USER:-jv}"
|
|
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
|
|
local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
|
|
local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
|
|
local root_url="${LAB_GITEA_ROOT_URL:?LAB_GITEA_ROOT_URL is required from homelab.yml}"
|
|
local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}"
|
|
local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}"
|
|
local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}"
|
|
local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}"
|
|
local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}"
|
|
local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}"
|
|
local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}"
|
|
local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}"
|
|
local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}"
|
|
local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
|
|
local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}"
|
|
local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}"
|
|
local api_base
|
|
local public_repo_url
|
|
local direct_repo_url
|
|
local push_url
|
|
local askpass
|
|
local credentials_dir
|
|
local remote_status
|
|
local worktree_status
|
|
|
|
require_debian_server "bootstrap-gitea-repo"
|
|
|
|
if disabled_value "${mode}"; then
|
|
return 0
|
|
fi
|
|
|
|
ensure_python3
|
|
for value_name in repo_owner repo_name default_branch bootstrap_user; do
|
|
local value="${!value_name}"
|
|
if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
|
|
echo "${value_name} contains unsupported characters." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
if [[ "${bootstrap_email}" == *"'"* ]]; then
|
|
echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2
|
|
exit 1
|
|
fi
|
|
if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then
|
|
echo "LAB_GITEA_SSH_PORT must be numeric." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "${credentials_file}"
|
|
bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}"
|
|
bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}"
|
|
bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}"
|
|
fi
|
|
|
|
if [[ -z "${bootstrap_password}" ]]; then
|
|
bootstrap_password="$(gitea_bootstrap_password)"
|
|
credentials_dir="$(dirname "${credentials_file}")"
|
|
mkdir -p "${credentials_dir}"
|
|
chmod 0700 "${credentials_dir}"
|
|
{
|
|
printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}"
|
|
printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}"
|
|
printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}"
|
|
} > "${credentials_file}"
|
|
chmod 0600 "${credentials_file}"
|
|
echo "Generated Gitea bootstrap credentials at ${credentials_file}."
|
|
fi
|
|
|
|
for value_name in repo_owner repo_name default_branch bootstrap_user; do
|
|
local value="${!value_name}"
|
|
if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
|
|
echo "${value_name} contains unsupported characters." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
for value_name in bootstrap_email bootstrap_password; do
|
|
local value="${!value_name}"
|
|
if [[ "${value}" == *"'"* ]]; then
|
|
echo "${value_name} cannot contain a single quote." >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..."
|
|
|
|
# shellcheck disable=SC2087
|
|
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" <<EOF
|
|
set -euo pipefail
|
|
|
|
container_name='${container_name}'
|
|
bootstrap_user='${bootstrap_user}'
|
|
bootstrap_email='${bootstrap_email}'
|
|
bootstrap_password='${bootstrap_password}'
|
|
|
|
if ! sudo docker inspect "\${container_name}" >/dev/null 2>&1; then
|
|
echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2
|
|
exit 1
|
|
fi
|
|
|
|
for attempt in \$(seq 1 60); do
|
|
if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 ||
|
|
curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then
|
|
break
|
|
fi
|
|
if [ "\${attempt}" = "60" ]; then
|
|
echo "Timed out waiting for Gitea API inside \${container_name}." >&2
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \
|
|
--username "\${bootstrap_user}" \
|
|
--password "\${bootstrap_password}" \
|
|
--email "\${bootstrap_email}" \
|
|
--admin \
|
|
--must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then
|
|
if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \
|
|
--username "\${bootstrap_user}" \
|
|
--password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || {
|
|
cat /tmp/homelab-gitea-user-password.log >&2
|
|
exit 1
|
|
}
|
|
else
|
|
cat /tmp/homelab-gitea-user-create.log >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
EOF
|
|
|
|
api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")"
|
|
|
|
if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then
|
|
echo "Gitea repository ${repo_owner}/${repo_name} already exists."
|
|
else
|
|
if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then
|
|
echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2
|
|
exit 1
|
|
fi
|
|
create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}"
|
|
echo "Created Gitea repository ${repo_owner}/${repo_name}."
|
|
fi
|
|
|
|
public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git"
|
|
if [[ "${api_base}" == */git/api/v1 ]]; then
|
|
direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git"
|
|
else
|
|
direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git"
|
|
fi
|
|
push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}"
|
|
|
|
git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null
|
|
git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null ||
|
|
git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}"
|
|
|
|
if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then
|
|
echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged."
|
|
else
|
|
worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)"
|
|
if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then
|
|
echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2
|
|
echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2
|
|
exit 1
|
|
fi
|
|
|
|
askpass="$(mktemp)"
|
|
trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT
|
|
cat > "${askpass}" <<ASKPASS_EOT
|
|
#!/usr/bin/env bash
|
|
case "\$1" in
|
|
*Username*) printf '%s\n' '${bootstrap_user}' ;;
|
|
*Password*) printf '%s\n' '${bootstrap_password}' ;;
|
|
*) printf '\n' ;;
|
|
esac
|
|
ASKPASS_EOT
|
|
chmod 0700 "${askpass}"
|
|
|
|
GIT_ASKPASS="${askpass}" GIT_TERMINAL_PROMPT=0 \
|
|
git -C "${REPO_ROOT}" push "${push_url}" "HEAD:refs/heads/${default_branch}"
|
|
rm -f "${askpass}"
|
|
trap 'rm -f "${BUILDX_CONFIG}"' EXIT
|
|
echo "Pushed current HEAD to Gitea branch ${default_branch}."
|
|
fi
|
|
|
|
remote_status="$(git -C "${REPO_ROOT}" remote get-url gitea)"
|
|
echo "Gitea remote: ${remote_status}"
|
|
if ! disabled_value "${ssh_bootstrap}"; then
|
|
ensure_gitea_repo_ssh_access \
|
|
"${api_base}" \
|
|
"${bootstrap_user}" \
|
|
"${bootstrap_password}" \
|
|
"${repo_owner}" \
|
|
"${repo_name}" \
|
|
"${gitea_host}" \
|
|
"${ssh_port}" \
|
|
"${ssh_key_path}" \
|
|
"${ssh_key_title}" \
|
|
"${ssh_key_read_only}"
|
|
fi
|
|
}
|
|
|
|
install_gitea_backup_timer() {
|
|
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
|
|
local gitea_user="${LAB_GITEA_USER:-jv}"
|
|
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local gitea_container="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
|
|
local backup_dir="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
|
|
local backup_script="/usr/local/sbin/homelab-gitea-backup.sh"
|
|
local restore_drill_script="/usr/local/sbin/homelab-gitea-restore-drill.sh"
|
|
|
|
sudo tee "${backup_script}" >/dev/null <<BACKUP_SCRIPT_EOT
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
GITEA_HOST="\${GITEA_HOST:-${gitea_host}}"
|
|
GITEA_USER="\${GITEA_USER:-${gitea_user}}"
|
|
GITEA_SSH_KEY_PATH="\${GITEA_SSH_KEY_PATH:-${gitea_key}}"
|
|
GITEA_CONTAINER="\${GITEA_CONTAINER:-${gitea_container}}"
|
|
GITEA_BACKUP_DIR="\${GITEA_BACKUP_DIR:-${backup_dir}}"
|
|
GITEA_BACKUP_RETENTION_DAYS="\${GITEA_BACKUP_RETENTION_DAYS:-30}"
|
|
REMOTE_ARCHIVE="/tmp/homelab-gitea-dump.zip"
|
|
timestamp="\$(date -u +%Y%m%dT%H%M%SZ)"
|
|
tmp_archive="\$(mktemp "/tmp/gitea-\${timestamp}.XXXXXX.zip")"
|
|
backup_archive="\${GITEA_BACKUP_DIR}/gitea-\${timestamp}.zip"
|
|
remote_host_archive="/tmp/gitea-\${timestamp}.zip"
|
|
|
|
ssh_gitea() {
|
|
ssh -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "\${GITEA_USER}@\${GITEA_HOST}" "\$@"
|
|
}
|
|
|
|
cleanup() {
|
|
rm -f "\${tmp_archive}"
|
|
ssh_gitea "rm -f '\${remote_host_archive}'; sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
ssh_gitea "set -eu
|
|
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true
|
|
sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories'
|
|
sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}'
|
|
sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}'
|
|
sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}'
|
|
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true"
|
|
|
|
scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
|
"\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}"
|
|
|
|
sudo mkdir -p "\${GITEA_BACKUP_DIR}"
|
|
sudo chown jv:jv "\${GITEA_BACKUP_DIR}"
|
|
sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}"
|
|
sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete
|
|
|
|
echo "Created \${backup_archive}"
|
|
BACKUP_SCRIPT_EOT
|
|
sudo chmod 0755 "${backup_script}"
|
|
|
|
sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT'
|
|
[Unit]
|
|
Description=Back up external Homelab Gitea to Debian host storage
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/sbin/homelab-gitea-backup.sh
|
|
SERVICE_EOT
|
|
|
|
sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT'
|
|
[Unit]
|
|
Description=Run daily Homelab Gitea backups
|
|
|
|
[Timer]
|
|
OnCalendar=*-*-* 02:35:00
|
|
RandomizedDelaySec=20m
|
|
Persistent=true
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
TIMER_EOT
|
|
|
|
sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
|
|
GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}"
|
|
GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}"
|
|
|
|
if ! command -v python3 >/dev/null 2>&1; then
|
|
echo "python3 is required for Gitea restore drills." >&2
|
|
exit 1
|
|
fi
|
|
|
|
latest_archive="$(
|
|
{ find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } |
|
|
sort -nr |
|
|
awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }'
|
|
)"
|
|
|
|
if [[ -z "${latest_archive}" ]]; then
|
|
echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}."
|
|
exit 0
|
|
fi
|
|
|
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")"
|
|
tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")"
|
|
report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt"
|
|
|
|
cleanup() {
|
|
rm -rf "${tmp_dir}"
|
|
rm -f "${tmp_report}"
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY'
|
|
import os
|
|
import sys
|
|
import zipfile
|
|
|
|
archive_path, extract_dir, report_path = sys.argv[1:4]
|
|
|
|
with zipfile.ZipFile(archive_path) as archive:
|
|
bad_member = archive.testzip()
|
|
if bad_member:
|
|
raise SystemExit(f"ZIP integrity check failed at {bad_member}")
|
|
|
|
members = archive.infolist()
|
|
if not members:
|
|
raise SystemExit("ZIP archive is empty")
|
|
|
|
extract_root = os.path.abspath(extract_dir)
|
|
for member in members:
|
|
target = os.path.abspath(os.path.join(extract_root, member.filename))
|
|
if target != extract_root and not target.startswith(extract_root + os.sep):
|
|
raise SystemExit(f"Unsafe archive path: {member.filename}")
|
|
|
|
archive.extractall(extract_root)
|
|
|
|
file_count = 0
|
|
total_bytes = 0
|
|
for root, _, files in os.walk(extract_dir):
|
|
for name in files:
|
|
file_count += 1
|
|
total_bytes += os.path.getsize(os.path.join(root, name))
|
|
|
|
if file_count == 0:
|
|
raise SystemExit("Archive extracted no files")
|
|
|
|
with open(report_path, "w", encoding="utf-8") as handle:
|
|
handle.write("Gitea restore drill report\n")
|
|
handle.write(f"archive={archive_path}\n")
|
|
handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n")
|
|
handle.write(f"extracted_files={file_count}\n")
|
|
handle.write(f"extracted_bytes={total_bytes}\n")
|
|
handle.write("result=ok\n")
|
|
PY
|
|
|
|
sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}"
|
|
sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}"
|
|
sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete
|
|
|
|
echo "Created ${report_path}"
|
|
RESTORE_DRILL_SCRIPT_EOT
|
|
sudo chmod 0755 "${restore_drill_script}"
|
|
|
|
sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT'
|
|
[Unit]
|
|
Description=Run a non-destructive Gitea backup restore drill
|
|
After=network-online.target homelab-gitea-backup.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh
|
|
RESTORE_DRILL_SERVICE_EOT
|
|
|
|
sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT'
|
|
[Unit]
|
|
Description=Run monthly Homelab Gitea restore drills
|
|
|
|
[Timer]
|
|
OnCalendar=monthly
|
|
RandomizedDelaySec=2h
|
|
Persistent=true
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
RESTORE_DRILL_TIMER_EOT
|
|
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null
|
|
sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null
|
|
}
|
|
|
|
|
|
backup_gitea() {
|
|
require_debian_server "backup-gitea"
|
|
|
|
install_gitea_backup_timer
|
|
sudo /usr/local/sbin/homelab-gitea-backup.sh
|
|
}
|
|
|
|
drill_gitea_restore() {
|
|
require_debian_server "drill-gitea-restore"
|
|
|
|
install_gitea_backup_timer
|
|
sudo /usr/local/sbin/homelab-gitea-restore-drill.sh
|
|
}
|
|
|
|
install_gitea_runner() {
|
|
local runner_arch
|
|
local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}"
|
|
local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}"
|
|
local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}"
|
|
local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}"
|
|
local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}"
|
|
local runner_user="${GITEA_RUNNER_USER:-jv}"
|
|
local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}"
|
|
local missing_packages=()
|
|
|
|
require_debian_server "install-gitea-runner"
|
|
|
|
case "$(dpkg --print-architecture)" in
|
|
amd64)
|
|
runner_arch="linux-amd64"
|
|
;;
|
|
arm64)
|
|
runner_arch="linux-arm64"
|
|
;;
|
|
*)
|
|
echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
for package in ca-certificates curl git nodejs python3; do
|
|
if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
|
|
missing_packages+=("$package")
|
|
fi
|
|
done
|
|
if [[ ${#missing_packages[@]} -gt 0 ]]; then
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
|
|
fi
|
|
|
|
sudo curl -fsSL \
|
|
-o /usr/local/bin/act_runner \
|
|
"https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}"
|
|
sudo chmod 0755 /usr/local/bin/act_runner
|
|
sudo chown root:root /usr/local/bin/act_runner
|
|
|
|
sudo -u "${runner_user}" mkdir -p "${runner_home}"
|
|
|
|
if [[ ! -f "${runner_home}/.runner" ]]; then
|
|
if [[ -z "${runner_token}" ]]; then
|
|
echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2
|
|
exit 1
|
|
fi
|
|
|
|
sudo -u "${runner_user}" env \
|
|
HOME="/home/${runner_user}" \
|
|
GITEA_RUNNER_HOME="${runner_home}" \
|
|
GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \
|
|
GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \
|
|
GITEA_RUNNER_NAME="${runner_name}" \
|
|
GITEA_RUNNER_LABELS="${runner_labels}" \
|
|
bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"'
|
|
else
|
|
echo "Existing runner registration found at ${runner_home}/.runner; keeping it."
|
|
fi
|
|
|
|
sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null <<SERVICE_EOT
|
|
[Unit]
|
|
Description=Homelab Gitea Actions runner for my-homelab-configs
|
|
After=network-online.target docker.service
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=simple
|
|
User=${runner_user}
|
|
Group=${runner_user}
|
|
WorkingDirectory=${runner_home}
|
|
Environment=HOME=/home/${runner_user}
|
|
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
|
ExecStart=/usr/local/bin/act_runner daemon
|
|
Restart=always
|
|
RestartSec=5
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
SERVICE_EOT
|
|
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable --now homelab-gitea-runner.service >/dev/null
|
|
sudo systemctl status homelab-gitea-runner.service --no-pager -l
|
|
}
|
|
|
|
recreate_pods_for_selector() {
|
|
local namespace="$1"
|
|
local selector="$2"
|
|
local app="$3"
|
|
|
|
if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then
|
|
echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2
|
|
dump_argocd_debug "${app}"
|
|
dump_namespace_debug "${namespace}"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
refresh_argocd_application() {
|
|
local app="$1"
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null
|
|
}
|
|
|
|
website_translation_model() {
|
|
local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}"
|
|
local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}"
|
|
|
|
require_debian_server "website-translation-model"
|
|
|
|
if ! command -v ollama >/dev/null 2>&1; then
|
|
echo "ollama is not installed or not in PATH on this Debian server." >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -r "${modelfile}" ]]; then
|
|
echo "Ollama Modelfile is not readable: ${modelfile}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Creating or updating Ollama model ${model_name} from ${modelfile}..."
|
|
ollama create "${model_name}" -f "${modelfile}"
|
|
|
|
if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo "Ollama model ${model_name} was not found after creation." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Ollama model ${model_name} is ready."
|
|
}
|
|
|
|
website_ollama_listen() {
|
|
local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}"
|
|
local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
|
|
local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}"
|
|
local dropin_dir="/etc/systemd/system/ollama.service.d"
|
|
local dropin_file="${dropin_dir}/homelab.conf"
|
|
local waited=0
|
|
|
|
require_debian_server "website-ollama-listen"
|
|
|
|
bind_address="${bind_address#http://}"
|
|
bind_address="${bind_address#https://}"
|
|
|
|
if ! systemctl cat ollama.service >/dev/null 2>&1; then
|
|
echo "ollama.service was not found on this Debian server." >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Configuring ollama.service to listen on ${bind_address}..."
|
|
sudo mkdir -p "${dropin_dir}"
|
|
sudo mkdir -p "${models_dir}"
|
|
if id ollama >/dev/null 2>&1; then
|
|
sudo chown ollama:ollama "${models_dir}"
|
|
fi
|
|
printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" |
|
|
sudo tee "${dropin_file}" >/dev/null
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl restart ollama.service
|
|
sudo systemctl is-active --quiet ollama.service
|
|
|
|
echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..."
|
|
until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do
|
|
waited=$((waited + 2))
|
|
if ((waited >= 60)); then
|
|
echo "Ollama service did not expose the local API after ${waited}s." >&2
|
|
echo "Recent service logs:" >&2
|
|
sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "Ollama API is reachable on the Debian host."
|
|
}
|
|
|
|
ollama_linux_arch() {
|
|
case "$(uname -m)" in
|
|
x86_64 | amd64)
|
|
printf 'amd64\n'
|
|
;;
|
|
aarch64 | arm64)
|
|
printf 'arm64\n'
|
|
;;
|
|
*)
|
|
echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
ollama_write_systemd_service() {
|
|
sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF'
|
|
[Unit]
|
|
Description=Ollama Service
|
|
After=network-online.target
|
|
|
|
[Service]
|
|
ExecStart=/usr/bin/ollama serve
|
|
User=ollama
|
|
Group=ollama
|
|
Restart=always
|
|
RestartSec=3
|
|
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
|
|
[Install]
|
|
WantedBy=default.target
|
|
EOF
|
|
}
|
|
|
|
ollama_install_from_tgz() {
|
|
local tarball="$1"
|
|
|
|
if [[ ! -r "${tarball}" ]]; then
|
|
echo "Ollama tarball is not readable: ${tarball}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
echo "Installing Ollama from local tarball ${tarball}..."
|
|
sudo tar -C /usr -xzf "${tarball}"
|
|
if ! id ollama >/dev/null 2>&1; then
|
|
sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
|
|
fi
|
|
if getent group render >/dev/null 2>&1; then
|
|
sudo usermod -aG render ollama
|
|
fi
|
|
if getent group video >/dev/null 2>&1; then
|
|
sudo usermod -aG video ollama
|
|
fi
|
|
ollama_write_systemd_service
|
|
}
|
|
|
|
ollama_setup() {
|
|
local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}"
|
|
local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
|
|
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
|
|
local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}"
|
|
local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}"
|
|
local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}"
|
|
local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}"
|
|
local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}"
|
|
local arch
|
|
local tmp_installer
|
|
local curl_ip_args=()
|
|
|
|
require_debian_server "ollama-setup"
|
|
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "curl is required to install Ollama." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -n "${curl_ip_version}" ]]; then
|
|
curl_ip_args+=("${curl_ip_version}")
|
|
fi
|
|
|
|
if ! command -v ollama >/dev/null 2>&1; then
|
|
if [[ -n "${local_tarball}" ]]; then
|
|
ollama_install_from_tgz "${local_tarball}"
|
|
else
|
|
echo "Installing Ollama from ${installer_url}..."
|
|
tmp_installer="$(mktemp)"
|
|
if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then
|
|
rm -f "${tmp_installer}"
|
|
arch="$(ollama_linux_arch || true)"
|
|
cat >&2 <<EOF
|
|
Could not download the Ollama installer from ${installer_url}.
|
|
|
|
If this Debian host cannot reach ollama.com, download the Linux tarball from a
|
|
machine with internet access and copy it to the Debian host:
|
|
|
|
curl -L https://ollama.com/download/ollama-linux-${arch:-amd64}.tgz -o /tmp/ollama-linux-${arch:-amd64}.tgz
|
|
scp /tmp/ollama-linux-${arch:-amd64}.tgz jv@${LAB_DEBIAN_LAN_IP:-192.168.100.73}:/tmp/
|
|
|
|
Then run on Debian:
|
|
|
|
LAB_OLLAMA_TARBALL_FILE=/tmp/ollama-linux-${arch:-amd64}.tgz ./jeannie ollama-setup
|
|
EOF
|
|
exit 1
|
|
fi
|
|
sudo sh "${tmp_installer}"
|
|
rm -f "${tmp_installer}"
|
|
fi
|
|
else
|
|
echo "Ollama is already installed: $(command -v ollama)"
|
|
fi
|
|
|
|
website_ollama_listen
|
|
|
|
echo "Ensuring Ollama model ${model} is available..."
|
|
if ollama list | awk -v model="${model}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
|
|
echo "Ollama model ${model} is already present."
|
|
else
|
|
OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}"
|
|
fi
|
|
|
|
echo "Ollama homelab setup is ready."
|
|
}
|
|
|
|
require_argocd_application_controller() {
|
|
local namespace="${TF_VAR_argocd_namespace:-argocd}"
|
|
local expected_resources="10m|1|2Gi"
|
|
local actual_resources
|
|
|
|
if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then
|
|
echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide >&2 || true
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true
|
|
exit 1
|
|
fi
|
|
|
|
actual_resources="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get statefulset argocd-application-controller \
|
|
-o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')"
|
|
if [[ "${actual_resources}" != "${expected_resources}" ]]; then
|
|
echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2
|
|
echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
require_pimox_app_worker_capacity() {
|
|
local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}"
|
|
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
|
|
local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}"
|
|
local worker_key
|
|
local worker_host
|
|
local worker_user
|
|
local worker_node
|
|
local worker_key_path
|
|
local capacity
|
|
local ready
|
|
local failures=0
|
|
|
|
if disabled_value "${enabled}"; then
|
|
echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}."
|
|
return 0
|
|
fi
|
|
if [[ ! -s "${spec_file}" ]]; then
|
|
echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate."
|
|
return 0
|
|
fi
|
|
if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then
|
|
echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2
|
|
exit 1
|
|
fi
|
|
|
|
while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do
|
|
[[ -n "${worker_node}" ]] || continue
|
|
capacity="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)"
|
|
ready="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
|
if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then
|
|
echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2
|
|
failures=$((failures + 1))
|
|
fi
|
|
done <"${spec_file}"
|
|
|
|
if ((failures > 0)); then
|
|
echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
apps() {
|
|
local buildx_builder_ready=false
|
|
local demos_image_built=false
|
|
local demos_image_ref
|
|
local demos_image_state_file
|
|
local demos_platforms
|
|
local demos_registry_endpoint
|
|
local demos_source_hash
|
|
local registry_endpoint
|
|
local website_image_built=false
|
|
local website_image_ref
|
|
local website_image_state_file
|
|
local website_platforms
|
|
local website_source_hash
|
|
|
|
require_debian_server "apps"
|
|
|
|
registry_endpoint="$(apps_registry_endpoint)"
|
|
demos_registry_endpoint="$(demos_registry_endpoint)"
|
|
demos_image_ref="${registry_endpoint}/demos-static:latest"
|
|
demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state"
|
|
demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
|
|
demos_source_hash="$(demos_source_hash)"
|
|
website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state"
|
|
website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
|
|
website_source_hash="$(website_source_hash)"
|
|
website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}"
|
|
export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}"
|
|
export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}"
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
|
|
export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"
|
|
|
|
require_argocd_application_controller
|
|
require_pimox_app_worker_capacity
|
|
|
|
ensure_cosign_keypair
|
|
publish_cosign_public_key_config_map
|
|
|
|
if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then
|
|
echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then
|
|
echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then
|
|
echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then
|
|
echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2
|
|
exit 1
|
|
fi
|
|
ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}"
|
|
|
|
echo "Deploying homelab applications..."
|
|
|
|
run_tofu_stack "bootstrap/apps"
|
|
|
|
refresh_argocd_application container-registry
|
|
refresh_argocd_application supply-chain-policy
|
|
|
|
wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300
|
|
wait_for_namespace container-registry container-registry 300
|
|
wait_for_namespaced_resource container-registry deployment local-registry container-registry 300
|
|
wait_for_deployment_ready container-registry local-registry container-registry 300
|
|
|
|
if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" &&
|
|
image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then
|
|
echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build."
|
|
else
|
|
echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..."
|
|
ensure_docker_build_space
|
|
if [[ "${buildx_builder_ready}" != "true" ]]; then
|
|
prepare_buildx_builder "${registry_endpoint}"
|
|
buildx_builder_ready=true
|
|
fi
|
|
|
|
docker buildx build \
|
|
--network host \
|
|
--platform "${website_platforms}" \
|
|
--provenance=false \
|
|
--sbom=true \
|
|
--label "dev.homelab.website.source-hash=${website_source_hash}" \
|
|
-t "${website_image_ref}" \
|
|
-f "${REPO_ROOT}/apps/website/Dockerfile" \
|
|
"${REPO_ROOT}/apps/website/" \
|
|
--push
|
|
website_image_built=true
|
|
fi
|
|
publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}"
|
|
|
|
if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" &&
|
|
image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then
|
|
echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build."
|
|
else
|
|
echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..."
|
|
ensure_docker_build_space
|
|
if [[ "${buildx_builder_ready}" != "true" ]]; then
|
|
prepare_buildx_builder "${registry_endpoint}"
|
|
buildx_builder_ready=true
|
|
fi
|
|
|
|
docker buildx build \
|
|
--network host \
|
|
--platform "${demos_platforms}" \
|
|
--provenance=false \
|
|
--sbom=true \
|
|
--label "dev.homelab.demos.source-hash=${demos_source_hash}" \
|
|
-t "${demos_image_ref}" \
|
|
-f "${REPO_ROOT}/apps/demos-static/Dockerfile" \
|
|
"${REPO_ROOT}/apps/demos-static/" \
|
|
--push
|
|
demos_image_built=true
|
|
fi
|
|
publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}"
|
|
|
|
refresh_argocd_application website-production
|
|
wait_for_namespace website-production website-production 300
|
|
wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300
|
|
if [[ "${website_image_built}" == "true" ]]; then
|
|
recreate_pods_for_selector website-production app=php-website website-production
|
|
else
|
|
echo "Skipping website pod restart because the image did not change."
|
|
fi
|
|
wait_for_deployment_ready website-production php-website-deployment website-production 300
|
|
if [[ "${website_image_built}" == "true" ]]; then
|
|
write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}"
|
|
fi
|
|
|
|
refresh_argocd_application demos-static
|
|
wait_for_namespace demos-static demos-static 300
|
|
wait_for_namespaced_resource demos-static deployment demos-static demos-static 300
|
|
if [[ "${demos_image_built}" == "true" ]]; then
|
|
recreate_pods_for_selector demos-static app=demos-static demos-static
|
|
else
|
|
echo "Skipping demos pod restart because the image did not change."
|
|
fi
|
|
wait_for_deployment_ready demos-static demos-static demos-static 300
|
|
if [[ "${demos_image_built}" == "true" ]]; then
|
|
write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}"
|
|
fi
|
|
|
|
refresh_argocd_application heimdall
|
|
wait_for_namespaced_resource monitoring ingress grafana heimdall 300
|
|
wait_for_namespaced_resource monitoring ingress prometheus heimdall 300
|
|
wait_for_namespaced_resource monitoring ingress alertmanager heimdall 300
|
|
wait_for_namespaced_resource argocd ingress argocd-server heimdall 300
|
|
|
|
refresh_argocd_application n8n
|
|
wait_for_namespace n8n n8n 300
|
|
wait_for_namespaced_resource n8n deployment n8n n8n 300
|
|
wait_for_deployment_ready n8n n8n n8n 300
|
|
|
|
echo "Application deployment successfully completed."
|
|
}
|
|
|
|
platform_apply() {
|
|
require_debian_server "platform"
|
|
|
|
run_tofu_stack "bootstrap/platform"
|
|
}
|
|
|
|
edge_apply() {
|
|
require_debian_server "edge"
|
|
|
|
run_tofu_stack "bootstrap/edge"
|
|
}
|
|
|
|
edge_haproxy_stats_password_valid() {
|
|
local password="${1:-}"
|
|
|
|
[[ -n "${password}" ]] && ((${#password} >= 12)) && [[ "${password}" != "adminpassword" ]]
|
|
}
|
|
|
|
ensure_edge_tailscale_routes() {
|
|
local enabled="${LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES:-true}"
|
|
local route="${LAB_EDGE_TAILSCALE_SUBNET_ROUTE:-${LAB_LAN_CIDR:-192.168.100.0/24}}"
|
|
local router="${LAB_EDGE_TAILSCALE_ROUTER:-debian}"
|
|
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
|
|
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
|
|
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}"
|
|
local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}"
|
|
local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
|
|
if disabled_value "${enabled}"; then
|
|
return 0
|
|
fi
|
|
if ! truthy "${enabled}"; then
|
|
echo "LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES must be true or false." >&2
|
|
exit 1
|
|
fi
|
|
if ! validate_ipv4_cidr_or_host "${route}"; then
|
|
echo "Invalid LAB_EDGE_TAILSCALE_SUBNET_ROUTE '${route}'." >&2
|
|
exit 1
|
|
fi
|
|
if [[ -z "${edge_host}" ]]; then
|
|
echo "LAB_EDGE_HOST or TF_VAR_edge_host is required to configure edge Tailscale routes." >&2
|
|
exit 1
|
|
fi
|
|
|
|
case "${router}" in
|
|
debian)
|
|
echo "Ensuring Debian advertises ${route} and OCI edge accepts Tailscale routes..."
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo "tailscale is not installed on the Debian subnet router." >&2
|
|
exit 1
|
|
fi
|
|
sudo mkdir -p /etc/sysctl.d
|
|
printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null
|
|
sudo sysctl -w net.ipv4.ip_forward=1 >/dev/null
|
|
sudo tailscale set --advertise-routes="${route}"
|
|
;;
|
|
rpi | rpi4 | raspberrypi)
|
|
echo "Ensuring RPi advertises ${route} and OCI edge accepts Tailscale routes..."
|
|
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo 'tailscale is not installed on the RPi subnet router.' >&2
|
|
exit 1
|
|
fi
|
|
sysctl_bin=\"\$(command -v sysctl 2>/dev/null || true)\"
|
|
if [ -z \"\$sysctl_bin\" ] && [ -x /usr/sbin/sysctl ]; then
|
|
sysctl_bin=/usr/sbin/sysctl
|
|
fi
|
|
if [ -z \"\$sysctl_bin\" ]; then
|
|
echo 'sysctl is not installed on the RPi subnet router.' >&2
|
|
exit 1
|
|
fi
|
|
sudo mkdir -p /etc/sysctl.d
|
|
printf '%s\n' 'net.ipv4.ip_forward = 1' | sudo tee /etc/sysctl.d/99-homelab-tailscale-subnet-router.conf >/dev/null
|
|
sudo \"\$sysctl_bin\" -w net.ipv4.ip_forward=1 >/dev/null
|
|
sudo tailscale set --advertise-routes='${route}'
|
|
"
|
|
;;
|
|
*)
|
|
echo "LAB_EDGE_TAILSCALE_ROUTER must be debian or rpi." >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu
|
|
if ! command -v tailscale >/dev/null 2>&1; then
|
|
echo 'tailscale is not installed on the OCI edge host.' >&2
|
|
exit 1
|
|
fi
|
|
sudo tailscale set --accept-routes=true
|
|
"
|
|
}
|
|
|
|
edge_haproxy_stats_credentials_file() {
|
|
printf '%s\n' "${LAB_EDGE_HAPROXY_STATS_CREDENTIALS_FILE:-${HOME}/.config/homelab/edge-haproxy.env}"
|
|
}
|
|
|
|
generate_edge_haproxy_stats_password() {
|
|
if command -v openssl >/dev/null 2>&1; then
|
|
openssl rand -hex 24
|
|
elif command -v python3 >/dev/null 2>&1; then
|
|
python3 - <<'PY'
|
|
import secrets
|
|
|
|
print(secrets.token_hex(24))
|
|
PY
|
|
else
|
|
echo "openssl or python3 is required to generate an edge HAProxy stats password." >&2
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
write_edge_haproxy_stats_credentials() {
|
|
local credentials_file="$1"
|
|
local password="$2"
|
|
local credentials_dir
|
|
|
|
credentials_dir="$(dirname "${credentials_file}")"
|
|
mkdir -p "${credentials_dir}"
|
|
chmod 0700 "${credentials_dir}"
|
|
{
|
|
printf 'TF_VAR_haproxy_stats_password='
|
|
printf '%q' "${password}"
|
|
printf '\n'
|
|
} > "${credentials_file}"
|
|
chmod 0600 "${credentials_file}"
|
|
}
|
|
|
|
recover_edge_haproxy_stats_password() {
|
|
local edge_host="${TF_VAR_edge_host:-${LAB_EDGE_HOST:-}}"
|
|
local edge_user="${TF_VAR_edge_user:-${LAB_EDGE_USER:-ubuntu}}"
|
|
local edge_key="${TF_VAR_edge_ssh_key_path:-${LAB_EDGE_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
|
|
local edge_install_dir="${TF_VAR_edge_install_dir:-${LAB_EDGE_INSTALL_DIR:-/opt/homelab-edge}}"
|
|
local remote_auth
|
|
local remote_password
|
|
|
|
if [[ -z "${edge_host}" || "${edge_install_dir}" == *"'"* ]]; then
|
|
return 1
|
|
fi
|
|
|
|
remote_auth="$(ssh -i "${edge_key}" -o BatchMode=yes -o ConnectTimeout=10 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
|
|
"sudo sed -n 's/^[[:space:]]*stats auth //p' '${edge_install_dir}/config_files/haproxy.cfg' | head -n 1" 2>/dev/null || true)"
|
|
remote_auth="${remote_auth//$'\r'/}"
|
|
if [[ "${remote_auth}" != *:* ]]; then
|
|
return 1
|
|
fi
|
|
|
|
remote_password="${remote_auth#*:}"
|
|
if ! edge_haproxy_stats_password_valid "${remote_password}"; then
|
|
return 1
|
|
fi
|
|
|
|
printf '%s\n' "${remote_password}"
|
|
}
|
|
|
|
ensure_edge_haproxy_stats_password() {
|
|
local credentials_file
|
|
local recovered_password
|
|
|
|
if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then
|
|
export TF_VAR_haproxy_stats_password
|
|
return 0
|
|
fi
|
|
|
|
credentials_file="$(edge_haproxy_stats_credentials_file)"
|
|
if [[ -r "${credentials_file}" ]]; then
|
|
# shellcheck disable=SC1090
|
|
source "${credentials_file}"
|
|
if edge_haproxy_stats_password_valid "${TF_VAR_haproxy_stats_password:-}"; then
|
|
export TF_VAR_haproxy_stats_password
|
|
return 0
|
|
fi
|
|
echo "Ignoring invalid edge HAProxy stats credentials in ${credentials_file}." >&2
|
|
fi
|
|
|
|
if recovered_password="$(recover_edge_haproxy_stats_password)"; then
|
|
TF_VAR_haproxy_stats_password="${recovered_password}"
|
|
export TF_VAR_haproxy_stats_password
|
|
write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}"
|
|
echo "Recovered edge HAProxy stats password from the live edge host and stored it at ${credentials_file}."
|
|
return 0
|
|
fi
|
|
|
|
TF_VAR_haproxy_stats_password="$(generate_edge_haproxy_stats_password)"
|
|
export TF_VAR_haproxy_stats_password
|
|
write_edge_haproxy_stats_credentials "${credentials_file}" "${TF_VAR_haproxy_stats_password}"
|
|
echo "Generated edge HAProxy stats password at ${credentials_file}."
|
|
}
|
|
|
|
ensure_cluster_worker_var_file() {
|
|
if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
|
|
prepare_cluster_worker_var_file false
|
|
fi
|
|
if truthy "${TF_VAR_allow_empty_worker_nodes:-false}"; then
|
|
return 0
|
|
fi
|
|
if ! cluster_worker_var_file_has_workers "${LAB_CLUSTER_VAR_FILE}"; then
|
|
echo "Cluster worker var file has no worker_nodes: ${LAB_CLUSTER_VAR_FILE}" >&2
|
|
echo "Run the Pimox worker stage first or set LAB_PIMOX_WORKER_COUNT and LAB_PIMOX_WORKER_STATIC_IPS." >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
run_up_parallel_stage() {
|
|
local spec_file
|
|
local parallelism="${JEANNIE_UP_PARALLELISM:-4}"
|
|
local status
|
|
|
|
if ! [[ "${parallelism}" =~ ^[0-9]+$ ]] || ((parallelism < 1)); then
|
|
echo "JEANNIE_UP_PARALLELISM must be a positive integer." >&2
|
|
return 1
|
|
fi
|
|
|
|
spec_file="$(mktemp)"
|
|
python3 - "${spec_file}" "${REPO_ROOT}/jeannie" "${parallelism}" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
spec_file, jeannie, parallelism = sys.argv[1:4]
|
|
spec = {
|
|
"parallelism": int(parallelism),
|
|
"tasks": [
|
|
{
|
|
"id": "gitea-deploy",
|
|
"title": "Gitea deploy",
|
|
"command": [jeannie, "__up-task", "gitea-deploy"],
|
|
},
|
|
{
|
|
"id": "gitea-bootstrap",
|
|
"title": "Gitea repo bootstrap",
|
|
"needs": ["gitea-deploy"],
|
|
"command": [jeannie, "__up-task", "gitea-bootstrap"],
|
|
},
|
|
{
|
|
"id": "heimdall-deploy",
|
|
"title": "Heimdall deploy",
|
|
"command": [jeannie, "__up-task", "heimdall-deploy"],
|
|
},
|
|
{
|
|
"id": "matrix-deploy",
|
|
"title": "Matrix deploy",
|
|
"command": [jeannie, "__up-task", "matrix-deploy"],
|
|
},
|
|
{
|
|
"id": "omniroute-deploy",
|
|
"title": "OmniRoute deploy",
|
|
"command": [jeannie, "__up-task", "omniroute-deploy"],
|
|
},
|
|
{
|
|
"id": "hermes-deploy",
|
|
"title": "Hermes deploy",
|
|
"needs": ["omniroute-deploy", "matrix-deploy"],
|
|
"command": [jeannie, "__up-task", "hermes-deploy"],
|
|
},
|
|
{
|
|
"id": "rpi-services",
|
|
"title": "RPi services",
|
|
"command": [jeannie, "__up-task", "rpi-services"],
|
|
},
|
|
{
|
|
"id": "pimox-workers",
|
|
"title": "Pimox provisioning and workers",
|
|
"command": [jeannie, "__up-task", "pimox-workers"],
|
|
},
|
|
{
|
|
"id": "openwrt",
|
|
"title": "OpenWrt VM",
|
|
"command": [jeannie, "__up-task", "openwrt"],
|
|
},
|
|
{
|
|
"id": "edge-host",
|
|
"title": "OCI edge host check",
|
|
"command": [jeannie, "__up-task", "edge-host"],
|
|
},
|
|
],
|
|
}
|
|
with open(spec_file, "w", encoding="utf-8") as handle:
|
|
json.dump(spec, handle, indent=2)
|
|
handle.write("\n")
|
|
PY
|
|
set +e
|
|
"${REPO_ROOT}/scripts/jeannie-core" run-dag --spec "${spec_file}"
|
|
status=$?
|
|
set -e
|
|
rm -f "${spec_file}"
|
|
return "${status}"
|
|
}
|
|
|
|
graphiphy_map() {
|
|
if ! command -v graphiphy >/dev/null 2>&1; then
|
|
echo "graphiphy is not installed; skipping repo map generation."
|
|
return 0
|
|
fi
|
|
|
|
graphiphy "${REPO_ROOT}"
|
|
}
|
|
|
|
up_task() {
|
|
require_debian_server "__up-task"
|
|
|
|
case "${1:-}" in
|
|
gitea-deploy)
|
|
deploy_gitea
|
|
;;
|
|
gitea-bootstrap)
|
|
bootstrap_gitea_repo
|
|
;;
|
|
heimdall-deploy)
|
|
deploy_heimdall
|
|
;;
|
|
matrix-deploy)
|
|
deploy_matrix
|
|
;;
|
|
omniroute-deploy)
|
|
deploy_omniroute
|
|
;;
|
|
hermes-deploy)
|
|
deploy_hermes
|
|
;;
|
|
rpi-services)
|
|
deploy_rpi_services
|
|
;;
|
|
pimox-workers)
|
|
run_pimox_pipeline
|
|
;;
|
|
openwrt)
|
|
run_openwrt_pipeline
|
|
;;
|
|
edge-host)
|
|
check_edge_ssh
|
|
;;
|
|
*)
|
|
echo "Unknown __up-task '${1:-}'." >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
up() {
|
|
require_debian_server "up"
|
|
|
|
echo "Deploying the homelab infrastructure..."
|
|
jeannie_log_start "up"
|
|
jeannie_step_plan 12
|
|
|
|
run_step "Early preflight" homelab_preflight early
|
|
run_step "Go toolchain" ensure_go_toolchain
|
|
run_step "Repo Map (Graphiphy)" graphiphy_map
|
|
run_step "Independent host services" run_up_parallel_stage
|
|
run_step "Full preflight" homelab_preflight full
|
|
run_step "Pre-apply doctor" doctor_preapply
|
|
run_step "Worker var file" ensure_cluster_worker_var_file
|
|
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
|
|
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
|
|
run_step "Version report" doctor_versions_report
|
|
run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
|
|
run_step "Applications" apps
|
|
run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"
|
|
|
|
echo "Deployment successfully completed."
|
|
echo "Log: ${JEANNIE_LOG_FILE}"
|
|
}
|
|
|
|
tofu_stack_from_plan_target() {
|
|
case "$1" in
|
|
provisioning)
|
|
printf 'bootstrap/provisioning\n'
|
|
;;
|
|
cluster)
|
|
printf 'bootstrap/cluster\n'
|
|
;;
|
|
platform)
|
|
printf 'bootstrap/platform\n'
|
|
;;
|
|
apps)
|
|
printf 'bootstrap/apps\n'
|
|
;;
|
|
edge)
|
|
printf 'bootstrap/edge\n'
|
|
;;
|
|
*)
|
|
echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
}
|