jv
/
my-homelab-configs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
my-homelab-configs/jeannie

5881 lines
206 KiB
Bash
Executable File
Raw Blame History

#!/usr/bin/env bash
set -euo pipefail
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BUILDX_CONFIG="/tmp/buildx-config.toml"
HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}"
COSIGN_KEY_PREFIX="${COSIGN_KEY_PREFIX:-${HOMELAB_STATE_DIR}/cosign/cosign}"
COSIGN_KEY_PATH="${COSIGN_KEY_PATH:-${COSIGN_KEY_PREFIX}.key}"
COSIGN_PUBLIC_KEY_PATH="${COSIGN_PUBLIC_KEY_PATH:-${COSIGN_KEY_PREFIX}.pub}"
COSIGN_PASSWORD_FILE="${COSIGN_PASSWORD_FILE:-${HOMELAB_STATE_DIR}/cosign/cosign.password}"
COSIGN_VERSION="${COSIGN_VERSION:-2.6.3}"
COSIGN_BIN="${COSIGN_BIN:-}"
HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-homelab-cosign-public-key}"
HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}"
JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}"
JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}"
JEANNIE_STEP_INDEX=0
JEANNIE_STEP_TOTAL=0
trap 'rm -f "${BUILDX_CONFIG}"' EXIT
load_homelab_inventory_defaults() {
local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
local rendered_defaults
if [[ ! -s "${inventory_file}" ]] || ! command -v python3 >/dev/null 2>&1; then
return 0
fi
rendered_defaults="$(python3 - "${inventory_file}" <<'PY'
import re
import shlex
import sys
inventory_file = sys.argv[1]
mapping = {
"domain.base": "LAB_DOMAIN",
"domain.public_url": "LAB_PUBLIC_URL",
"domain.gitea_url": "LAB_GITEA_ROOT_URL",
"network.lan_cidr": "LAB_LAN_CIDR",
"network.lan_ip_prefix": "LAB_LAN_IP_PREFIX",
"network.metallb.traefik_ip": "LAB_TRAEFIK_LB_IP",
"hosts.debian.user": "LAB_DEBIAN_USER",
"hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP",
"hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP",
"hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT",
"hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH",
"hosts.rpi4.user": "LAB_RPI_USER",
"hosts.rpi4.lan_ip": "LAB_RPI_HOST",
"hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP",
"hosts.rpi4.docker_nvme_root": "LAB_RPI_DOCKER_NVME_ROOT",
"hosts.rpi4.docker_fallback_root": "LAB_RPI_DOCKER_FALLBACK_ROOT",
"hosts.opi5_pimox.user": "LAB_PIMOX_USER",
"hosts.opi5_pimox.lan_ip": "LAB_PIMOX_HOST",
"hosts.opi5_pimox.bridge": "LAB_PIMOX_BRIDGE",
"hosts.opi5_pimox.worker_storage": "LAB_PIMOX_WORKER_STORAGE",
"hosts.oci_edge.user": "LAB_EDGE_USER",
"hosts.oci_edge.public_ip": "LAB_EDGE_HOST",
"hosts.oci_edge.install_dir": "LAB_EDGE_INSTALL_DIR",
"services.gitea.http_port": "LAB_GITEA_HTTP_PORT",
"services.gitea.ssh_port": "LAB_GITEA_SSH_PORT",
"services.gitea.root_url": "LAB_GITEA_ROOT_URL",
"services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL",
"services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT",
"services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS",
"services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR",
"services.ollama.igpu_enable": "LAB_OLLAMA_IGPU_ENABLE",
"services.rpi_dns.pihole_web_port": "PIHOLE_WEB_PORT",
"services.rpi_dns.uptime_kuma_port": "UPTIME_KUMA_PORT",
"ai_gateway.provider": "LAB_AI_GATEWAY_PROVIDER",
"ai_gateway.enabled": "LAB_BACKSTAGE_BRAIN_ENABLED",
"ai_gateway.url": "LAB_AI_GATEWAY_URL",
"ai_gateway.model": "LAB_AI_GATEWAY_MODEL",
"ai_gateway.timeout_seconds": "LAB_AI_GATEWAY_TIMEOUT_SECONDS",
"ai_gateway.knowledge_index_dir": "LAB_AI_KNOWLEDGE_INDEX_DIR",
"pimox.worker_base_vmid": "LAB_PIMOX_WORKER_BASE_VMID",
"pimox.default_worker_count": "LAB_PIMOX_WORKER_COUNT",
}
def flatten(document, prefix=""):
values = {}
if isinstance(document, dict):
for key, value in document.items():
path = f"{prefix}.{key}" if prefix else str(key)
if isinstance(value, dict):
values.update(flatten(value, path))
elif isinstance(value, list):
values[path] = ",".join(str(item) for item in value)
elif value is not None:
values[path] = str(value)
return values
def parse_simple_inventory(path):
stack = []
values = {}
pattern = re.compile(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*):(?:\s*(.*?))?\s*$")
with open(path, encoding="utf-8") as handle:
for raw_line in handle:
if not raw_line.strip() or raw_line.lstrip().startswith("#") or raw_line.lstrip().startswith("- "):
continue
match = pattern.match(raw_line.rstrip("\n"))
if not match:
continue
indent = len(match.group(1))
key = match.group(2)
value = (match.group(3) or "").strip()
while stack and stack[-1][0] >= indent:
stack.pop()
current_path = ".".join([item[1] for item in stack] + [key])
if value == "":
stack.append((indent, key))
continue
if " #" in value:
value = value.split(" #", 1)[0].strip()
values[current_path] = value.strip("\"'")
return values
try:
import yaml
except ImportError:
values = parse_simple_inventory(inventory_file)
else:
with open(inventory_file, encoding="utf-8") as handle:
values = flatten(yaml.safe_load(handle) or {})
for path, env_name in mapping.items():
value = values.get(path)
if value:
print(f": ${{{env_name}:={shlex.quote(value)}}}")
debian_ip = values.get("hosts.debian.lan_ip")
gitea_http = values.get("services.gitea.http_port")
if debian_ip:
print(f": ${{LAB_GITEA_HOST:={shlex.quote(debian_ip)}}}")
if debian_ip and gitea_http:
print(f": ${{LAB_GITEA_LOCAL_URL:={shlex.quote(f'http://{debian_ip}:{gitea_http}/')}}}")
gitea_ts = values.get("hosts.debian.tailscale_ip")
if gitea_ts:
print(f": ${{LAB_GITEA_TAILSCALE_IP:={shlex.quote(gitea_ts)}}}")
PY
)"
if [[ -n "${rendered_defaults}" ]]; then
eval "${rendered_defaults}"
fi
}
load_homelab_inventory_defaults
export_if_unset() {
local name="$1"
local value="$2"
if [[ -z "${value}" ]]; then
return 0
fi
if [[ -z "${!name:-}" ]]; then
printf -v "${name}" '%s' "${value}"
fi
# shellcheck disable=SC2163
export "${name?}"
}
export_homelab_inventory_tf_vars() {
export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}"
export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}"
export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}"
export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}"
export_if_unset TF_VAR_provisioning_user "${LAB_DEBIAN_USER:-}"
export_if_unset TF_VAR_http_host "${LAB_DEBIAN_LAN_IP:-}"
export_if_unset TF_VAR_pimox_host "${LAB_PIMOX_HOST:-}"
export_if_unset TF_VAR_pimox_user "${LAB_PIMOX_USER:-}"
export_if_unset TF_VAR_pimox_worker_storage "${LAB_PIMOX_WORKER_STORAGE:-}"
export_if_unset TF_VAR_pimox_template_bridge "${LAB_PIMOX_BRIDGE:-}"
export_if_unset TF_VAR_pimox_template_build_host "${LAB_DEBIAN_LAN_IP:-}"
export_if_unset TF_VAR_pimox_template_build_user "${LAB_DEBIAN_USER:-}"
export_if_unset TF_VAR_pimox_template_guest_ip_prefix "${LAB_LAN_IP_PREFIX:-}"
export_if_unset TF_VAR_edge_host "${LAB_EDGE_HOST:-}"
export_if_unset TF_VAR_edge_user "${LAB_EDGE_USER:-}"
export_if_unset TF_VAR_edge_install_dir "${LAB_EDGE_INSTALL_DIR:-}"
export_if_unset TF_VAR_server_name "${LAB_DOMAIN:-}"
export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}"
export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}"
export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}"
export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}"
}
export_homelab_inventory_tf_vars
KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}"
require_debian_server() {
local command_name="$1"
local os_id=""
if [[ "$(uname -s)" != "Linux" ]]; then
echo "Refusing to run '${command_name}' from this machine. Run it on the Debian homelab server." >&2
exit 1
fi
if [[ -r /etc/os-release ]]; then
os_id="$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2; exit}' /etc/os-release)"
fi
if [[ "${os_id}" != "debian" ]]; then
echo "Refusing to run '${command_name}' on ${os_id:-unknown OS}. Run it on the Debian homelab server." >&2
exit 1
fi
}
tofu_state_has_resource() {
local stack="$1"
local resource_address="$2"
tofu -chdir="${REPO_ROOT}/${stack}" state show "${resource_address}" >/dev/null 2>&1
}
helm_release_secret_exists() {
local namespace="$1"
local release_name="$2"
local secret_name
secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \
-l "owner=helm,name=${release_name}" \
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
[[ -n "${secret_name}" ]]
}
kubernetes_resource_exists() {
local namespace="$1"
local resource_kind="$2"
local resource_name="$3"
if [[ -n "${namespace}" ]]; then
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
return $?
fi
kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
}
adopt_tofu_helm_release() {
local stack="$1"
local resource_address="$2"
local namespace="$3"
local release_name="$4"
if tofu_state_has_resource "${stack}" "${resource_address}"; then
return 0
fi
if ! helm_release_secret_exists "${namespace}" "${release_name}"; then
return 0
fi
echo "Importing existing Helm release ${namespace}/${release_name} into ${stack} state (${resource_address})..."
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${namespace}/${release_name}"
}
adopt_tofu_kubernetes_resource() {
local stack="$1"
local resource_address="$2"
local namespace="$3"
local resource_kind="$4"
local resource_name="$5"
local import_id="$6"
if tofu_state_has_resource "${stack}" "${resource_address}"; then
return 0
fi
if ! kubernetes_resource_exists "${namespace}" "${resource_kind}" "${resource_name}"; then
return 0
fi
echo "Importing existing Kubernetes ${resource_kind} ${resource_name} into ${stack} state (${resource_address})..."
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
}
adopt_tofu_kubernetes_manifest() {
local stack="$1"
local resource_address="$2"
local namespace="$3"
local kubectl_kind="$4"
local api_version="$5"
local manifest_kind="$6"
local resource_name="$7"
local import_id
if tofu_state_has_resource "${stack}" "${resource_address}"; then
return 0
fi
if ! kubernetes_resource_exists "${namespace}" "${kubectl_kind}" "${resource_name}"; then
return 0
fi
import_id="apiVersion=${api_version},kind=${manifest_kind},namespace=${namespace},name=${resource_name}"
echo "Importing existing Kubernetes ${manifest_kind} ${namespace}/${resource_name} into ${stack} state (${resource_address})..."
tofu -chdir="${REPO_ROOT}/${stack}" import -input=false "${resource_address}" "${import_id}"
}
adopt_platform_existing_resources() {
local stack="bootstrap/platform"
adopt_tofu_helm_release "${stack}" "helm_release.calico_crds" "tigera-operator" "calico-crds"
adopt_tofu_helm_release "${stack}" "helm_release.calico" "tigera-operator" "calico"
adopt_tofu_helm_release "${stack}" "helm_release.openebs" "openebs" "openebs"
adopt_tofu_helm_release "${stack}" "helm_release.metallb[\"enabled\"]" "metallb-system" "metallb"
adopt_tofu_helm_release "${stack}" "helm_release.traefik[\"enabled\"]" "traefik" "traefik"
adopt_tofu_helm_release "${stack}" "helm_release.argocd" "argocd" "argocd"
adopt_tofu_helm_release "${stack}" "helm_release.kyverno" "kyverno" "kyverno"
adopt_tofu_helm_release "${stack}" "helm_release.kyverno_policies" "kyverno" "kyverno-policies"
adopt_tofu_helm_release "${stack}" "helm_release.loki" "monitoring" "loki"
adopt_tofu_helm_release "${stack}" "helm_release.promtail" "monitoring" "promtail"
adopt_tofu_helm_release "${stack}" "helm_release.prometheus_stack" "monitoring" "prometheus-stack"
adopt_tofu_kubernetes_resource \
"${stack}" \
"kubernetes_storage_class_v1.openebs_hostpath_retain" \
"" \
"storageclass" \
"openebs-hostpath-retain" \
"openebs-hostpath-retain"
adopt_tofu_kubernetes_resource \
"${stack}" \
"kubernetes_namespace_v1.monitoring" \
"" \
"namespace" \
"monitoring" \
"monitoring"
}
adopt_apps_existing_resources() {
local stack="bootstrap/apps"
local namespace="${TF_VAR_argocd_namespace:-argocd}"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["container-registry"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"container-registry"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["website-production"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"website-production"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["demos-static"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"demos-static"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["heimdall"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"heimdall"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["n8n"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"n8n"
adopt_tofu_kubernetes_manifest \
"${stack}" \
'kubernetes_manifest.argocd_application["supply-chain-policy"]' \
"${namespace}" \
"applications.argoproj.io" \
"argoproj.io/v1alpha1" \
"Application" \
"supply-chain-policy"
}
ensure_homelab_node_labels() {
local control_plane_node="${LAB_CONTROL_PLANE_NODE_NAME:-debian}"
local raspberry_node="${LAB_RASPBERRY_NODE_NAME:-raspberry}"
local prometheus_selector="homelab.dev/node-role=app,homelab.dev/storage=ssd"
local node
local target_nodes
echo "Applying homelab labels to existing Kubernetes nodes..."
while IFS= read -r node; do
[[ -n "${node}" ]] || continue
if [[ "${node}" == "${control_plane_node}" ]]; then
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
homelab.dev/node-role=control-plane \
homelab.dev/storage=local \
homelab.dev/workload-class=control-plane \
--overwrite
continue
fi
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
node-role.kubernetes.io/worker=worker \
--overwrite
if [[ "${node}" == pimox-worker-* ]]; then
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
homelab.dev/node-role=app \
homelab.dev/storage=ssd \
homelab.dev/workload-class=platform \
--overwrite
elif [[ "${node}" == "${raspberry_node}" ]]; then
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
homelab.dev/node-role=edge-app \
homelab.dev/storage=local \
homelab.dev/workload-class=edge \
--overwrite
fi
done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)"
if [[ -z "${target_nodes}" ]]; then
echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2
exit 1
fi
}
delete_prometheus_stack_storage() {
local namespace="${1:-monitoring}"
local pattern='(^|-)prometheus-stack-(prometheus|alertmanager|grafana)(-|$)|^prometheus-prometheus-stack|^alertmanager-prometheus-stack|^storage-prometheus-stack-grafana'
local pvc_names
local pv_names
pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null |
awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')"
pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \
-o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null |
awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')"
if [[ -n "${pvc_names}" ]]; then
echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..."
printf '%s\n' "${pvc_names}" |
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s
fi
if [[ -n "${pv_names}" ]]; then
echo "Deleting old prometheus-stack retained PV objects..."
printf '%s\n' "${pv_names}" |
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false
fi
}
run_tofu_stack() {
local stack="$1"
local auto_approve="${LAB_AUTO_APPROVE:-true}"
local -a apply_args=()
if truthy "${auto_approve}"; then
apply_args+=("-auto-approve")
elif ! disabled_value "${auto_approve}"; then
echo "LAB_AUTO_APPROVE must be true or false." >&2
exit 1
fi
if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
apply_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
fi
tofu -chdir="${REPO_ROOT}/${stack}" init
if [[ "${stack}" == "bootstrap/platform" ]]; then
adopt_platform_existing_resources
fi
if [[ "${stack}" == "bootstrap/apps" ]]; then
adopt_apps_existing_resources
fi
tofu -chdir="${REPO_ROOT}/${stack}" apply "${apply_args[@]}"
}
run_tofu_plan_stack() {
local stack="$1"
local -a plan_args=()
if [[ "${stack}" == "bootstrap/cluster" && -n "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
plan_args+=("-var-file=${LAB_CLUSTER_VAR_FILE}")
fi
tofu -chdir="${REPO_ROOT}/${stack}" init
tofu -chdir="${REPO_ROOT}/${stack}" plan "${plan_args[@]}"
}
backup_tofu_state() {
local backup_dir="${HOMELAB_TOFU_STATE_BACKUP_DIR:-${HOMELAB_STATE_DIR}/tofu-state-backups}"
local timestamp
local archive
local -a paths=()
local path
timestamp="$(date +%Y%m%d-%H%M%S)"
archive="${backup_dir}/tofu-state-${timestamp}.tgz"
mkdir -p "${backup_dir}"
for path in \
bootstrap/provisioning/terraform.tfstate \
bootstrap/provisioning/terraform.tfstate.backup \
bootstrap/cluster/terraform.tfstate \
bootstrap/cluster/terraform.tfstate.backup \
bootstrap/platform/terraform.tfstate \
bootstrap/platform/terraform.tfstate.backup \
bootstrap/apps/terraform.tfstate \
bootstrap/apps/terraform.tfstate.backup \
bootstrap/edge/terraform.tfstate \
bootstrap/edge/terraform.tfstate.backup \
.lab/cluster-workers.auto.tfvars.json \
.lab/pimox-workers.tsv \
.lab/manual-workers.tsv; do
if [[ -e "${REPO_ROOT}/${path}" ]]; then
paths+=("${path}")
fi
done
if ((${#paths[@]} == 0)); then
echo "No local OpenTofu state files found to back up."
return 0
fi
tar -C "${REPO_ROOT}" -czf "${archive}" "${paths[@]}"
chmod 0600 "${archive}"
echo "Backed up local OpenTofu state to ${archive}."
}
move_prometheus_stack_workers() {
local stack="bootstrap/platform"
local namespace="${LAB_MONITORING_NAMESPACE:-monitoring}"
local auto_approve="${LAB_AUTO_APPROVE:-true}"
local -a approve_args=()
require_debian_server "move-prometheus-stack-workers"
if [[ "${LAB_CONFIRM_DELETE_PROMETHEUS_DATA:-}" != "true" ]]; then
cat >&2 <<'EOF'
move-prometheus-stack-workers deletes existing prometheus-stack PVC/PV data.
Rerun with LAB_CONFIRM_DELETE_PROMETHEUS_DATA=true when that data loss is intended.
EOF
exit 1
fi
if truthy "${auto_approve}"; then
approve_args+=("-auto-approve")
elif ! disabled_value "${auto_approve}"; then
echo "LAB_AUTO_APPROVE must be true or false." >&2
exit 1
fi
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
export KUBECONFIG="${TF_VAR_kubeconfig_path}"
echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted."
ensure_homelab_node_labels
tofu -chdir="${REPO_ROOT}/${stack}" init
adopt_platform_existing_resources
tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}"
delete_prometheus_stack_storage "${namespace}"
tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}"
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
}
doctor_versions_report() {
local api_version
local api_minor
local containerd_major_count
local kubelet_minor_count
local node_rows
local status=0
require_debian_server "doctor-versions"
ensure_python3
api_version="$(
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
)"
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
node_rows="$(
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
)"
if [[ -z "${node_rows}" ]]; then
echo "No Kubernetes nodes found." >&2
return 1
fi
printf 'API server: %s\n\n' "${api_version}"
printf '%-22s %-12s %-18s %s\n' "NODE" "KUBELET" "KUBELET_MINOR" "RUNTIME"
printf '%s\n' "${node_rows}" |
awk -F '\t' '{
version = $2
gsub(/^v/, "", version)
split(version, parts, ".")
printf "%-22s %-12s %-18s %s\n", $1, $2, parts[1] "." parts[2], $3
}'
kubelet_minor_count="$(
printf '%s\n' "${node_rows}" |
awk -F '\t' '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); print parts[1] "." parts[2] }' |
sort -u |
wc -l |
tr -d ' '
)"
containerd_major_count="$(
printf '%s\n' "${node_rows}" |
awk -F '\t' '$3 ~ /^containerd:\/\// { runtime = $3; sub(/^containerd:\/\//, "", runtime); split(runtime, parts, "."); print parts[1] }' |
sort -u |
wc -l |
tr -d ' '
)"
if [[ "${kubelet_minor_count}" != "1" ]]; then
echo
echo "Kubernetes kubelet minors are mixed. Rebuild or upgrade workers until every node matches the API server minor (${api_minor})." >&2
status=1
fi
if printf '%s\n' "${node_rows}" |
awk -F '\t' -v expected="${api_minor}" '{ version = $2; gsub(/^v/, "", version); split(version, parts, "."); if (parts[1] "." parts[2] != expected) found = 1 } END { exit found ? 0 : 1 }'; then
echo
echo "At least one kubelet does not match the API server minor (${api_minor})." >&2
status=1
fi
if [[ "${containerd_major_count}" -gt 1 ]]; then
echo
echo "Containerd major versions are mixed. Standardize the node image/runtime path before relying on destructive rebuilds." >&2
status=1
fi
if [[ "${status}" -eq 0 ]]; then
echo
echo "Node Kubernetes and containerd versions are aligned."
fi
return "${status}"
}
doctor_versions() {
doctor_versions_report
exit "$?"
}
truthy() {
case "${1,,}" in
1 | true | yes | on)
return 0
;;
*)
return 1
;;
esac
}
disabled_value() {
case "${1,,}" in
0 | false | no | off | disabled)
return 0
;;
*)
return 1
;;
esac
}
jeannie_log_start() {
local command_name="$1"
local timestamp
if [[ -n "${JEANNIE_LOG_FILE}" ]]; then
return 0
fi
timestamp="$(date +%Y%m%d-%H%M%S)"
mkdir -p "${JEANNIE_LOG_DIR}"
JEANNIE_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.log"
: >"${JEANNIE_LOG_FILE}"
printf 'Log: %s\n' "${JEANNIE_LOG_FILE}"
}
jeannie_step_plan() {
JEANNIE_STEP_INDEX=0
JEANNIE_STEP_TOTAL="$1"
}
jeannie_verbose_enabled() {
truthy "${JEANNIE_VERBOSE:-false}"
}
redact_sensitive_output() {
sed -E \
-e 's/(GITEA_BOOTSTRAP_PASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(PIHOLE_WEBPASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(COSIGN_PASSWORD=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(GITEA_RUNNER_REGISTRATION_TOKEN=)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(--password[=[:space:]]+)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(--token[=[:space:]]+)["'\'']?[^"'\''[:space:]]+["'\'']?/\1[REDACTED]/g' \
-e 's/(password[=:][[:space:]]*)[^[:space:]]+/\1[REDACTED]/Ig' \
-e 's/(token[=:][[:space:]]*)[^[:space:]]+/\1[REDACTED]/Ig' \
-e 's#(https?://[^:/[:space:]]+):[^@/[:space:]]+@#\1:[REDACTED]@#g'
}
run_step() {
local description="$1"
local step_log
local status
shift
JEANNIE_STEP_INDEX=$((JEANNIE_STEP_INDEX + 1))
if ((JEANNIE_STEP_TOTAL > 0)); then
printf '[%d/%d] %-42s ' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}"
else
printf '[%d] %-46s ' "${JEANNIE_STEP_INDEX}" "${description}"
fi
step_log="$(mktemp)"
{
printf '\n== [%d/%d] %s ==\n' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}"
printf 'Started: %s\n' "$(date -Is)"
} >>"${JEANNIE_LOG_FILE}"
if jeannie_verbose_enabled; then
set +e
"$@" 2>&1 | redact_sensitive_output | tee "${step_log}"
status=${PIPESTATUS[0]}
set -e
else
set +e
"$@" 2>&1 | redact_sensitive_output >"${step_log}"
status=${PIPESTATUS[0]}
set -e
fi
cat "${step_log}" >>"${JEANNIE_LOG_FILE}"
printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
if ((status == 0)); then
printf 'ok\n'
rm -f "${step_log}"
return 0
fi
printf 'failed\n\n'
printf 'Step failed: %s\n' "${description}" >&2
printf 'Exit status: %s\n' "${status}" >&2
printf 'Full step output:\n' >&2
sed 's/^/ /' "${step_log}" >&2
printf '\nFull log: %s\n' "${JEANNIE_LOG_FILE}" >&2
rm -f "${step_log}"
return "${status}"
}
worker_index_is_skipped() {
local index="$1"
local skip_indexes="$2"
local skip_index
skip_indexes="${skip_indexes//,/ }"
for skip_index in ${skip_indexes}; do
[[ -z "${skip_index}" ]] && continue
if ! [[ "${skip_index}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_SKIP_WORKER_INDEXES must contain only comma or space separated positive integers." >&2
exit 1
fi
if ((skip_index == index)); then
return 0
fi
done
return 1
}
ensure_python3() {
if command -v python3 >/dev/null 2>&1; then
return 0
fi
sudo apt-get update
sudo apt-get install -y --no-install-recommends python3
}
detect_route_interface() {
local target="$1"
ip route get "${target}" 2>/dev/null | awk '
{
for (i = 1; i <= NF; i++) {
if ($i == "dev") {
print $(i + 1)
exit
}
}
}
'
}
pimox_ssh() {
local host="$1"
local user="$2"
local key_path="$3"
shift 3
ssh -i "${key_path}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${user}@${host}" "$@"
}
pimox_guest_ipv4() {
local guest_json
local host="$1"
local user="$2"
local key_path="$3"
local vmid="$4"
local ip_prefix="$5"
local qm_bin="${6:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
guest_json="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" 2>/dev/null || true)"
if [[ -z "${guest_json}" ]]; then
return 1
fi
GUEST_JSON="${guest_json}" python3 - "${ip_prefix}" <<'PY'
import json
import os
import sys
prefix = sys.argv[1]
try:
interfaces = json.loads(os.environ.get("GUEST_JSON", ""))
except Exception:
sys.exit(1)
for iface in interfaces or []:
for address in iface.get("ip-addresses") or []:
if address.get("ip-address-type") != "ipv4":
continue
ip = address.get("ip-address", "")
if not ip or ip.startswith(("127.", "169.254.")):
continue
if prefix and not ip.startswith(prefix):
continue
print(ip)
sys.exit(0)
sys.exit(1)
PY
}
pimox_worker_vm_debug() {
local host="$1"
local user="$2"
local key_path="$3"
local vmid="$4"
local qm_bin="$5"
pimox_ssh "${host}" "${user}" "${key_path}" "set +e
echo 'Pimox VM ${vmid} status:'
sudo '${qm_bin}' status '${vmid}'
echo 'Pimox VM ${vmid} config summary:'
sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" >&2 || true
}
wait_for_pimox_guest_ssh() {
local host="$1"
local user="$2"
local key_path="$3"
local vmid="$4"
local guest_user="$5"
local guest_key_path="$6"
local ip_prefix="$7"
local timeout_seconds="$8"
local qm_bin="${9:-${LAB_PIMOX_QM_BIN:-/usr/sbin/qm}}"
local deadline
local elapsed
local guest_ip
local ip_filter_description
local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
local last_guest_ip=""
local last_known_hosts_ip=""
local last_ssh_output=""
local next_log
local ssh_deadline=0
local ssh_output
local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}"
ip_filter_description="matching prefix ${ip_prefix}"
if [[ -z "${ip_prefix}" ]]; then
ip_filter_description="that is not loopback or link-local"
fi
if ! [[ "${ssh_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_timeout_seconds == 0)); then
echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
return 1
fi
mkdir -p "$(dirname "${known_hosts_file}")"
touch "${known_hosts_file}"
chmod 0600 "${known_hosts_file}"
deadline=$((SECONDS + timeout_seconds))
next_log="${SECONDS}"
while ((SECONDS < deadline)); do
guest_ip="$(pimox_guest_ipv4 "${host}" "${user}" "${key_path}" "${vmid}" "${ip_prefix}" "${qm_bin}" || true)"
if [[ -n "${guest_ip}" ]]; then
if ((ssh_deadline == 0)); then
ssh_deadline=$((SECONDS + ssh_timeout_seconds))
elif ((SECONDS >= ssh_deadline)); then
break
fi
last_guest_ip="${guest_ip}"
if [[ "${last_known_hosts_ip}" != "${guest_ip}" ]]; then
ssh-keygen -R "${guest_ip}" -f "${known_hosts_file}" >/dev/null 2>&1 || true
last_known_hosts_ip="${guest_ip}"
fi
if ssh_output="$(ssh -i "${guest_key_path}" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${known_hosts_file}" "${guest_user}@${guest_ip}" true 2>&1)"; then
printf '%s\n' "${guest_ip}"
return 0
fi
last_ssh_output="${ssh_output}"
fi
if ((SECONDS >= next_log)); then
elapsed=$((timeout_seconds - (deadline - SECONDS)))
if [[ -n "${last_guest_ip}" ]]; then
echo "Waiting for SSH to worker VM ${vmid} at ${last_guest_ip} as ${guest_user} (${elapsed}s elapsed)..." >&2
if [[ -n "${last_ssh_output}" ]]; then
echo "Last SSH failure: ${last_ssh_output}" >&2
fi
else
echo "Waiting for worker VM ${vmid} to report an IPv4 address ${ip_filter_description} through qemu-guest-agent (${elapsed}s elapsed)..." >&2
fi
next_log=$((SECONDS + 60))
fi
sleep 10
done
if [[ -n "${last_guest_ip}" ]]; then
echo "Worker VM ${vmid} reported guest IP ${last_guest_ip}, but SSH as ${guest_user} never became reachable." >&2
if [[ -n "${last_ssh_output}" ]]; then
echo "Last SSH failure: ${last_ssh_output}" >&2
fi
else
echo "Worker VM ${vmid} did not report an IPv4 address ${ip_filter_description} through qemu-guest-agent." >&2
fi
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
return 1
}
pimox_generated_mac() {
local vmid="$1"
printf '02:68:10:%02x:%02x:%02x\n' \
$(((vmid >> 16) & 255)) \
$(((vmid >> 8) & 255)) \
$((vmid & 255))
}
cpuset_cpu_count() {
local cpuset="$1"
local count=0
local part
local start
local end
local -a parts
IFS=',' read -r -a parts <<<"${cpuset}"
for part in "${parts[@]}"; do
if [[ "${part}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
start="${BASH_REMATCH[1]}"
end="${BASH_REMATCH[2]}"
if ((end < start)); then
return 1
fi
count=$((count + end - start + 1))
elif [[ "${part}" =~ ^[0-9]+$ ]]; then
count=$((count + 1))
else
return 1
fi
done
printf '%s\n' "${count}"
}
pimox_worker_cpu_affinity() {
local index="$1"
local affinities="$2"
local worker_cores="$3"
local affinity
local affinity_index=1
local cpu_count
for affinity in ${affinities}; do
if ((affinity_index == index)); then
if ! cpu_count="$(cpuset_cpu_count "${affinity}")"; then
echo "Invalid Pimox worker CPU affinity '${affinity}'. Use CPU IDs or ranges, such as 4-5." >&2
exit 1
fi
if ((cpu_count != worker_cores)); then
echo "Pimox worker index ${index} uses ${worker_cores} cores but affinity '${affinity}' contains ${cpu_count} CPUs." >&2
exit 1
fi
printf '%s\n' "${affinity}"
return 0
fi
affinity_index=$((affinity_index + 1))
done
echo "No LAB_PIMOX_WORKER_CPU_AFFINITIES entry exists for Pimox worker index ${index}." >&2
exit 1
}
ensure_pimox_worker_node() {
local index="$1"
local spec_file="$2"
local pimox_host="$3"
local pimox_user="$4"
local pimox_key="$5"
local template_vmid="$6"
local bridge="$7"
local worker_base_vmid="$8"
local worker_name_prefix="$9"
local worker_node_prefix="${10}"
local worker_key_prefix="${11}"
local worker_cores="${12}"
local worker_memory="${13}"
local worker_user="${14}"
local worker_key_path="${15}"
local ip_prefix="${16}"
local timeout_seconds="${17}"
local qm_bin="${18}"
local worker_storage="${19}"
local worker_replace_existing="${20}"
local worker_cpu_affinity="${21}"
local padded
local vmid
local worker_key
local worker_name
local node_name
local mac
local guest_ip
printf -v padded '%02d' "${index}"
vmid=$((worker_base_vmid + index - 1))
worker_key="${worker_key_prefix}${padded}"
worker_name="${worker_name_prefix}-${padded}"
node_name="${worker_node_prefix}-${padded}"
mac="$(pimox_generated_mac "${vmid}")"
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2
exit 1
fi
if ! truthy "${worker_replace_existing}" &&
! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | awk -F': ' -v storage='${worker_storage}' '/^(scsi|virtio|sata|ide)[0-9]+:/ { disks = 1; if (\$2 !~ \"^\" storage \":\") bad = 1 } END { exit (disks && !bad) ? 0 : 1 }'"; then
echo "Existing Pimox worker VM ${vmid} (${worker_name}) is not fully on storage ${worker_storage}; replacing it from template ${template_vmid}."
worker_replace_existing=true
fi
if truthy "${worker_replace_existing}"; then
echo "Replacing existing Pimox worker VM ${vmid} (${worker_name}) before cloning from template ${template_vmid}..."
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true
elapsed=0
while [ \"\$elapsed\" -lt 300 ]; do
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
break
fi
sleep 5
elapsed=\$((elapsed + 5))
done
sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
else
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}'
if [ -n '${worker_cpu_affinity}' ]; then
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
case \"\$affinity_output\" in
*'Unknown option: affinity'*)
echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
;;
*)
printf '%s\n' \"\$affinity_output\" >&2
exit 1
;;
esac
}
fi
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi"
fi
fi
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! ip link show '${bridge}' >/dev/null 2>&1; then
echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2
exit 1
fi
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2
exit 1
fi
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2
exit 1
fi
sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}'
sudo '${qm_bin}' set '${vmid}' --agent enabled=1
sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}'
if [ -n '${worker_cpu_affinity}' ]; then
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
case \"\$affinity_output\" in
*'Unknown option: affinity'*)
echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.'
;;
*)
printf '%s\n' \"\$affinity_output\" >&2
exit 1
;;
esac
}
fi
sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}'
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
sudo '${qm_bin}' set '${vmid}' --onboot 1
sudo '${qm_bin}' start '${vmid}'"
fi
if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
echo "Timed out waiting for worker VM ${vmid} (${worker_name}) to report a reachable guest IP." >&2
exit 1
fi
printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
}
write_cluster_worker_var_file() {
local var_file="$1"
local default_platform_labels_json='{"node-role.kubernetes.io/worker":"worker","homelab.dev/node-role":"app","homelab.dev/storage":"ssd","homelab.dev/workload-class":"platform"}'
local pimox_labels_json="${LAB_PIMOX_WORKER_NODE_LABELS_JSON:-${default_platform_labels_json}}"
local manual_labels_json="${LAB_MANUAL_WORKER_NODE_LABELS_JSON:-${pimox_labels_json}}"
shift
LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-false}" \
LAB_RASPBERRY_HOST="${LAB_RASPBERRY_HOST:-192.168.100.89}" \
LAB_RASPBERRY_USER="${LAB_RASPBERRY_USER:-jv}" \
LAB_RASPBERRY_NODE_NAME="${LAB_RASPBERRY_NODE_NAME:-raspberry}" \
LAB_RASPBERRY_SSH_KEY_PATH="${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" \
LAB_RASPBERRY_NODE_LABELS_JSON="${LAB_RASPBERRY_NODE_LABELS_JSON:-{\"node-role.kubernetes.io/worker\":\"worker\",\"homelab.dev/node-role\":\"edge-app\",\"homelab.dev/storage\":\"local\",\"homelab.dev/workload-class\":\"edge\"}}" \
LAB_MANUAL_WORKER_NODE_LABELS_JSON="${manual_labels_json}" \
LAB_PIMOX_WORKER_NODE_LABELS_JSON="${pimox_labels_json}" \
python3 - "${var_file}" "$@" <<'PY'
import json
import os
from pathlib import Path
import sys
var_file = sys.argv[1]
spec_files = sys.argv[2:]
nodes = {}
node_labels = {}
try:
raspberry_labels = json.loads(os.environ["LAB_RASPBERRY_NODE_LABELS_JSON"])
manual_labels = json.loads(os.environ["LAB_MANUAL_WORKER_NODE_LABELS_JSON"])
pimox_labels = json.loads(os.environ["LAB_PIMOX_WORKER_NODE_LABELS_JSON"])
except json.JSONDecodeError as exc:
raise SystemExit(f"Invalid node label JSON: {exc}") from exc
if os.environ["LAB_INCLUDE_RASPBERRY_WORKER"].lower() not in {"0", "false", "no", "off", "disabled"}:
nodes["raspberrypi"] = {
"host": os.environ["LAB_RASPBERRY_HOST"],
"user": os.environ["LAB_RASPBERRY_USER"],
"node_name": os.environ["LAB_RASPBERRY_NODE_NAME"],
"ssh_key_path": os.environ["LAB_RASPBERRY_SSH_KEY_PATH"],
}
node_labels["raspberrypi"] = raspberry_labels
for spec_file in spec_files:
path = Path(spec_file)
if not path.exists() or path.stat().st_size == 0:
continue
labels = pimox_labels if path.name == "pimox-workers.tsv" else manual_labels
with path.open(encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
line = line.rstrip("\n")
if not line or line.startswith("#"):
continue
try:
key, host, user, node_name, ssh_key_path = line.split("\t")
except ValueError as exc:
raise SystemExit(
f"{spec_file}:{line_number}: expected tab-separated "
"key, host, user, node_name, ssh_key_path"
) from exc
nodes[key] = {
"host": host,
"user": user,
"node_name": node_name,
"ssh_key_path": ssh_key_path,
}
node_labels[key] = labels
with open(var_file, "w", encoding="utf-8") as handle:
json.dump({"worker_nodes": nodes, "worker_node_labels": node_labels}, handle, indent=2)
handle.write("\n")
PY
}
prepare_cluster_worker_var_file() {
local include_raspberry_default="$1"
local manual_spec_file="${REPO_ROOT}/.lab/manual-workers.tsv"
local pimox_spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-${include_raspberry_default}}"
mkdir -p "${REPO_ROOT}/.lab"
write_cluster_worker_var_file "${var_file}" "${manual_spec_file}" "${pimox_spec_file}"
export LAB_CLUSTER_VAR_FILE="${var_file}"
}
preflight_check() {
local description="$1"
local output
shift
printf '%s %s... ' '-->' "${description}"
if output="$("$@" 2>&1)"; then
printf 'ok\n'
return 0
fi
printf 'failed\n'
if [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed 's/^/ /' >&2
fi
return 1
}
preflight_warn() {
local description="$1"
local output
shift
printf '%s %s... ' '-->' "${description}"
if output="$("$@" 2>&1)"; then
printf 'ok\n'
return 0
fi
printf 'warning\n'
if [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed 's/^/ /' >&2
fi
return 0
}
check_gitea_reachable() {
local gitea_url="${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}"
curl -fsS --max-time 10 "${gitea_url}" >/dev/null
}
check_debian_docker_root() {
local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
local actual_root
if ! command -v docker >/dev/null 2>&1; then
echo "docker command was not found on Debian host" >&2
return 1
fi
actual_root="$(sudo docker info --format '{{.DockerRootDir}}')"
if [[ "${actual_root}" != "${expected_root}" ]]; then
echo "Debian Docker root is ${actual_root}; expected ${expected_root}" >&2
return 1
fi
}
fix_debian_docker_root() {
local expected_root="${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}"
local daemon_file="/etc/docker/daemon.json"
local current_root=""
require_debian_server "fix-debian-docker-root"
if ! command -v docker >/dev/null 2>&1; then
echo "docker command was not found on Debian host" >&2
exit 1
fi
current_root="$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)"
if [[ "${current_root}" == "${expected_root}" ]]; then
echo "Debian Docker root is already ${expected_root}."
return 0
fi
echo "Moving Debian Docker root from ${current_root:-unknown} to ${expected_root}..."
sudo mkdir -p "${expected_root}" /etc/docker
echo "Stopping Docker services..."
sudo systemctl stop docker 2>/dev/null || true
sudo systemctl stop containerd 2>/dev/null || true
if [[ -n "${current_root}" && -d "${current_root}" && "${current_root}" != "${expected_root}" ]]; then
echo "Copying existing Docker data to ${expected_root}..."
sudo rsync -aHAX --numeric-ids "${current_root}/" "${expected_root}/"
fi
sudo python3 - "${daemon_file}" "${expected_root}" <<'PY'
import json
import os
import sys
path, expected_root = sys.argv[1:3]
document = {}
if os.path.exists(path) and os.path.getsize(path) > 0:
with open(path, encoding="utf-8") as handle:
document = json.load(handle)
document["data-root"] = expected_root
with open(path, "w", encoding="utf-8") as handle:
json.dump(document, handle, indent=2, sort_keys=True)
handle.write("\n")
PY
echo "Starting Docker services..."
sudo systemctl start containerd
sudo systemctl start docker
check_debian_docker_root
echo "Debian Docker root is now ${expected_root}."
}
check_debian_tailscale_ip() {
local expected_ip="${LAB_DEBIAN_TAILSCALE_IP:-}"
if [[ -z "${expected_ip}" ]]; then
return 0
fi
if ! command -v tailscale >/dev/null 2>&1; then
echo "tailscale command was not found on Debian host" >&2
return 1
fi
if ! tailscale ip -4 | grep -Fxq "${expected_ip}"; then
echo "Debian Tailscale IP does not include ${expected_ip}" >&2
tailscale ip -4 >&2 || true
return 1
fi
}
check_edge_ssh() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "true"
}
check_rpi_ssh() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "true"
}
check_rpi_docker_root_state() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
local fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
actual_root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
if [ \"\$actual_root\" = '${nvme_root}' ]; then
mountpoint -q '${nvme_root}' || { echo 'Docker uses ${nvme_root}, but it is not a mount point' >&2; exit 1; }
exit 0
fi
if [ \"\$actual_root\" = '${fallback_root}' ]; then
echo 'Docker is currently using fallback root ${fallback_root}; NVMe may be unavailable.' >&2
exit 0
fi
echo \"RPi Docker root is \$actual_root; expected ${nvme_root} or ${fallback_root}\" >&2
exit 1"
}
check_rpi_tailscale_ip() {
local expected_ip="${LAB_RPI_TAILSCALE_IP:-}"
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
if [[ -z "${expected_ip}" ]]; then
return 0
fi
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "tailscale ip -4 | grep -Fx '${expected_ip}'"
}
check_pimox_storage() {
local mode="${LAB_PIMOX_PIPELINE:-true}"
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
if disabled_value "${mode}"; then
return 0
fi
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
echo 'pvesm was not found' >&2
exit 1
fi
sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'"
}
check_local_disk_free_gib() {
local path="$1"
local min_gib="$2"
local available_kib
local min_kib
available_kib="$(df -Pk "${path}" | awk 'NR == 2 { print $4 }')"
min_kib=$((min_gib * 1024 * 1024))
if ((available_kib < min_kib)); then
echo "${path} has $((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB" >&2
return 1
fi
}
check_systemd_active() {
local unit="$1"
systemctl is-active --quiet "${unit}"
}
check_rpi_docker_writable() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "set -eu
root=\"\$(sudo docker info --format '{{.DockerRootDir}}')\"
sudo mkdir -p \"\$root/.homelab-check\"
sudo sh -c \"echo ok > '\$root/.homelab-check/write-test'\"
sudo rm -f \"\$root/.homelab-check/write-test\""
}
check_edge_disk_free() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
local min_gib="${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2}"
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "set -eu
available_kib=\"\$(df -Pk / | awk 'NR == 2 { print \$4 }')\"
min_kib=$((min_gib * 1024 * 1024))
if [ \"\$available_kib\" -lt \"\$min_kib\" ]; then
echo \"edge / has \$((available_kib / 1024 / 1024)) GiB free; expected at least ${min_gib} GiB\" >&2
exit 1
fi"
}
check_pihole_dns_query() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local query_name="${LAB_PIHOLE_STATUS_QUERY:-cloudflare.com}"
local output
if command -v dig >/dev/null 2>&1; then
output="$(dig @"${rpi_host}" "${query_name}" A +time=3 +tries=1 +short 2>&1)" || {
printf '%s\n' "${output}" >&2
echo "Pi-hole DNS query to ${rpi_host}:53 failed; check RPi firewall and that the Pi-hole container publishes 53/udp on the LAN address." >&2
return 1
}
if [[ -z "${output}" ]]; then
echo "Pi-hole returned no A records for ${query_name}" >&2
return 1
fi
return 0
fi
if command -v nslookup >/dev/null 2>&1; then
nslookup "${query_name}" "${rpi_host}" >/dev/null
return $?
fi
echo "dig or nslookup is required to test Pi-hole DNS directly." >&2
return 1
}
homelab_preflight() {
local phase="${1:-full}"
local failures=0
require_debian_server "preflight"
if truthy "${LAB_SKIP_PREFLIGHT:-false}"; then
echo "Skipping homelab preflight because LAB_SKIP_PREFLIGHT=${LAB_SKIP_PREFLIGHT}."
return 0
fi
case "${phase}" in
early | full)
;;
*)
echo "Unsupported preflight phase '${phase}'." >&2
exit 1
;;
esac
echo "Running ${phase} homelab preflight checks from ${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}..."
preflight_check "Debian Docker root is ${LAB_DEBIAN_DOCKER_ROOT:-/var/lib/docker}" check_debian_docker_root || failures=$((failures + 1))
preflight_check "Debian Tailscale IP ${LAB_DEBIAN_TAILSCALE_IP:-unset}" check_debian_tailscale_ip || failures=$((failures + 1))
preflight_check "RPi SSH ${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}@${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" check_rpi_ssh || failures=$((failures + 1))
if [[ "${phase}" == "full" ]]; then
preflight_check "Gitea reachable at ${LAB_GITEA_LOCAL_URL:-http://${LAB_GITEA_HOST:-192.168.100.73}:${LAB_GITEA_HTTP_PORT:-3000}/}" check_gitea_reachable || failures=$((failures + 1))
preflight_check "RPi Docker root is NVMe or fallback" check_rpi_docker_root_state || failures=$((failures + 1))
fi
preflight_warn "RPi Tailscale IP ${LAB_RPI_TAILSCALE_IP:-unset}" check_rpi_tailscale_ip
preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
preflight_check "OCI edge SSH ${LAB_EDGE_USER:-ubuntu}@${LAB_EDGE_HOST:-132.145.170.74}" check_edge_ssh || failures=$((failures + 1))
if ((failures > 0)); then
echo "Preflight failed with ${failures} blocking check(s). Fix the inventory or host state before continuing." >&2
exit 1
fi
echo "${phase^} preflight checks passed."
}
doctor_preapply() {
local failures=0
require_debian_server "doctor-preapply"
echo "Running pre-apply readiness checks..."
preflight_check "Debian / has at least ${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10} GiB free" check_local_disk_free_gib / "${LAB_PREFLIGHT_ROOT_MIN_FREE_GIB:-10}" || failures=$((failures + 1))
preflight_check "Debian /data has at least ${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20} GiB free" check_local_disk_free_gib /data "${LAB_PREFLIGHT_DATA_MIN_FREE_GIB:-20}" || failures=$((failures + 1))
preflight_check "Docker is active" check_systemd_active docker || failures=$((failures + 1))
preflight_check "containerd is active" check_systemd_active containerd || failures=$((failures + 1))
preflight_check "RPi Docker root is writable" check_rpi_docker_writable || failures=$((failures + 1))
preflight_check "Pimox storage ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}} is active" check_pimox_storage || failures=$((failures + 1))
preflight_check "OCI edge / has at least ${LAB_PREFLIGHT_EDGE_MIN_FREE_GIB:-2} GiB free" check_edge_disk_free || failures=$((failures + 1))
preflight_check "Pi-hole DNS query resolves" check_pihole_dns_query || failures=$((failures + 1))
if ((failures > 0)); then
echo "Pre-apply doctor failed with ${failures} blocking check(s)." >&2
exit 1
fi
echo "Pre-apply doctor checks passed."
}
inventory_check() {
"${REPO_ROOT}/scripts/validate-homelab-inventory" "${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
}
run_pimox_pipeline() {
local mode="${LAB_PIMOX_PIPELINE:-true}"
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local bridge="${LAB_PIMOX_BRIDGE:-${TF_VAR_pimox_template_bridge:-vmbr0}}"
local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}"
local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}"
local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}"
local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-4096}}"
local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}"
local provisioning_interface
local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_name_prefix="${LAB_PIMOX_WORKER_NAME_PREFIX:-pimox-worker}"
local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}"
local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}"
local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}"
local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}"
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}"
local timeout_seconds="${LAB_PIMOX_GUEST_TIMEOUT_SECONDS:-3600}"
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
local index
local readiness_output
local readiness_status
local worker_cpu_affinity
if disabled_value "${mode}"; then
return 0
fi
if [[ "${mode}" == "auto" && -n "${LAB_PIMOX_WORKER_COUNT+x}" ]]; then
mode="true"
fi
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2
exit 1
fi
if ! [[ "${template_cores}" =~ ^[0-9]+$ && "${worker_cores}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_TEMPLATE_CORES and LAB_PIMOX_WORKER_CORES must be positive integers." >&2
exit 1
fi
if ! [[ "${template_memory}" =~ ^[0-9]+$ && "${worker_memory}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_TEMPLATE_MEMORY and LAB_PIMOX_WORKER_MEMORY must be positive integer MiB values." >&2
exit 1
fi
if ((template_cores == 0 || worker_cores == 0 || template_memory == 0 || worker_memory == 0)); then
echo "Pimox template and worker CPU and memory values must be greater than zero." >&2
exit 1
fi
if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then
echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2
exit 1
fi
if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2
exit 1
fi
if [[ "${worker_storage}" == "local" ]]; then
echo "LAB_PIMOX_WORKER_STORAGE cannot be local; only the Pimox template VM should live on local storage." >&2
exit 1
fi
set +e
readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! { command -v qm >/dev/null 2>&1 || [ -x '${qm_bin}' ]; }; then
echo 'qm was not found in PATH and ${qm_bin} is not executable'
exit 1
fi
if ! ip link show '${bridge}' >/dev/null 2>&1; then
echo 'bridge ${bridge} was not found'
exit 1
fi
if ! sudo -n true >/dev/null 2>&1; then
echo 'passwordless sudo is not available for ${pimox_user}'
exit 1
fi
pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\"
if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then
pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z \"\$pvesm_cmd\" ]; then
echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}'
exit 1
fi
if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage && \$3 == \"active\" { found = 1 } END { exit found ? 0 : 1 }'; then
echo 'Pimox worker storage ${worker_storage} was not found or is not active'
exit 1
fi" 2>&1)"
readiness_status=$?
set -e
if ((readiness_status != 0)); then
if [[ "${mode}" == "auto" ]]; then
echo "Skipping Pimox automation because ${pimox_user}@${pimox_host} with bridge ${bridge} is not ready."
return 0
fi
echo "Pimox automation requested, but ${pimox_user}@${pimox_host} is not ready: ${readiness_output}" >&2
exit 1
fi
ensure_python3
provisioning_interface="${TF_VAR_provisioning_interface:-${LAB_PROVISIONING_INTERFACE:-$(detect_route_interface "${pimox_host}")}}"
if [[ -z "${provisioning_interface}" ]]; then
echo "Could not detect the Debian interface used to reach ${pimox_host}; set LAB_PROVISIONING_INTERFACE." >&2
exit 1
fi
export TF_VAR_provisioning_interface="${provisioning_interface}"
export TF_VAR_pimox_host="${pimox_host}"
export TF_VAR_pimox_user="${pimox_user}"
export TF_VAR_pimox_ssh_key_path="${pimox_key}"
export TF_VAR_pimox_qm_bin="${qm_bin}"
export TF_VAR_pimox_template_bridge="${bridge}"
export TF_VAR_pimox_template_vmid="${template_vmid}"
export TF_VAR_pimox_template_name="${template_name}"
export TF_VAR_pimox_template_cores="${template_cores}"
export TF_VAR_pimox_template_memory="${template_memory}"
export TF_VAR_pimox_template_replace_existing="${template_replace_existing}"
export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}"
export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}"
export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}"
export TF_VAR_pimox_template_guest_ip_prefix="${TF_VAR_pimox_template_guest_ip_prefix:-${ip_prefix}}"
export TF_VAR_pimox_template_build_timeout_seconds="${TF_VAR_pimox_template_build_timeout_seconds:-${timeout_seconds}}"
echo "Preparing Pimox provisioning and Debian worker template on ${pimox_host} without changing Orange Pi host networking..."
run_tofu_stack "bootstrap/provisioning"
if ((worker_count == 0)); then
return 0
fi
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${template_vmid}' | grep -q '^template: 1$'"; then
echo "Template VM ${template_vmid} is not available as a Pimox template after provisioning." >&2
exit 1
fi
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1"
echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage."
mkdir -p "${REPO_ROOT}/.lab"
: >"${spec_file}"
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
continue
fi
worker_cpu_affinity=""
if [[ -n "${worker_cpu_affinities}" ]]; then
worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")"
fi
ensure_pimox_worker_node \
"${index}" \
"${spec_file}" \
"${pimox_host}" \
"${pimox_user}" \
"${pimox_key}" \
"${template_vmid}" \
"${bridge}" \
"${worker_base_vmid}" \
"${worker_name_prefix}" \
"${worker_node_prefix}" \
"${worker_key_prefix}" \
"${worker_cores}" \
"${worker_memory}" \
"${worker_user}" \
"${worker_key_path}" \
"${ip_prefix}" \
"${timeout_seconds}" \
"${qm_bin}" \
"${worker_storage}" \
"${worker_replace_existing}" \
"${worker_cpu_affinity}"
done
write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}"
export LAB_CLUSTER_VAR_FILE="${var_file}"
}
run_openwrt_pipeline() {
local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}"
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local vmid="${LAB_OPENWRT_VMID:-9100}"
local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}"
local storage="${LAB_OPENWRT_STORAGE:-opi5_ssd}"
local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}"
local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}"
local cores="${LAB_OPENWRT_CORES:-2}"
local memory="${LAB_OPENWRT_MEMORY:-512}"
local version="${LAB_OPENWRT_VERSION:-24.10.6}"
local image_url="${LAB_OPENWRT_IMAGE_URL:-}"
local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}"
local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}"
local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}"
local start_vm="${LAB_OPENWRT_START:-false}"
local onboot="${LAB_OPENWRT_ONBOOT:-false}"
local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}"
local root_key_b64=""
local lan_dhcp_ignore="1"
local start_vm_flag="false"
local onboot_flag="0"
if disabled_value "${mode}"; then
return 0
fi
if ! truthy "${mode}"; then
echo "LAB_OPENWRT_VM must be true or false." >&2
exit 1
fi
if [[ -z "${image_url}" ]]; then
image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz"
fi
if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then
echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2
exit 1
fi
for value_name in storage wan_bridge lan_bridge vm_name; do
local value="${!value_name}"
if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2
exit 1
fi
done
if [[ "${storage}" == "local" ]]; then
echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2
exit 1
fi
if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then
echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2
exit 1
fi
if truthy "${lan_dhcp_enabled}"; then
lan_dhcp_ignore="0"
fi
if ! truthy "${start_vm}" && ! disabled_value "${start_vm}"; then
echo "LAB_OPENWRT_START must be true or false." >&2
exit 1
fi
if truthy "${start_vm}"; then
start_vm_flag="true"
fi
if ! truthy "${onboot}" && ! disabled_value "${onboot}"; then
echo "LAB_OPENWRT_ONBOOT must be true or false." >&2
exit 1
fi
if truthy "${onboot}"; then
onboot_flag="1"
fi
if [[ -r "${root_key_path}" ]]; then
root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')"
fi
echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..."
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <<EOF
set -euo pipefail
vmid="${vmid}"
vm_name="${vm_name}"
storage="${storage}"
wan_bridge="${wan_bridge}"
lan_bridge="${lan_bridge}"
cores="${cores}"
memory="${memory}"
image_url="${image_url}"
lan_ip="${lan_ip}"
lan_netmask="${lan_netmask}"
lan_dhcp_ignore="${lan_dhcp_ignore}"
start_vm="${start_vm_flag}"
onboot="${onboot_flag}"
root_key_b64="${root_key_b64}"
qm_cmd="${qm_bin}"
if [ ! -x "\$qm_cmd" ]; then
qm_cmd="\$(command -v qm 2>/dev/null || true)"
fi
if [ -z "\$qm_cmd" ]; then
echo "qm is not installed on this Pimox host" >&2
exit 1
fi
pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)"
if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then
pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z "\$pvesm_cmd" ]; then
echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2
exit 1
fi
if ! sudo -n true >/dev/null 2>&1; then
echo "passwordless sudo is required for OpenWrt VM automation" >&2
exit 1
fi
if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then
echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2
exit 1
fi
if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then
echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2
exit 1
fi
if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
echo "Pimox storage \$storage was not found." >&2
exit 1
fi
if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then
if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then
echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2
exit 1
fi
sudo "\$qm_cmd" set "\$vmid" \\
--net0 "virtio,bridge=\$wan_bridge" \\
--net1 "virtio,bridge=\$lan_bridge" \\
--cores "\$cores" \\
--memory "\$memory" \\
--onboot "\$onboot"
if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then
sudo "\$qm_cmd" start "\$vmid"
fi
exit 0
fi
for required_cmd in curl gzip losetup mount umount awk sed; do
if ! command -v "\$required_cmd" >/dev/null 2>&1; then
echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2
exit 1
fi
done
tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)"
mnt_dir="\$tmp_dir/root"
loopdev=""
cleanup() {
if mountpoint -q "\$mnt_dir" 2>/dev/null; then
sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true
fi
if [ -n "\$loopdev" ]; then
sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true
fi
rm -rf "\$tmp_dir"
}
trap cleanup EXIT
mkdir -p "\$mnt_dir"
curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz"
gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img"
loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")"
root_part="\${loopdev}p2"
if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then
root_part="\${loopdev}p2"
fi
if [ ! -b "\$root_part" ]; then
echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2
exit 1
fi
sudo mount "\$root_part" "\$mnt_dir"
sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh"
cat >"\$tmp_dir/network" <<NETWORK
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
config globals 'globals'
option ula_prefix 'fd00:68:50::/48'
config interface 'wan'
option device 'eth0'
option proto 'dhcp'
config interface 'lan'
option device 'eth1'
option proto 'static'
option ipaddr '\$lan_ip'
option netmask '\$lan_netmask'
option ip6assign '60'
NETWORK
cat >"\$tmp_dir/dhcp" <<DHCP
config dnsmasq
option domainneeded '1'
option boguspriv '1'
option filterwin2k '0'
option localise_queries '1'
option rebind_protection '1'
option rebind_localhost '1'
option local '/lan/'
option domain 'lan'
option expandhosts '1'
option cachesize '1000'
option authoritative '1'
option readethers '1'
option leasefile '/tmp/dhcp.leases'
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
config dhcp 'lan'
option interface 'lan'
option start '100'
option limit '150'
option leasetime '12h'
option ignore '\$lan_dhcp_ignore'
config dhcp 'wan'
option interface 'wan'
option ignore '1'
DHCP
cat >"\$tmp_dir/firewall" <<'FIREWALL'
config defaults
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option synflood_protect '1'
config zone
option name 'lan'
list network 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
config zone
option name 'wan'
list network 'wan'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
config forwarding
option src 'lan'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
FIREWALL
cat >"\$tmp_dir/system" <<SYSTEM
config system
option hostname '\$vm_name'
option timezone 'UTC'
option ttylogin '0'
option log_size '64'
option urandom_seed '0'
SYSTEM
sudo cp "\$tmp_dir/network" "\$mnt_dir/etc/config/network"
sudo cp "\$tmp_dir/dhcp" "\$mnt_dir/etc/config/dhcp"
sudo cp "\$tmp_dir/firewall" "\$mnt_dir/etc/config/firewall"
sudo cp "\$tmp_dir/system" "\$mnt_dir/etc/config/system"
if [ -n "\$root_key_b64" ]; then
printf '%s' "\$root_key_b64" | base64 -d >"\$tmp_dir/authorized_keys"
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys"
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
fi
sync
sudo umount "\$mnt_dir"
sudo losetup -d "\$loopdev"
loopdev=""
sudo "\$qm_cmd" create "\$vmid" \\
--name "\$vm_name" \\
--bios ovmf \\
--cores "\$cores" \\
--memory "\$memory" \\
--net0 "virtio,bridge=\$wan_bridge" \\
--net1 "virtio,bridge=\$lan_bridge" \\
--numa 0 \\
--ostype l26 \\
--scsihw virtio-scsi-pci \\
--sockets 1 \\
--vga virtio \\
--onboot "\$onboot"
sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0"
sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null
disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')"
if [ -z "\$disk_volume" ]; then
echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2
exit 1
fi
sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume"
sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0"
if [ "\$start_vm" = "true" ]; then
sudo "\$qm_cmd" start "\$vmid"
fi
EOF
}
openwrt() {
require_debian_server "openwrt"
LAB_OPENWRT_VM=true run_openwrt_pipeline
}
cleanup_calico_links() {
ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true
sudo ip link delete vxlan.calico 2>/dev/null || true
sudo ip link delete tunl0 2>/dev/null || true
sudo ip link delete cni0 2>/dev/null || true
sudo ip link delete kube-ipvs0 2>/dev/null || true
ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true
}
cleanup_iptables() {
sudo iptables -F || true
sudo iptables -X || true
sudo iptables -t nat -F || true
sudo iptables -t nat -X || true
sudo iptables -t mangle -F || true
sudo iptables -t mangle -X || true
sudo iptables -t raw -F || true
sudo iptables -t raw -X || true
if command -v ipvsadm >/dev/null 2>&1; then
sudo ipvsadm --clear || true
fi
}
cleanup_calico_runtime_files() {
local path
for path in /run/calico /var/run/calico; do
if sudo test -e "${path}"; then
sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true
sudo rmdir "${path}" 2>/dev/null || true
fi
done
}
restore_node_dns() {
sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf
if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then
sudo rm -f /etc/resolv.conf
sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf
fi
sudo systemctl restart systemd-resolved 2>/dev/null || true
}
cleanup_mounts() {
if command -v findmnt >/dev/null 2>&1; then
local mount_root
while IFS= read -r mountpoint; do
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
done < <(
for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do
findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true
done | sort -ru
)
fi
while IFS= read -r mountpoint; do
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true)
sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true
}
cleanup_node() {
sudo kubeadm reset --force || true
sudo systemctl stop kubelet 2>/dev/null || true
sudo systemctl stop containerd 2>/dev/null || true
sudo killall containerd-shim-runc-v2 2>/dev/null || true
cleanup_mounts
sudo rm -rf \
/etc/kubernetes/ \
/var/lib/etcd/ \
/var/lib/kubelet/ \
/var/lib/cni/ \
/etc/cni/net.d \
/run/flannel \
/var/lib/calico \
/var/log/calico \
/var/lib/containerd/* \
/run/containerd/* \
/etc/containerd/certs.d \
/etc/containerd/config.toml
cleanup_calico_runtime_files
sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam
cleanup_iptables
cleanup_calico_links
restore_node_dns
sudo mkdir -p /etc/containerd/certs.d
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
sudo systemctl start containerd 2>/dev/null || true
}
image_ref_tag() {
local image_ref="$1"
local tag
tag="${image_ref##*:}"
if [[ "${tag}" == "${image_ref}" || "${tag}" == */* ]]; then
echo "Image reference ${image_ref} must include an immutable tag." >&2
exit 1
fi
printf '%s\n' "${tag}"
}
website_image_tag() {
local source_hash="$1"
printf 'src-%s\n' "${source_hash:0:12}"
}
apps_registry_endpoint() {
if [[ -n "${TF_VAR_registry_endpoint:-}" ]]; then
printf '%s\n' "${TF_VAR_registry_endpoint}"
return 0
fi
demos_registry_endpoint
}
demos_registry_endpoint() {
local image
image="$(awk '$1 == "image:" && $2 ~ /demos-static/ {print $2; exit}' "${REPO_ROOT}/apps/demos-static/web-app.yaml")"
if [[ -z "${image}" || "${image}" != */* ]]; then
echo "Could not determine demos registry endpoint from apps/demos-static/web-app.yaml" >&2
exit 1
fi
printf '%s\n' "${image%%/*}"
}
website_source_hash() {
(
cd "${REPO_ROOT}"
find apps/website -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
)
}
demos_source_hash() {
(
cd "${REPO_ROOT}"
find apps/demos-static -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum | awk '{print $1}'
)
}
registry_image_exists() {
local registry_endpoint="$1"
local repository="$2"
local tag="$3"
local accept_header
if ! command -v curl >/dev/null 2>&1; then
return 1
fi
accept_header="application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json"
curl -fsS \
-H "Accept: ${accept_header}" \
"http://${registry_endpoint}/v2/${repository}/manifests/${tag}" >/dev/null
}
image_state_value() {
local state_file="$1"
local key="$2"
awk -F= -v key="${key}" '$1 == key {print substr($0, index($0, "=") + 1); exit}' "${state_file}" 2>/dev/null || true
}
website_image_is_current() {
local state_file="$1"
local source_hash="$2"
local platforms="$3"
local image_ref="$4"
local registry_endpoint="$5"
local image_tag
local saved_hash
local saved_platforms
local saved_image
[[ -f "${state_file}" ]] || return 1
saved_hash="$(image_state_value "${state_file}" source_hash)"
saved_platforms="$(image_state_value "${state_file}" platforms)"
saved_image="$(image_state_value "${state_file}" image)"
[[ "${saved_hash}" == "${source_hash}" ]] || return 1
[[ "${saved_platforms}" == "${platforms}" ]] || return 1
[[ "${saved_image}" == "${image_ref}" ]] || return 1
image_tag="$(image_ref_tag "${image_ref}")"
registry_image_exists "${registry_endpoint}" php-website "${image_tag}"
}
ensure_website_image_tag_not_reused() {
local state_file="$1"
local source_hash="$2"
local image_ref="$3"
local saved_hash
local saved_image
[[ -f "${state_file}" ]] || return 0
saved_hash="$(image_state_value "${state_file}" source_hash)"
saved_image="$(image_state_value "${state_file}" image)"
if [[ -n "${saved_hash}" && -n "${saved_image}" && "${saved_hash}" != "${source_hash}" && "${saved_image}" == "${image_ref}" ]]; then
echo "Website source changed but the computed php-website image ref is still ${image_ref}." >&2
echo "Check WEBSITE_IMAGE_TAG or the website image tag generator before rebuilding." >&2
exit 1
fi
}
demos_image_is_current() {
local state_file="$1"
local source_hash="$2"
local platforms="$3"
local image_ref="$4"
local registry_endpoint="$5"
local saved_hash
local saved_platforms
local saved_image
[[ -f "${state_file}" ]] || return 1
saved_hash="$(image_state_value "${state_file}" source_hash)"
saved_platforms="$(image_state_value "${state_file}" platforms)"
saved_image="$(image_state_value "${state_file}" image)"
[[ "${saved_hash}" == "${source_hash}" ]] || return 1
[[ "${saved_platforms}" == "${platforms}" ]] || return 1
[[ "${saved_image}" == "${image_ref}" ]] || return 1
registry_image_exists "${registry_endpoint}" demos-static latest
}
write_website_image_state() {
local state_file="$1"
local source_hash="$2"
local platforms="$3"
local image_ref="$4"
mkdir -p "$(dirname "${state_file}")"
{
printf 'source_hash=%s\n' "${source_hash}"
printf 'platforms=%s\n' "${platforms}"
printf 'image=%s\n' "${image_ref}"
} > "${state_file}"
}
write_demos_image_state() {
local state_file="$1"
local source_hash="$2"
local platforms="$3"
local image_ref="$4"
mkdir -p "$(dirname "${state_file}")"
{
printf 'source_hash=%s\n' "${source_hash}"
printf 'platforms=%s\n' "${platforms}"
printf 'image=%s\n' "${image_ref}"
} > "${state_file}"
}
path_available_mb() {
local path="$1"
while [[ ! -e "${path}" && "${path}" != "/" ]]; do
path="$(dirname "${path}")"
done
df -Pm "${path}" | awk 'NR == 2 {print $4}'
}
docker_root_dir() {
docker info --format '{{.DockerRootDir}}' 2>/dev/null || printf '/var/lib/docker\n'
}
prune_unused_docker_build_data() {
docker buildx rm lab-builder 2>/dev/null || true
docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true
docker builder prune -af 2>/dev/null || true
docker system prune -af 2>/dev/null || true
}
ensure_docker_build_space() {
local docker_root
local free_mb
local min_free_mb
min_free_mb="${DOCKER_BUILD_MIN_FREE_MB:-4096}"
docker_root="$(docker_root_dir)"
free_mb="$(path_available_mb "${docker_root}")"
if (( free_mb >= min_free_mb )); then
return 0
fi
echo "Docker data root ${docker_root} has ${free_mb}MiB free; pruning unused Docker build data..."
prune_unused_docker_build_data
free_mb="$(path_available_mb "${docker_root}")"
if (( free_mb < min_free_mb )); then
echo "Docker data root ${docker_root} still has only ${free_mb}MiB free after cleanup." >&2
echo "Free space there or move Docker's data-root to a larger filesystem such as /home before building." >&2
echo "Override the threshold with DOCKER_BUILD_MIN_FREE_MB if this host can build with less space." >&2
exit 1
fi
}
prepare_buildx_builder() {
local registry_endpoint="$1"
docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
cat <<EOF > "${BUILDX_CONFIG}"
[registry."${registry_endpoint}"]
http = true
insecure = true
[registry."127.0.0.1:30500"]
http = true
insecure = true
[registry."localhost:30500"]
http = true
insecure = true
EOF
docker buildx rm lab-builder 2>/dev/null || true
docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use
docker buildx inspect --bootstrap
}
ensure_cosign_available() {
local arch
local cosign_url
local target_dir
local target_path
local tmp_path
if [[ -n "${COSIGN_BIN}" ]]; then
if [[ -x "${COSIGN_BIN}" ]]; then
return 0
fi
echo "COSIGN_BIN points to ${COSIGN_BIN}, but it is not executable." >&2
exit 1
fi
if command -v cosign >/dev/null 2>&1; then
COSIGN_BIN="$(command -v cosign)"
return 0
fi
case "$(uname -m)" in
x86_64 | amd64)
arch="amd64"
;;
aarch64 | arm64)
arch="arm64"
;;
*)
echo "Unsupported Cosign install architecture: $(uname -m)" >&2
exit 1
;;
esac
target_dir="${HOMELAB_STATE_DIR}/bin"
target_path="${target_dir}/cosign-v${COSIGN_VERSION}-linux-${arch}"
if [[ -x "${target_path}" ]]; then
COSIGN_BIN="${target_path}"
return 0
fi
if ! command -v curl >/dev/null 2>&1; then
echo "curl is required to download Cosign ${COSIGN_VERSION}." >&2
exit 1
fi
mkdir -p "${target_dir}"
tmp_path="${target_path}.tmp"
cosign_url="https://github.com/sigstore/cosign/releases/download/v${COSIGN_VERSION}/cosign-linux-${arch}"
echo "Installing Cosign ${COSIGN_VERSION} into ${target_path}..."
curl -fsSL "${cosign_url}" -o "${tmp_path}"
chmod 0755 "${tmp_path}"
mv "${tmp_path}" "${target_path}"
COSIGN_BIN="${target_path}"
}
ensure_cosign_password() {
if [[ -n "${COSIGN_PASSWORD:-}" ]]; then
return 0
fi
if [[ -r "${COSIGN_PASSWORD_FILE}" ]]; then
COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
export COSIGN_PASSWORD
return 0
fi
mkdir -p "$(dirname "${COSIGN_PASSWORD_FILE}")"
if command -v openssl >/dev/null 2>&1; then
openssl rand -hex 32 > "${COSIGN_PASSWORD_FILE}"
elif command -v python3 >/dev/null 2>&1; then
python3 - <<'PY' > "${COSIGN_PASSWORD_FILE}"
import secrets
print(secrets.token_hex(32))
PY
else
echo "openssl or python3 is required to generate a non-interactive Cosign key password." >&2
exit 1
fi
chmod 0600 "${COSIGN_PASSWORD_FILE}"
COSIGN_PASSWORD="$(<"${COSIGN_PASSWORD_FILE}")"
export COSIGN_PASSWORD
}
run_cosign() {
ensure_cosign_available
ensure_cosign_password
COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" "$@"
}
ensure_cosign_keypair() {
ensure_cosign_available
ensure_cosign_password
if [[ -f "${COSIGN_KEY_PATH}" && -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
return 0
fi
if [[ -f "${COSIGN_KEY_PATH}" || -f "${COSIGN_PUBLIC_KEY_PATH}" ]]; then
echo "Found only one Cosign key file. Expected both ${COSIGN_KEY_PATH} and ${COSIGN_PUBLIC_KEY_PATH}." >&2
exit 1
fi
mkdir -p "$(dirname "${COSIGN_KEY_PATH}")"
echo "Generating homelab Cosign key pair under ${REPO_ROOT}/.lab..."
COSIGN_PASSWORD="${COSIGN_PASSWORD}" "${COSIGN_BIN}" generate-key-pair --output-key-prefix "${COSIGN_KEY_PREFIX}"
chmod 0600 "${COSIGN_KEY_PATH}"
chmod 0644 "${COSIGN_PUBLIC_KEY_PATH}"
}
publish_cosign_public_key_config_map() {
kubectl --kubeconfig "${KUBECONFIG}" create namespace kyverno --dry-run=client -o yaml |
kubectl --kubeconfig "${KUBECONFIG}" apply -f -
kubectl --kubeconfig "${KUBECONFIG}" -n kyverno create configmap "${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP}" \
--from-file=cosign.pub="${COSIGN_PUBLIC_KEY_PATH}" \
--dry-run=client -o yaml |
kubectl --kubeconfig "${KUBECONFIG}" apply -f -
}
cosign_registry_flags() {
local registry_endpoint="$1"
local image_ref="$2"
if [[ "${image_ref}" == "${registry_endpoint}/"* ]]; then
printf '%s\n' "--allow-http-registry"
fi
}
image_supply_chain_metadata_exists() {
local image_ref="$1"
local registry_endpoint="$2"
local digest_ref
local -a registry_flags=()
if ! digest_ref="$(image_digest_ref "${image_ref}")"; then
return 1
fi
mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")
run_cosign verify \
--key "${COSIGN_PUBLIC_KEY_PATH}" \
--insecure-ignore-tlog=true \
"${registry_flags[@]}" \
"${digest_ref}" >/dev/null 2>&1 &&
run_cosign verify-attestation \
--key "${COSIGN_PUBLIC_KEY_PATH}" \
--type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
--insecure-ignore-tlog=true \
"${registry_flags[@]}" \
"${digest_ref}" >/dev/null 2>&1
}
write_image_sbom_predicate() {
local image_ref="$1"
local output_file="$2"
local source_hash="$3"
docker buildx imagetools inspect "${image_ref}" --format '{{ json .SBOM.SPDX }}' > "${output_file}"
python3 - "${output_file}" "${image_ref}" "${source_hash}" <<'PY'
from datetime import datetime, timezone
import json
import re
import sys
path = sys.argv[1]
image_ref = sys.argv[2]
source_hash = sys.argv[3]
try:
with open(path, encoding="utf-8") as handle:
document = json.load(handle)
except json.JSONDecodeError:
document = None
if not isinstance(document, dict):
safe_ref = re.sub(r"[^A-Za-z0-9.-]+", "-", image_ref).strip("-")
document = {
"spdxVersion": "SPDX-2.3",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": f"homelab image SBOM for {image_ref}",
"documentNamespace": f"https://homelab.local/spdx/{safe_ref}/{source_hash}",
"creationInfo": {
"created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
"creators": ["Tool: jeannie"],
},
"packages": [
{
"name": image_ref,
"SPDXID": "SPDXRef-Image",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": False,
"licenseConcluded": "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"copyrightText": "NOASSERTION",
}
],
"relationships": [
{
"spdxElementId": "SPDXRef-DOCUMENT",
"relationshipType": "DESCRIBES",
"relatedSpdxElement": "SPDXRef-Image",
}
],
}
with open(path, "w", encoding="utf-8") as handle:
json.dump(document, handle, sort_keys=True)
handle.write("\n")
if not isinstance(document, dict):
raise SystemExit("image SBOM predicate is not a JSON object")
if document.get("SPDXID") != "SPDXRef-DOCUMENT":
raise SystemExit("image SBOM predicate is not an SPDX document")
if not document.get("packages"):
raise SystemExit("image SBOM predicate does not include packages")
PY
}
image_digest_ref() {
local image_ref="$1"
local digest
local repository
if [[ "${image_ref}" == *@sha256:* ]]; then
printf '%s\n' "${image_ref}"
return 0
fi
digest="$(docker buildx imagetools inspect "${image_ref}" --format '{{ .Manifest.Digest }}')"
if [[ -z "${digest}" || "${digest}" == "<no value>" ]]; then
echo "Unable to resolve image digest for ${image_ref}." >&2
return 1
fi
if [[ "${image_ref##*/}" == *:* ]]; then
repository="${image_ref%:*}"
else
repository="${image_ref}"
fi
printf '%s@%s\n' "${repository}" "${digest}"
}
publish_image_supply_chain_metadata() {
local image_ref="$1"
local registry_endpoint="$2"
local source_hash="$3"
local digest_ref
local sbom_file
local -a registry_flags=()
if image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
echo "Image ${image_ref} already has a valid Cosign signature and signed SPDX SBOM."
return 0
fi
digest_ref="$(image_digest_ref "${image_ref}")"
mapfile -t registry_flags < <(cosign_registry_flags "${registry_endpoint}" "${digest_ref}")
sbom_file="$(mktemp)"
write_image_sbom_predicate "${digest_ref}" "${sbom_file}" "${source_hash}"
echo "Signing image ${digest_ref} and attaching signed SPDX SBOM..."
run_cosign sign \
--yes \
--key "${COSIGN_KEY_PATH}" \
--tlog-upload=false \
-a "homelab.dev/source-hash=${source_hash}" \
"${registry_flags[@]}" \
"${digest_ref}"
run_cosign attest \
--yes \
--key "${COSIGN_KEY_PATH}" \
--predicate "${sbom_file}" \
--type "${HOMELAB_SBOM_PREDICATE_TYPE}" \
--tlog-upload=false \
"${registry_flags[@]}" \
"${digest_ref}"
rm -f "${sbom_file}"
if ! image_supply_chain_metadata_exists "${image_ref}" "${registry_endpoint}"; then
echo "Cosign verification failed after signing ${digest_ref}." >&2
exit 1
fi
}
dump_argocd_debug() {
local app="$1"
kubectl --kubeconfig "${KUBECONFIG}" -n argocd get application "${app}" -o yaml || true
kubectl --kubeconfig "${KUBECONFIG}" -n argocd describe application "${app}" || true
kubectl --kubeconfig "${KUBECONFIG}" -n argocd get pods -o wide || true
kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs deployment/argocd-repo-server --tail=120 || true
kubectl --kubeconfig "${KUBECONFIG}" -n argocd logs statefulset/argocd-application-controller --tail=120 || true
}
dump_namespace_debug() {
local namespace="$1"
kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get all -o wide || true
kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pvc -o wide || true
kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe pods || true
kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
}
wait_for_namespace() {
local namespace="$1"
local app="$2"
local timeout_seconds="$3"
local elapsed=0
until kubectl --kubeconfig "${KUBECONFIG}" get namespace "${namespace}" >/dev/null 2>&1; do
if ((elapsed >= timeout_seconds)); then
echo "Timed out waiting for namespace ${namespace} from Argo CD app ${app}" >&2
dump_argocd_debug "${app}"
exit 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
wait_for_namespaced_resource() {
local namespace="$1"
local kind="$2"
local name="$3"
local app="$4"
local timeout_seconds="$5"
local elapsed=0
until kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get "${kind}/${name}" >/dev/null 2>&1; do
if ((elapsed >= timeout_seconds)); then
echo "Timed out waiting for ${kind}/${name} in namespace ${namespace} from Argo CD app ${app}" >&2
dump_argocd_debug "${app}"
kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get events --sort-by=.lastTimestamp 2>/dev/null | tail -80 || true
exit 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
wait_for_cluster_resource() {
local kind="$1"
local name="$2"
local app="$3"
local timeout_seconds="$4"
local elapsed=0
until kubectl --kubeconfig "${KUBECONFIG}" get "${kind}/${name}" >/dev/null 2>&1; do
if ((elapsed >= timeout_seconds)); then
echo "Timed out waiting for ${kind}/${name} from Argo CD app ${app}" >&2
dump_argocd_debug "${app}"
exit 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
wait_for_deployment_ready() {
local namespace="$1"
local deployment="$2"
local app="$3"
local timeout_seconds="$4"
local desired_replicas
local ready_replicas
local elapsed=0
desired_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.spec.replicas}' 2>/dev/null || true)"
desired_replicas="${desired_replicas:-1}"
until ready_replicas="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get deployment "${deployment}" -o jsonpath='{.status.readyReplicas}' 2>/dev/null)"; \
(( ${ready_replicas:-0} >= desired_replicas )); do
if ((elapsed >= timeout_seconds)); then
echo "Timed out waiting for deployment/${deployment} in namespace ${namespace} to have ${desired_replicas} ready replicas" >&2
dump_argocd_debug "${app}"
dump_namespace_debug "${namespace}"
exit 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
deploy_gitea() {
local mode="${LAB_GITEA_DEPLOY:-true}"
local gitea_host="${LAB_GITEA_HOST:-${LAB_DEBIAN_LAN_IP:-192.168.100.73}}"
local gitea_user="${LAB_GITEA_USER:-${LAB_DEBIAN_USER:-jv}}"
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local install_dir="${LAB_GITEA_INSTALL_DIR:-/data/homelab-gitea}"
local image="${LAB_GITEA_IMAGE:-gitea/gitea:1.21.7}"
local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
local domain="${LAB_GITEA_DOMAIN:-${LAB_DOMAIN:-lab2025.duckdns.org}}"
local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}"
local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
local compose_file="${REPO_ROOT}/infra/gitea/docker-compose.yml"
local install_docker="${LAB_GITEA_INSTALL_DOCKER:-false}"
require_debian_server "deploy-gitea"
if disabled_value "${mode}"; then
install_gitea_backup_timer
return 0
fi
if [[ ! -s "${compose_file}" ]]; then
echo "Missing ${compose_file}" >&2
exit 1
fi
echo "Deploying external Gitea on ${gitea_user}@${gitea_host}:${http_port}..."
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "rm -rf /tmp/homelab-gitea && mkdir -p /tmp/homelab-gitea"
scp -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${compose_file}" "${gitea_user}@${gitea_host}:/tmp/homelab-gitea/docker-compose.yml"
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "set -eu
install_dir='${install_dir}'
install_docker='${install_docker}'
install_missing_packages() {
missing_packages=''
for package in \"\$@\"; do
if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then
missing_packages=\"\$missing_packages \$package\"
fi
done
if [ -n \"\$missing_packages\" ]; then
sudo apt-get update
sudo apt-get install -y --no-install-recommends \$missing_packages
fi
}
install_missing_packages ca-certificates curl iptables
if ! command -v docker >/dev/null 2>&1; then
if [ \"\$install_docker\" = 'true' ]; then
curl -fsSL https://get.docker.com | sudo sh
else
echo 'Docker is not installed on the Gitea host. Install Docker through the host bootstrap first, or rerun with LAB_GITEA_INSTALL_DOCKER=true to allow get.docker.com installation.' >&2
exit 1
fi
fi
if ! sudo docker compose version >/dev/null 2>&1; then
install_missing_packages docker-compose-plugin
fi
repair_docker_iptables() {
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
return 0
fi
echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...'
sudo systemctl restart docker
sleep 3
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
return 0
fi
echo 'Docker NAT chain is still missing after restarting Docker.' >&2
sudo iptables -t nat -S >&2 || true
sudo systemctl status docker --no-pager -l >&2 || true
exit 1
}
repair_docker_iptables
sudo mkdir -p \"\$install_dir/data\"
sudo cp /tmp/homelab-gitea/docker-compose.yml \"\$install_dir/docker-compose.yml\"
sudo chown -R 1000:1000 \"\$install_dir/data\"
sudo tee \"\$install_dir/.env\" >/dev/null <<ENV_EOT
GITEA_IMAGE=${image}
GITEA_CONTAINER_NAME=${container_name}
GITEA_HTTP_PORT=${http_port}
GITEA_SSH_PORT=${ssh_port}
GITEA_DOMAIN=${domain}
GITEA_ROOT_URL=${root_url}
GITEA_UID=1000
GITEA_GID=1000
ENV_EOT
cd \"\$install_dir\"
sudo docker compose pull
sudo docker compose up -d --remove-orphans
sudo docker compose ps
"
install_gitea_backup_timer
}
deploy_rpi_services() {
local mode="${LAB_RPI_SERVICES_DEPLOY:-true}"
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
local data_dir="${LAB_RPI_SERVICES_DATA_DIR:-${install_dir}/data}"
local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
local stop_legacy_pihole="${LAB_RPI_STOP_LEGACY_PIHOLE:-true}"
local install_docker="${LAB_RPI_INSTALL_DOCKER:-false}"
local pihole_webpassword="${PIHOLE_WEBPASSWORD:-}"
local source_dir="${REPO_ROOT}/infra/rpi-services"
local value_name
local value
require_debian_server "rpi-services"
if disabled_value "${mode}"; then
return 0
fi
if [[ ! -d "${source_dir}" ]]; then
echo "Missing ${source_dir}" >&2
exit 1
fi
for value_name in install_dir data_dir docker_nvme_root docker_fallback_root stop_legacy_pihole install_docker pihole_webpassword; do
value="${!value_name}"
if [[ "${value}" == *"'"* ]]; then
echo "${value_name} cannot contain a single quote." >&2
exit 1
fi
done
echo "Deploying RPi services on ${rpi_user}@${rpi_host}..."
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "rm -rf /tmp/homelab-rpi-services && mkdir -p /tmp/homelab-rpi-services"
scp -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
"${source_dir}/docker-compose.yml" \
"${source_dir}/adlists.txt" \
"${source_dir}/local-dns-records.txt" \
"${source_dir}/cname-records.txt" \
"${source_dir}/static-dhcp-hosts.txt" \
"${source_dir}/uptime-kuma-monitors.json" \
"${source_dir}/seed-uptime-kuma.js" \
"${source_dir}/bootstrap.sh" \
"${rpi_user}@${rpi_host}:/tmp/homelab-rpi-services/"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
LAB_RPI_SERVICES_INSTALL_DIR='${install_dir}' \
LAB_RPI_SERVICES_DATA_DIR='${data_dir}' \
LAB_RPI_DOCKER_NVME_ROOT='${docker_nvme_root}' \
LAB_RPI_DOCKER_FALLBACK_ROOT='${docker_fallback_root}' \
LAB_RPI_STOP_LEGACY_PIHOLE='${stop_legacy_pihole}' \
LAB_RPI_INSTALL_DOCKER='${install_docker}' \
PIHOLE_WEBPASSWORD='${pihole_webpassword}' \
bash /tmp/homelab-rpi-services/bootstrap.sh"
}
gitea_bootstrap_password() {
if command -v openssl >/dev/null 2>&1; then
openssl rand -hex 32
return 0
fi
python3 - <<'PY'
import secrets
print(secrets.token_hex(32))
PY
}
gitea_api_base_url() {
local gitea_host="$1"
local http_port="$2"
local candidate
local api_base_override="${LAB_GITEA_API_BASE_URL:-}"
if [[ -n "${api_base_override}" ]]; then
printf '%s\n' "${api_base_override%/}"
return 0
fi
for candidate in "http://${gitea_host}:${http_port}/api/v1" "http://${gitea_host}:${http_port}/git/api/v1"; do
if curl -fsS "${candidate}/version" >/dev/null 2>&1; then
printf '%s\n' "${candidate}"
return 0
fi
done
echo "Could not reach the Gitea API on ${gitea_host}:${http_port}." >&2
exit 1
}
gitea_repo_exists() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local owner="$4"
local repo_name="$5"
local status
status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}")"
case "${status}" in
200)
return 0
;;
404)
return 1
;;
401 | 403)
echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}." >&2
exit 1
;;
*)
echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}." >&2
exit 1
;;
esac
}
gitea_branch_exists() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local owner="$4"
local repo_name="$5"
local branch="$6"
local status
status="$(curl -sS -o /dev/null -w '%{http_code}' -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/branches/${branch}")"
case "${status}" in
200)
return 0
;;
404)
return 1
;;
401 | 403)
echo "Gitea API authentication failed for ${auth_user} while checking ${owner}/${repo_name}:${branch}." >&2
exit 1
;;
*)
echo "Unexpected Gitea API response ${status} while checking ${owner}/${repo_name}:${branch}." >&2
exit 1
;;
esac
}
create_gitea_repo() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local repo_name="$4"
local default_branch="$5"
local payload
payload="$(python3 - "${repo_name}" "${default_branch}" <<'PY'
import json
import sys
repo_name, default_branch = sys.argv[1:3]
print(json.dumps({
"name": repo_name,
"private": False,
"auto_init": False,
"default_branch": default_branch,
"description": "Homelab infrastructure configuration",
}))
PY
)"
curl -fsS \
-u "${auth_user}:${auth_password}" \
-H "Content-Type: application/json" \
-X POST \
-d "${payload}" \
"${api_base}/user/repos" >/dev/null
}
gitea_public_key_registered() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local owner="$4"
local repo_name="$5"
local public_key_path="$6"
local repo_keys
local user_keys
user_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/user/keys?limit=100")"
repo_keys="$(curl -fsS -u "${auth_user}:${auth_password}" "${api_base}/repos/${owner}/${repo_name}/keys?limit=100")"
GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
GITEA_USER_KEYS="${user_keys}" \
GITEA_REPO_KEYS="${repo_keys}" \
python3 - <<'PY'
import json
import os
import sys
public_key = os.environ["GITEA_PUBLIC_KEY"].strip()
for env_name in ("GITEA_USER_KEYS", "GITEA_REPO_KEYS"):
for key in json.loads(os.environ[env_name]) or []:
if key.get("key", "").strip() == public_key:
sys.exit(0)
sys.exit(1)
PY
}
create_gitea_repo_deploy_key() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local owner="$4"
local repo_name="$5"
local title="$6"
local public_key_path="$7"
local read_only="$8"
local payload
payload="$(
GITEA_DEPLOY_KEY_TITLE="${title}" \
GITEA_PUBLIC_KEY="$(<"${public_key_path}")" \
GITEA_DEPLOY_KEY_READ_ONLY="${read_only}" \
python3 - <<'PY'
import json
import os
print(json.dumps({
"title": os.environ["GITEA_DEPLOY_KEY_TITLE"],
"key": os.environ["GITEA_PUBLIC_KEY"].strip(),
"read_only": os.environ["GITEA_DEPLOY_KEY_READ_ONLY"] == "true",
}))
PY
)"
curl -fsS \
-u "${auth_user}:${auth_password}" \
-H "Content-Type: application/json" \
-X POST \
-d "${payload}" \
"${api_base}/repos/${owner}/${repo_name}/keys" >/dev/null
}
ensure_gitea_repo_ssh_access() {
local api_base="$1"
local auth_user="$2"
local auth_password="$3"
local owner="$4"
local repo_name="$5"
local ssh_host="$6"
local ssh_port="$7"
local key_path="$8"
local key_title="$9"
local key_read_only="${10}"
local key_dir
local known_hosts
local public_key_path
local read_only_json="false"
local ssh_repo_url
if [[ "${key_path}" =~ [[:space:]] || "${key_path}" == *"'"* ]]; then
echo "LAB_GITEA_REPO_SSH_KEY_PATH cannot contain whitespace or single quotes." >&2
exit 1
fi
key_dir="$(dirname "${key_path}")"
public_key_path="${key_path}.pub"
mkdir -p "${key_dir}"
chmod 0700 "${key_dir}"
if [[ ! -s "${key_path}" && ! -s "${public_key_path}" ]]; then
ssh-keygen -t ed25519 -N "" -f "${key_path}" -C "${key_title}" >/dev/null
elif [[ -s "${key_path}" && ! -s "${public_key_path}" ]]; then
ssh-keygen -y -f "${key_path}" >"${public_key_path}"
elif [[ ! -s "${key_path}" ]]; then
echo "Public key ${public_key_path} exists, but private key ${key_path} is missing." >&2
exit 1
fi
chmod 0600 "${key_path}"
chmod 0644 "${public_key_path}"
if truthy "${key_read_only}"; then
read_only_json="true"
fi
if gitea_public_key_registered "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${public_key_path}"; then
echo "Gitea already has Debian host SSH key ${public_key_path}."
else
create_gitea_repo_deploy_key "${api_base}" "${auth_user}" "${auth_password}" "${owner}" "${repo_name}" "${key_title}" "${public_key_path}" "${read_only_json}"
echo "Added Debian host SSH key ${public_key_path} to ${owner}/${repo_name}."
fi
known_hosts="${HOME}/.ssh/known_hosts"
touch "${known_hosts}"
chmod 0644 "${known_hosts}"
if ! ssh-keygen -F "[${ssh_host}]:${ssh_port}" -f "${known_hosts}" >/dev/null 2>&1; then
ssh-keyscan -p "${ssh_port}" "${ssh_host}" >>"${known_hosts}" 2>/dev/null
fi
ssh_repo_url="ssh://git@${ssh_host}:${ssh_port}/${owner}/${repo_name}.git"
git -C "${REPO_ROOT}" remote set-url gitea "${ssh_repo_url}" 2>/dev/null ||
git -C "${REPO_ROOT}" remote add gitea "${ssh_repo_url}"
git -C "${REPO_ROOT}" config core.sshCommand "ssh -i ${key_path} -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new"
git -C "${REPO_ROOT}" ls-remote gitea HEAD >/dev/null
echo "Gitea SSH remote: ${ssh_repo_url}"
}
bootstrap_gitea_repo() {
local mode="${LAB_GITEA_REPO_BOOTSTRAP:-true}"
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
local gitea_user="${LAB_GITEA_USER:-jv}"
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local container_name="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
local http_port="${LAB_GITEA_HTTP_PORT:-3000}"
local ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
local root_url="${LAB_GITEA_ROOT_URL:-https://lab2025.duckdns.org/git/}"
local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}"
local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}"
local default_branch="${LAB_GITEA_REPO_DEFAULT_BRANCH:-main}"
local bootstrap_user="${LAB_GITEA_BOOTSTRAP_USER:-${repo_owner}}"
local bootstrap_email="${LAB_GITEA_BOOTSTRAP_EMAIL:-${bootstrap_user}@homelab.local}"
local credentials_file="${LAB_GITEA_BOOTSTRAP_CREDENTIALS_FILE:-${HOME}/.config/homelab/gitea-bootstrap.env}"
local bootstrap_password="${LAB_GITEA_BOOTSTRAP_PASSWORD:-}"
local allow_dirty="${LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY:-false}"
local ssh_bootstrap="${LAB_GITEA_REPO_SSH_BOOTSTRAP:-true}"
local ssh_key_path="${LAB_GITEA_REPO_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}"
local ssh_key_title="${LAB_GITEA_REPO_DEPLOY_KEY_TITLE:-debian-host-${repo_name}}"
local ssh_key_read_only="${LAB_GITEA_REPO_DEPLOY_KEY_READ_ONLY:-false}"
local api_base
local public_repo_url
local direct_repo_url
local push_url
local askpass
local credentials_dir
local remote_status
local worktree_status
require_debian_server "bootstrap-gitea-repo"
if disabled_value "${mode}"; then
return 0
fi
ensure_python3
for value_name in repo_owner repo_name default_branch bootstrap_user; do
local value="${!value_name}"
if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
echo "${value_name} contains unsupported characters." >&2
exit 1
fi
done
if [[ "${bootstrap_email}" == *"'"* ]]; then
echo "LAB_GITEA_BOOTSTRAP_EMAIL cannot contain a single quote." >&2
exit 1
fi
if ! [[ "${ssh_port}" =~ ^[0-9]+$ ]]; then
echo "LAB_GITEA_SSH_PORT must be numeric." >&2
exit 1
fi
if [[ -z "${bootstrap_password}" && -r "${credentials_file}" ]]; then
# shellcheck disable=SC1090
source "${credentials_file}"
bootstrap_user="${GITEA_BOOTSTRAP_USER:-${bootstrap_user}}"
bootstrap_email="${GITEA_BOOTSTRAP_EMAIL:-${bootstrap_email}}"
bootstrap_password="${GITEA_BOOTSTRAP_PASSWORD:-}"
fi
if [[ -z "${bootstrap_password}" ]]; then
bootstrap_password="$(gitea_bootstrap_password)"
credentials_dir="$(dirname "${credentials_file}")"
mkdir -p "${credentials_dir}"
chmod 0700 "${credentials_dir}"
{
printf "GITEA_BOOTSTRAP_USER='%s'\n" "${bootstrap_user}"
printf "GITEA_BOOTSTRAP_EMAIL='%s'\n" "${bootstrap_email}"
printf "GITEA_BOOTSTRAP_PASSWORD='%s'\n" "${bootstrap_password}"
} > "${credentials_file}"
chmod 0600 "${credentials_file}"
echo "Generated Gitea bootstrap credentials at ${credentials_file}."
fi
for value_name in repo_owner repo_name default_branch bootstrap_user; do
local value="${!value_name}"
if ! [[ "${value}" =~ ^[A-Za-z0-9_.-]+$ ]]; then
echo "${value_name} contains unsupported characters." >&2
exit 1
fi
done
for value_name in bootstrap_email bootstrap_password; do
local value="${!value_name}"
if [[ "${value}" == *"'"* ]]; then
echo "${value_name} cannot contain a single quote." >&2
exit 1
fi
done
echo "Bootstrapping Gitea repository ${repo_owner}/${repo_name}..."
# shellcheck disable=SC2087
ssh -i "${gitea_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${gitea_user}@${gitea_host}" "bash -s" <<EOF
set -euo pipefail
container_name='${container_name}'
bootstrap_user='${bootstrap_user}'
bootstrap_email='${bootstrap_email}'
bootstrap_password='${bootstrap_password}'
if ! sudo docker inspect "\${container_name}" >/dev/null 2>&1; then
echo "Gitea container \${container_name} is not running on ${gitea_host}." >&2
exit 1
fi
for attempt in \$(seq 1 60); do
if curl -fsS http://127.0.0.1:3000/api/v1/version >/dev/null 2>&1 ||
curl -fsS http://127.0.0.1:3000/git/api/v1/version >/dev/null 2>&1; then
break
fi
if [ "\${attempt}" = "60" ]; then
echo "Timed out waiting for Gitea API inside \${container_name}." >&2
exit 1
fi
sleep 2
done
if ! sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user create \
--username "\${bootstrap_user}" \
--password "\${bootstrap_password}" \
--email "\${bootstrap_email}" \
--admin \
--must-change-password=false >/tmp/homelab-gitea-user-create.log 2>&1; then
if sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user list | awk -v user="\${bootstrap_user}" 'NR > 1 && \$2 == user { found = 1 } END { exit found ? 0 : 1 }'; then
sudo docker exec -u git "\${container_name}" gitea -c /data/gitea/conf/app.ini admin user change-password \
--username "\${bootstrap_user}" \
--password "\${bootstrap_password}" >/tmp/homelab-gitea-user-password.log 2>&1 || {
cat /tmp/homelab-gitea-user-password.log >&2
exit 1
}
else
cat /tmp/homelab-gitea-user-create.log >&2
exit 1
fi
fi
EOF
api_base="$(gitea_api_base_url "${gitea_host}" "${http_port}")"
if gitea_repo_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}"; then
echo "Gitea repository ${repo_owner}/${repo_name} already exists."
else
if [[ "${repo_owner}" != "${bootstrap_user}" ]]; then
echo "Gitea repository owner ${repo_owner} does not exist yet; only user-owned bootstrap repos are supported." >&2
exit 1
fi
create_gitea_repo "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_name}" "${default_branch}"
echo "Created Gitea repository ${repo_owner}/${repo_name}."
fi
public_repo_url="${root_url%/}/${repo_owner}/${repo_name}.git"
if [[ "${api_base}" == */git/api/v1 ]]; then
direct_repo_url="http://${gitea_host}:${http_port}/git/${repo_owner}/${repo_name}.git"
else
direct_repo_url="http://${gitea_host}:${http_port}/${repo_owner}/${repo_name}.git"
fi
push_url="${LAB_GITEA_BOOTSTRAP_PUSH_URL:-${direct_repo_url}}"
git -C "${REPO_ROOT}" rev-parse --is-inside-work-tree >/dev/null
git -C "${REPO_ROOT}" remote set-url gitea "${public_repo_url}" 2>/dev/null ||
git -C "${REPO_ROOT}" remote add gitea "${public_repo_url}"
if gitea_branch_exists "${api_base}" "${bootstrap_user}" "${bootstrap_password}" "${repo_owner}" "${repo_name}" "${default_branch}"; then
echo "Gitea branch ${default_branch} already exists; leaving existing history unchanged."
else
worktree_status="$(git -C "${REPO_ROOT}" status --porcelain)"
if [[ -n "${worktree_status}" ]] && ! truthy "${allow_dirty}"; then
echo "Refusing to seed Gitea from a dirty working tree; commit or stash changes first." >&2
echo "Set LAB_GITEA_BOOTSTRAP_ALLOW_DIRTY=true to push committed HEAD anyway." >&2
exit 1
fi
askpass="$(mktemp)"
trap 'rm -f "${askpass}" "${BUILDX_CONFIG}"' EXIT
cat > "${askpass}" <<ASKPASS_EOT
#!/usr/bin/env bash
case "\$1" in
*Username*) printf '%s\n' '${bootstrap_user}' ;;
*Password*) printf '%s\n' '${bootstrap_password}' ;;
*) printf '\n' ;;
esac
ASKPASS_EOT
chmod 0700 "${askpass}"
GIT_ASKPASS="${askpass}" GIT_TERMINAL_PROMPT=0 \
git -C "${REPO_ROOT}" push "${push_url}" "HEAD:refs/heads/${default_branch}"
rm -f "${askpass}"
trap 'rm -f "${BUILDX_CONFIG}"' EXIT
echo "Pushed current HEAD to Gitea branch ${default_branch}."
fi
remote_status="$(git -C "${REPO_ROOT}" remote get-url gitea)"
echo "Gitea remote: ${remote_status}"
if ! disabled_value "${ssh_bootstrap}"; then
ensure_gitea_repo_ssh_access \
"${api_base}" \
"${bootstrap_user}" \
"${bootstrap_password}" \
"${repo_owner}" \
"${repo_name}" \
"${gitea_host}" \
"${ssh_port}" \
"${ssh_key_path}" \
"${ssh_key_title}" \
"${ssh_key_read_only}"
fi
}
install_gitea_backup_timer() {
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
local gitea_user="${LAB_GITEA_USER:-jv}"
local gitea_key="${LAB_GITEA_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local gitea_container="${LAB_GITEA_CONTAINER_NAME:-homelab-gitea}"
local backup_dir="${LAB_GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
local backup_script="/usr/local/sbin/homelab-gitea-backup.sh"
local restore_drill_script="/usr/local/sbin/homelab-gitea-restore-drill.sh"
sudo tee "${backup_script}" >/dev/null <<BACKUP_SCRIPT_EOT
#!/usr/bin/env bash
set -euo pipefail
GITEA_HOST="\${GITEA_HOST:-${gitea_host}}"
GITEA_USER="\${GITEA_USER:-${gitea_user}}"
GITEA_SSH_KEY_PATH="\${GITEA_SSH_KEY_PATH:-${gitea_key}}"
GITEA_CONTAINER="\${GITEA_CONTAINER:-${gitea_container}}"
GITEA_BACKUP_DIR="\${GITEA_BACKUP_DIR:-${backup_dir}}"
GITEA_BACKUP_RETENTION_DAYS="\${GITEA_BACKUP_RETENTION_DAYS:-30}"
REMOTE_ARCHIVE="/tmp/homelab-gitea-dump.zip"
timestamp="\$(date -u +%Y%m%dT%H%M%SZ)"
tmp_archive="\$(mktemp "/tmp/gitea-\${timestamp}.XXXXXX.zip")"
backup_archive="\${GITEA_BACKUP_DIR}/gitea-\${timestamp}.zip"
remote_host_archive="/tmp/gitea-\${timestamp}.zip"
ssh_gitea() {
ssh -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "\${GITEA_USER}@\${GITEA_HOST}" "\$@"
}
cleanup() {
rm -f "\${tmp_archive}"
ssh_gitea "rm -f '\${remote_host_archive}'; sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true" >/dev/null 2>&1 || true
}
trap cleanup EXIT
ssh_gitea "set -eu
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true
sudo docker exec -u git '\${GITEA_CONTAINER}' sh -c 'mkdir -p /data/git/repositories'
sudo docker exec -u git '\${GITEA_CONTAINER}' gitea dump -c /data/gitea/conf/app.ini --file '\${REMOTE_ARCHIVE}'
sudo docker cp '\${GITEA_CONTAINER}:\${REMOTE_ARCHIVE}' '\${remote_host_archive}'
sudo chown '\${GITEA_USER}:\${GITEA_USER}' '\${remote_host_archive}'
sudo docker exec -u git '\${GITEA_CONTAINER}' rm -f '\${REMOTE_ARCHIVE}' >/dev/null 2>&1 || true"
scp -i "\${GITEA_SSH_KEY_PATH}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
"\${GITEA_USER}@\${GITEA_HOST}:\${remote_host_archive}" "\${tmp_archive}"
sudo mkdir -p "\${GITEA_BACKUP_DIR}"
sudo chown jv:jv "\${GITEA_BACKUP_DIR}"
sudo install -m 0640 -o jv -g jv "\${tmp_archive}" "\${backup_archive}"
sudo find "\${GITEA_BACKUP_DIR}" -type f -name 'gitea-*.zip' -mtime +"\${GITEA_BACKUP_RETENTION_DAYS}" -delete
echo "Created \${backup_archive}"
BACKUP_SCRIPT_EOT
sudo chmod 0755 "${backup_script}"
sudo tee /etc/systemd/system/homelab-gitea-backup.service >/dev/null <<'SERVICE_EOT'
[Unit]
Description=Back up external Homelab Gitea to Debian host storage
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-gitea-backup.sh
SERVICE_EOT
sudo tee /etc/systemd/system/homelab-gitea-backup.timer >/dev/null <<'TIMER_EOT'
[Unit]
Description=Run daily Homelab Gitea backups
[Timer]
OnCalendar=*-*-* 02:35:00
RandomizedDelaySec=20m
Persistent=true
[Install]
WantedBy=timers.target
TIMER_EOT
sudo tee "${restore_drill_script}" >/dev/null <<'RESTORE_DRILL_SCRIPT_EOT'
#!/usr/bin/env bash
set -euo pipefail
GITEA_BACKUP_DIR="${GITEA_BACKUP_DIR:-/home/jv/backups/gitea}"
GITEA_RESTORE_DRILL_DIR="${GITEA_RESTORE_DRILL_DIR:-/home/jv/backups/gitea-restore-drills}"
GITEA_RESTORE_DRILL_RETENTION_DAYS="${GITEA_RESTORE_DRILL_RETENTION_DAYS:-90}"
if ! command -v python3 >/dev/null 2>&1; then
echo "python3 is required for Gitea restore drills." >&2
exit 1
fi
latest_archive="$(
{ find "${GITEA_BACKUP_DIR}" -maxdepth 1 -type f -name 'gitea-*.zip' -printf '%T@ %p\n' 2>/dev/null || true; } |
sort -nr |
awk 'NR == 1 { sub(/^[^ ]+ /, ""); print }'
)"
if [[ -z "${latest_archive}" ]]; then
echo "Skipping Gitea restore drill: no backup archive found in ${GITEA_BACKUP_DIR}."
exit 0
fi
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
tmp_dir="$(mktemp -d "/tmp/gitea-restore-drill-${timestamp}.XXXXXX")"
tmp_report="$(mktemp "/tmp/gitea-restore-drill-${timestamp}.XXXXXX.txt")"
report_path="${GITEA_RESTORE_DRILL_DIR}/gitea-restore-drill-${timestamp}.txt"
cleanup() {
rm -rf "${tmp_dir}"
rm -f "${tmp_report}"
}
trap cleanup EXIT
python3 - "${latest_archive}" "${tmp_dir}" "${tmp_report}" <<'PY'
import os
import sys
import zipfile
archive_path, extract_dir, report_path = sys.argv[1:4]
with zipfile.ZipFile(archive_path) as archive:
bad_member = archive.testzip()
if bad_member:
raise SystemExit(f"ZIP integrity check failed at {bad_member}")
members = archive.infolist()
if not members:
raise SystemExit("ZIP archive is empty")
extract_root = os.path.abspath(extract_dir)
for member in members:
target = os.path.abspath(os.path.join(extract_root, member.filename))
if target != extract_root and not target.startswith(extract_root + os.sep):
raise SystemExit(f"Unsafe archive path: {member.filename}")
archive.extractall(extract_root)
file_count = 0
total_bytes = 0
for root, _, files in os.walk(extract_dir):
for name in files:
file_count += 1
total_bytes += os.path.getsize(os.path.join(root, name))
if file_count == 0:
raise SystemExit("Archive extracted no files")
with open(report_path, "w", encoding="utf-8") as handle:
handle.write("Gitea restore drill report\n")
handle.write(f"archive={archive_path}\n")
handle.write(f"archive_size_bytes={os.path.getsize(archive_path)}\n")
handle.write(f"extracted_files={file_count}\n")
handle.write(f"extracted_bytes={total_bytes}\n")
handle.write("result=ok\n")
PY
sudo mkdir -p "${GITEA_RESTORE_DRILL_DIR}"
sudo install -m 0640 -o root -g root "${tmp_report}" "${report_path}"
sudo find "${GITEA_RESTORE_DRILL_DIR}" -type f -name 'gitea-restore-drill-*.txt' -mtime +"${GITEA_RESTORE_DRILL_RETENTION_DAYS}" -delete
echo "Created ${report_path}"
RESTORE_DRILL_SCRIPT_EOT
sudo chmod 0755 "${restore_drill_script}"
sudo tee /etc/systemd/system/homelab-gitea-restore-drill.service >/dev/null <<'RESTORE_DRILL_SERVICE_EOT'
[Unit]
Description=Run a non-destructive Gitea backup restore drill
After=network-online.target homelab-gitea-backup.service
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/sbin/homelab-gitea-restore-drill.sh
RESTORE_DRILL_SERVICE_EOT
sudo tee /etc/systemd/system/homelab-gitea-restore-drill.timer >/dev/null <<'RESTORE_DRILL_TIMER_EOT'
[Unit]
Description=Run monthly Homelab Gitea restore drills
[Timer]
OnCalendar=monthly
RandomizedDelaySec=2h
Persistent=true
[Install]
WantedBy=timers.target
RESTORE_DRILL_TIMER_EOT
sudo systemctl daemon-reload
sudo systemctl enable --now homelab-gitea-backup.timer >/dev/null
sudo systemctl enable --now homelab-gitea-restore-drill.timer >/dev/null
}
backup_gitea() {
require_debian_server "backup-gitea"
install_gitea_backup_timer
sudo /usr/local/sbin/homelab-gitea-backup.sh
}
drill_gitea_restore() {
require_debian_server "drill-gitea-restore"
install_gitea_backup_timer
sudo /usr/local/sbin/homelab-gitea-restore-drill.sh
}
install_gitea_runner() {
local runner_arch
local runner_home="${GITEA_RUNNER_HOME:-/home/jv/.local/share/gitea-runner/my-homelab-configs}"
local runner_instance="${GITEA_RUNNER_INSTANCE_URL:-https://lab2025.duckdns.org/git/}"
local runner_labels="${GITEA_RUNNER_LABELS:-homelab-debian:host}"
local runner_name="${GITEA_RUNNER_NAME:-homelab-debian-my-homelab-configs}"
local runner_token="${GITEA_RUNNER_REGISTRATION_TOKEN:-${1:-}}"
local runner_user="${GITEA_RUNNER_USER:-jv}"
local runner_version="${GITEA_ACT_RUNNER_VERSION:-0.2.11}"
local missing_packages=()
require_debian_server "install-gitea-runner"
case "$(dpkg --print-architecture)" in
amd64)
runner_arch="linux-amd64"
;;
arm64)
runner_arch="linux-arm64"
;;
*)
echo "Unsupported Debian architecture: $(dpkg --print-architecture)" >&2
exit 1
;;
esac
for package in ca-certificates curl git nodejs python3; do
if ! dpkg-query -W -f='${Status}' "$package" 2>/dev/null | grep -q "install ok installed"; then
missing_packages+=("$package")
fi
done
if [[ ${#missing_packages[@]} -gt 0 ]]; then
sudo apt-get update
sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
fi
sudo curl -fsSL \
-o /usr/local/bin/act_runner \
"https://gitea.com/gitea/act_runner/releases/download/v${runner_version}/act_runner-${runner_version}-${runner_arch}"
sudo chmod 0755 /usr/local/bin/act_runner
sudo chown root:root /usr/local/bin/act_runner
sudo -u "${runner_user}" mkdir -p "${runner_home}"
if [[ ! -f "${runner_home}/.runner" ]]; then
if [[ -z "${runner_token}" ]]; then
echo "Set GITEA_RUNNER_REGISTRATION_TOKEN to the repository-level runner token from Gitea." >&2
exit 1
fi
sudo -u "${runner_user}" env \
HOME="/home/${runner_user}" \
GITEA_RUNNER_HOME="${runner_home}" \
GITEA_RUNNER_INSTANCE_URL="${runner_instance}" \
GITEA_RUNNER_REGISTRATION_TOKEN="${runner_token}" \
GITEA_RUNNER_NAME="${runner_name}" \
GITEA_RUNNER_LABELS="${runner_labels}" \
bash -lc 'cd "${GITEA_RUNNER_HOME}" && /usr/local/bin/act_runner register --no-interactive --instance "${GITEA_RUNNER_INSTANCE_URL}" --token "${GITEA_RUNNER_REGISTRATION_TOKEN}" --name "${GITEA_RUNNER_NAME}" --labels "${GITEA_RUNNER_LABELS}"'
else
echo "Existing runner registration found at ${runner_home}/.runner; keeping it."
fi
sudo tee /etc/systemd/system/homelab-gitea-runner.service >/dev/null <<SERVICE_EOT
[Unit]
Description=Homelab Gitea Actions runner for my-homelab-configs
After=network-online.target docker.service
Wants=network-online.target
[Service]
Type=simple
User=${runner_user}
Group=${runner_user}
WorkingDirectory=${runner_home}
Environment=HOME=/home/${runner_user}
Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ExecStart=/usr/local/bin/act_runner daemon
Restart=always
RestartSec=5
[Install]
WantedBy=multi-user.target
SERVICE_EOT
sudo systemctl daemon-reload
sudo systemctl enable --now homelab-gitea-runner.service >/dev/null
sudo systemctl status homelab-gitea-runner.service --no-pager -l
}
recreate_pods_for_selector() {
local namespace="$1"
local selector="$2"
local app="$3"
if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" delete pod -l "${selector}" --ignore-not-found --wait=true --timeout=120s; then
echo "Failed to recreate pods matching ${selector} in namespace ${namespace}" >&2
dump_argocd_debug "${app}"
dump_namespace_debug "${namespace}"
exit 1
fi
}
refresh_argocd_application() {
local app="$1"
kubectl --kubeconfig "${KUBECONFIG}" patch application "${app}" -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"hard"}}}' >/dev/null
}
website_translation_model() {
local model_name="${WEBSITE_TRANSLATION_MODEL:-website-translator}"
local modelfile="${WEBSITE_TRANSLATION_MODELFILE:-${REPO_ROOT}/apps/website/ollama/Modelfile}"
require_debian_server "website-translation-model"
if ! command -v ollama >/dev/null 2>&1; then
echo "ollama is not installed or not in PATH on this Debian server." >&2
exit 1
fi
if [[ ! -r "${modelfile}" ]]; then
echo "Ollama Modelfile is not readable: ${modelfile}" >&2
exit 1
fi
echo "Creating or updating Ollama model ${model_name} from ${modelfile}..."
ollama create "${model_name}" -f "${modelfile}"
if ! ollama list | awk -v model="${model_name}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
echo "Ollama model ${model_name} was not found after creation." >&2
exit 1
fi
echo "Ollama model ${model_name} is ready."
}
website_ollama_listen() {
local bind_address="${WEBSITE_OLLAMA_BIND_ADDRESS:-${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}}"
local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
local igpu_enable="${LAB_OLLAMA_IGPU_ENABLE:-false}"
local dropin_dir="/etc/systemd/system/ollama.service.d"
local dropin_file="${dropin_dir}/homelab.conf"
local waited=0
require_debian_server "website-ollama-listen"
bind_address="${bind_address#http://}"
bind_address="${bind_address#https://}"
if ! systemctl cat ollama.service >/dev/null 2>&1; then
echo "ollama.service was not found on this Debian server." >&2
exit 1
fi
echo "Configuring ollama.service to listen on ${bind_address}..."
sudo mkdir -p "${dropin_dir}"
sudo mkdir -p "${models_dir}"
if id ollama >/dev/null 2>&1; then
sudo chown ollama:ollama "${models_dir}"
fi
printf '[Service]\nEnvironment="OLLAMA_HOST=%s"\nEnvironment="OLLAMA_MODELS=%s"\nEnvironment="OLLAMA_IGPU_ENABLE=%s"\n' "${bind_address}" "${models_dir}" "${igpu_enable}" |
sudo tee "${dropin_file}" >/dev/null
sudo systemctl daemon-reload
sudo systemctl restart ollama.service
sudo systemctl is-active --quiet ollama.service
echo "Ollama service is active. Waiting for http://127.0.0.1:11434/api/tags..."
until curl -fsS --connect-timeout 2 --max-time 5 "http://127.0.0.1:11434/api/tags" >/dev/null 2>&1; do
waited=$((waited + 2))
if ((waited >= 60)); then
echo "Ollama service did not expose the local API after ${waited}s." >&2
echo "Recent service logs:" >&2
sudo journalctl -u ollama.service -n 80 --no-pager >&2 || true
exit 1
fi
sleep 2
done
echo "Ollama API is reachable on the Debian host."
}
ollama_linux_arch() {
case "$(uname -m)" in
x86_64 | amd64)
printf 'amd64\n'
;;
aarch64 | arm64)
printf 'arm64\n'
;;
*)
echo "Unsupported Ollama Linux architecture: $(uname -m)" >&2
return 1
;;
esac
}
ollama_write_systemd_service() {
sudo tee /etc/systemd/system/ollama.service >/dev/null <<'EOF'
[Unit]
Description=Ollama Service
After=network-online.target
[Service]
ExecStart=/usr/bin/ollama serve
User=ollama
Group=ollama
Restart=always
RestartSec=3
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
[Install]
WantedBy=default.target
EOF
}
ollama_install_from_tgz() {
local tarball="$1"
if [[ ! -r "${tarball}" ]]; then
echo "Ollama tarball is not readable: ${tarball}" >&2
exit 1
fi
echo "Installing Ollama from local tarball ${tarball}..."
sudo tar -C /usr -xzf "${tarball}"
if ! id ollama >/dev/null 2>&1; then
sudo useradd -r -s /bin/false -U -m -d /usr/share/ollama ollama
fi
if getent group render >/dev/null 2>&1; then
sudo usermod -aG render ollama
fi
if getent group video >/dev/null 2>&1; then
sudo usermod -aG video ollama
fi
ollama_write_systemd_service
}
ollama_setup() {
local bind_address="${LAB_OLLAMA_BIND_ADDRESS:-0.0.0.0:11434}"
local models_dir="${LAB_OLLAMA_MODELS_DIR:-/data/ollama/models}"
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
local installer_url="${LAB_OLLAMA_INSTALL_URL:-https://ollama.com/install.sh}"
local connect_timeout="${LAB_OLLAMA_INSTALL_CONNECT_TIMEOUT:-15}"
local max_time="${LAB_OLLAMA_INSTALL_MAX_TIME:-120}"
local curl_ip_version="${LAB_OLLAMA_INSTALL_CURL_IP_VERSION:--4}"
local local_tarball="${LAB_OLLAMA_TARBALL_FILE:-}"
local arch
local tmp_installer
local curl_ip_args=()
require_debian_server "ollama-setup"
if ! command -v curl >/dev/null 2>&1; then
echo "curl is required to install Ollama." >&2
exit 1
fi
if [[ -n "${curl_ip_version}" ]]; then
curl_ip_args+=("${curl_ip_version}")
fi
if ! command -v ollama >/dev/null 2>&1; then
if [[ -n "${local_tarball}" ]]; then
ollama_install_from_tgz "${local_tarball}"
else
echo "Installing Ollama from ${installer_url}..."
tmp_installer="$(mktemp)"
if ! curl "${curl_ip_args[@]}" -fsSL --connect-timeout "${connect_timeout}" --max-time "${max_time}" "${installer_url}" -o "${tmp_installer}"; then
rm -f "${tmp_installer}"
arch="$(ollama_linux_arch || true)"
cat >&2 <<EOF
Could not download the Ollama installer from ${installer_url}.
If this Debian host cannot reach ollama.com, download the Linux tarball from a
machine with internet access and copy it to the Debian host:
curl -L https://ollama.com/download/ollama-linux-${arch:-amd64}.tgz -o /tmp/ollama-linux-${arch:-amd64}.tgz
scp /tmp/ollama-linux-${arch:-amd64}.tgz jv@${LAB_DEBIAN_LAN_IP:-192.168.100.73}:/tmp/
Then run on Debian:
LAB_OLLAMA_TARBALL_FILE=/tmp/ollama-linux-${arch:-amd64}.tgz ./jeannie ollama-setup
EOF
exit 1
fi
sudo sh "${tmp_installer}"
rm -f "${tmp_installer}"
fi
else
echo "Ollama is already installed: $(command -v ollama)"
fi
website_ollama_listen
echo "Ensuring Ollama model ${model} is available..."
if ollama list | awk -v model="${model}" 'NR > 1 && ($1 == model || $1 == model ":latest") { found = 1 } END { exit found ? 0 : 1 }'; then
echo "Ollama model ${model} is already present."
else
OLLAMA_HOST=http://127.0.0.1:11434 ollama pull "${model}"
fi
echo "Ollama homelab setup is ready."
}
apps() {
local buildx_builder_ready=false
local demos_image_built=false
local demos_image_ref
local demos_image_state_file
local demos_platforms
local demos_registry_endpoint
local demos_source_hash
local registry_endpoint
local website_image_built=false
local website_image_ref
local website_image_state_file
local website_platforms
local website_source_hash
require_debian_server "apps"
registry_endpoint="$(apps_registry_endpoint)"
demos_registry_endpoint="$(demos_registry_endpoint)"
demos_image_ref="${registry_endpoint}/demos-static:latest"
demos_image_state_file="${REPO_ROOT}/.lab/demos-static-image.state"
demos_platforms="${DEMOS_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
demos_source_hash="$(demos_source_hash)"
website_image_state_file="${REPO_ROOT}/.lab/php-website-image.state"
website_platforms="${WEBSITE_IMAGE_PLATFORMS:-linux/amd64,linux/arm64}"
website_source_hash="$(website_source_hash)"
website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}"
export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}"
export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}"
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
export KUBECONFIG="${TF_VAR_kubeconfig_path}"
ensure_cosign_keypair
publish_cosign_public_key_config_map
if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then
echo "TF_VAR_registry_endpoint changed after registry endpoint resolution (${registry_endpoint})" >&2
exit 1
fi
if [[ "${TF_VAR_website_image_ref}" != "${website_image_ref}" ]]; then
echo "TF_VAR_website_image_ref must match the buildx website image ref (${website_image_ref})" >&2
exit 1
fi
if [[ "${demos_registry_endpoint}" != "${registry_endpoint}" ]]; then
echo "apps/demos-static/web-app.yaml registry endpoint (${demos_registry_endpoint}) must match app registry endpoint (${registry_endpoint})" >&2
exit 1
fi
if [[ "$(image_ref_tag "${website_image_ref}")" == "latest" ]]; then
echo "apps/website/web-app.yaml must use an immutable php-website image tag, not latest." >&2
exit 1
fi
ensure_website_image_tag_not_reused "${website_image_state_file}" "${website_source_hash}" "${website_image_ref}"
echo "Deploying homelab applications..."
run_tofu_stack "bootstrap/apps"
refresh_argocd_application container-registry
refresh_argocd_application supply-chain-policy
wait_for_cluster_resource imagevalidatingpolicy.policies.kyverno.io homelab-local-registry-supply-chain supply-chain-policy 300
wait_for_namespace container-registry container-registry 300
wait_for_namespaced_resource container-registry deployment local-registry container-registry 300
wait_for_deployment_ready container-registry local-registry container-registry 300
if website_image_is_current "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}" "${registry_endpoint}" &&
image_supply_chain_metadata_exists "${website_image_ref}" "${registry_endpoint}"; then
echo "Website image ${website_image_ref} is already current (${website_source_hash}); skipping build."
else
echo "Building website image ${website_image_ref} for ${website_platforms} (${website_source_hash})..."
ensure_docker_build_space
if [[ "${buildx_builder_ready}" != "true" ]]; then
prepare_buildx_builder "${registry_endpoint}"
buildx_builder_ready=true
fi
docker buildx build \
--network host \
--platform "${website_platforms}" \
--provenance=false \
--sbom=true \
--label "dev.homelab.website.source-hash=${website_source_hash}" \
-t "${website_image_ref}" \
-f "${REPO_ROOT}/apps/website/Dockerfile" \
"${REPO_ROOT}/apps/website/" \
--push
website_image_built=true
fi
publish_image_supply_chain_metadata "${website_image_ref}" "${registry_endpoint}" "${website_source_hash}"
if demos_image_is_current "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}" "${registry_endpoint}" &&
image_supply_chain_metadata_exists "${demos_image_ref}" "${registry_endpoint}"; then
echo "Demos image ${demos_image_ref} is already current (${demos_source_hash}); skipping build."
else
echo "Building demos image ${demos_image_ref} for ${demos_platforms} (${demos_source_hash})..."
ensure_docker_build_space
if [[ "${buildx_builder_ready}" != "true" ]]; then
prepare_buildx_builder "${registry_endpoint}"
buildx_builder_ready=true
fi
docker buildx build \
--network host \
--platform "${demos_platforms}" \
--provenance=false \
--sbom=true \
--label "dev.homelab.demos.source-hash=${demos_source_hash}" \
-t "${demos_image_ref}" \
-f "${REPO_ROOT}/apps/demos-static/Dockerfile" \
"${REPO_ROOT}/apps/demos-static/" \
--push
demos_image_built=true
fi
publish_image_supply_chain_metadata "${demos_image_ref}" "${registry_endpoint}" "${demos_source_hash}"
refresh_argocd_application website-production
wait_for_namespace website-production website-production 300
wait_for_namespaced_resource website-production deployment php-website-deployment website-production 300
if [[ "${website_image_built}" == "true" ]]; then
recreate_pods_for_selector website-production app=php-website website-production
else
echo "Skipping website pod restart because the image did not change."
fi
wait_for_deployment_ready website-production php-website-deployment website-production 300
if [[ "${website_image_built}" == "true" ]]; then
write_website_image_state "${website_image_state_file}" "${website_source_hash}" "${website_platforms}" "${website_image_ref}"
fi
refresh_argocd_application demos-static
wait_for_namespace demos-static demos-static 300
wait_for_namespaced_resource demos-static deployment demos-static demos-static 300
if [[ "${demos_image_built}" == "true" ]]; then
recreate_pods_for_selector demos-static app=demos-static demos-static
else
echo "Skipping demos pod restart because the image did not change."
fi
wait_for_deployment_ready demos-static demos-static demos-static 300
if [[ "${demos_image_built}" == "true" ]]; then
write_demos_image_state "${demos_image_state_file}" "${demos_source_hash}" "${demos_platforms}" "${demos_image_ref}"
fi
refresh_argocd_application heimdall
wait_for_namespace heimdall heimdall 300
wait_for_namespaced_resource heimdall deployment heimdall heimdall 300
wait_for_deployment_ready heimdall heimdall heimdall 300
refresh_argocd_application n8n
wait_for_namespace n8n n8n 300
wait_for_namespaced_resource n8n deployment n8n n8n 300
wait_for_deployment_ready n8n n8n n8n 300
echo "Application deployment successfully completed."
}
ensure_cluster_worker_var_file() {
if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
prepare_cluster_worker_var_file true
fi
}
up() {
require_debian_server "up"
echo "Deploying the homelab infrastructure..."
jeannie_log_start "up"
jeannie_step_plan 15
run_step "Early preflight" homelab_preflight early
run_step "Gitea deploy" deploy_gitea
run_step "Gitea repo bootstrap" bootstrap_gitea_repo
run_step "RPi services" deploy_rpi_services
run_step "Full preflight" homelab_preflight full
run_step "Pre-apply doctor" doctor_preapply
run_step "Pimox provisioning and workers" run_pimox_pipeline
run_step "OpenWrt VM" run_openwrt_pipeline
run_step "Worker var file" ensure_cluster_worker_var_file
run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
run_step "Version report" doctor_versions_report
run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
run_step "Applications" apps
run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"
echo "Deployment successfully completed."
echo "Log: ${JEANNIE_LOG_FILE}"
}
tofu_stack_from_plan_target() {
case "$1" in
provisioning)
printf 'bootstrap/provisioning\n'
;;
cluster)
printf 'bootstrap/cluster\n'
;;
platform)
printf 'bootstrap/platform\n'
;;
apps)
printf 'bootstrap/apps\n'
;;
edge)
printf 'bootstrap/edge\n'
;;
*)
echo "Unknown plan target '$1'. Use all, provisioning, cluster, platform, apps, or edge." >&2
return 1
;;
esac
}
plan_homelab() {
local target="${1:-all}"
local stack
local -a stacks=()
require_debian_server "plan"
if [[ "${target}" == "all" ]]; then
if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
prepare_cluster_worker_var_file true
fi
stacks=(
"bootstrap/provisioning"
"bootstrap/cluster"
"bootstrap/platform"
"bootstrap/apps"
"bootstrap/edge"
)
else
stack="$(tofu_stack_from_plan_target "${target}")"
if [[ "${stack}" == "bootstrap/cluster" && -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
prepare_cluster_worker_var_file true
fi
stacks=("${stack}")
fi
for stack in "${stacks[@]}"; do
echo
echo "Planning ${stack}..."
run_tofu_plan_stack "${stack}"
done
}
rebuild_cluster() {
require_debian_server "rebuild-cluster"
export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}"
export LAB_INCLUDE_RASPBERRY_WORKER="${LAB_INCLUDE_RASPBERRY_WORKER:-true}"
export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}"
export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}"
export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}"
export TF_VAR_force_worker_rejoin="${TF_VAR_force_worker_rejoin:-true}"
echo "Rebuilding the Kubernetes cluster without touching external Gitea..."
jeannie_log_start "rebuild-cluster"
jeannie_step_plan 10
run_step "Preflight" homelab_preflight
run_step "Nuke existing cluster state" nuke_for_rebuild
run_step "Pimox provisioning and workers" run_pimox_pipeline
run_step "OpenWrt VM" run_openwrt_pipeline
run_step "Worker var file" ensure_cluster_worker_var_file
run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
run_step "Version report" doctor_versions_report
run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
run_step "Applications" apps
run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"
echo "Cluster rebuild successfully completed."
echo "Log: ${JEANNIE_LOG_FILE}"
}
nuke_for_rebuild() {
LAB_NUKE_SKIP_CONFIRM=true nuke
}
cluster_worker_targets() {
local worker_ssh_targets="${WORKER_SSH_TARGETS-}"
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
CLUSTER_WORKER_TARGETS=()
read -r -a CLUSTER_WORKER_TARGETS <<< "${worker_ssh_targets}"
if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then
while IFS= read -r target; do
[[ -n "${target}" ]] || continue
if [[ ! " ${CLUSTER_WORKER_TARGETS[*]} " =~ [[:space:]]${target}[[:space:]] ]]; then
CLUSTER_WORKER_TARGETS+=("${target}")
fi
done < <(python3 - "${var_file}" "${worker_key_prefix}" <<'PY'
import json
import sys
var_file, worker_key_prefix = sys.argv[1:3]
with open(var_file, encoding="utf-8") as handle:
document = json.load(handle)
for key, node in sorted((document.get("worker_nodes") or {}).items()):
if key.startswith(worker_key_prefix):
continue
host = node.get("host")
user = node.get("user")
if host and user:
print(f"{user}@{host}")
PY
)
fi
}
stop_local_kubernetes_services() {
echo "--> Stopping local Kubernetes services on the Debian control plane..."
sudo systemctl stop kubelet 2>/dev/null || true
if command -v crictl >/dev/null 2>&1; then
sudo crictl stop --all 2>/dev/null || true
fi
if truthy "${LAB_CLUSTER_STOP_CONTAINERD:-false}"; then
sudo systemctl stop containerd 2>/dev/null || true
sudo systemctl reset-failed containerd 2>/dev/null || true
fi
sudo systemctl reset-failed kubelet 2>/dev/null || true
}
start_local_kubernetes_services() {
echo "--> Starting local Kubernetes services on the Debian control plane..."
sudo systemctl start containerd
sudo systemctl start docker 2>/dev/null || true
sudo systemctl start kubelet
}
stop_remote_kubernetes_services() {
local target="$1"
echo "--> Stopping Kubernetes services on remote worker ${target}..."
ssh -o ConnectTimeout=5 "${target}" \
"sudo systemctl stop kubelet 2>/dev/null || true; if command -v crictl >/dev/null 2>&1; then sudo crictl stop --all 2>/dev/null || true; fi; if [ '${LAB_CLUSTER_STOP_CONTAINERD:-false}' = 'true' ]; then sudo systemctl stop containerd 2>/dev/null || true; sudo systemctl reset-failed containerd 2>/dev/null || true; fi; sudo systemctl reset-failed kubelet 2>/dev/null || true"
}
start_remote_kubernetes_services() {
local target="$1"
echo "--> Starting Kubernetes services on remote worker ${target}..."
ssh -o ConnectTimeout=5 "${target}" \
"sudo systemctl start containerd && sudo systemctl start kubelet"
}
pimox_worker_count_default() {
local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
local var_file="${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
local worker_node_prefix="${LAB_PIMOX_WORKER_NODE_PREFIX:-pimox-worker}"
local max_count=0
local count
local state_count
if [[ -s "${spec_file}" ]]; then
count="$(awk -F'\t' -v key_prefix="${worker_key_prefix}" -v node_prefix="${worker_node_prefix}-" '
$1 ~ "^" key_prefix "[0-9]+$" {
index = substr($1, length(key_prefix) + 1) + 0
if (index > max) max = index
}
$4 ~ "^" node_prefix "[0-9]+$" {
index = substr($4, length(node_prefix) + 1) + 0
if (index > max) max = index
}
END { print max + 0 }
' "${spec_file}")"
if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
max_count="${count}"
fi
fi
if [[ -s "${var_file}" ]] && command -v python3 >/dev/null 2>&1; then
count="$(python3 - "${var_file}" "${worker_key_prefix}" "${worker_node_prefix}" <<'PY'
import json
import re
import sys
var_file, worker_key_prefix, worker_node_prefix = sys.argv[1:4]
with open(var_file, encoding="utf-8") as handle:
document = json.load(handle)
nodes = document.get("worker_nodes") or {}
highest = 0
key_pattern = re.compile(rf"^{re.escape(worker_key_prefix)}(\d+)$")
node_pattern = re.compile(rf"^{re.escape(worker_node_prefix)}-(\d+)$")
for key, node in nodes.items():
for candidate in (key, str(node.get("node_name", ""))):
match = key_pattern.match(candidate) or node_pattern.match(candidate)
if match:
highest = max(highest, int(match.group(1)))
print(highest)
PY
)"
if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
max_count="${count}"
fi
fi
state_count="$(tofu -chdir="${REPO_ROOT}/bootstrap/cluster" state show null_resource.worker_nodes_required 2>/dev/null |
awk -F'"' '/"worker_count"[[:space:]]*=/ { print $4; found = 1 } END { exit found ? 0 : 1 }' || true)"
if [[ "${state_count}" =~ ^[0-9]+$ && "${state_count}" -gt "${max_count}" ]]; then
max_count="${state_count}"
fi
count="${LAB_PIMOX_DEFAULT_WORKER_COUNT:-1}"
if [[ "${count}" =~ ^[0-9]+$ && "${count}" -gt "${max_count}" ]]; then
max_count="${count}"
fi
printf '%s\n' "${max_count}"
}
stop_pimox_worker_vms() {
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local shutdown_timeout="${LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS:-120}"
local force_stop="${LAB_CLUSTER_STOP_FORCE:-true}"
local index
local vmid
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
exit 1
fi
if ! [[ "${shutdown_timeout}" =~ ^[0-9]+$ ]]; then
echo "LAB_CLUSTER_STOP_VM_TIMEOUT_SECONDS must be an integer, got '${shutdown_timeout}'." >&2
exit 1
fi
if ((worker_count == 0)); then
return 0
fi
echo "--> Stopping Pimox worker VMs on ${pimox_host}..."
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
continue
fi
vmid=$((worker_base_vmid + index - 1))
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then
echo 'Pimox worker VM ${vmid} does not exist; skipping.'
exit 0
fi
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
echo 'Pimox worker VM ${vmid} is already stopped.'
exit 0
fi
echo 'Gracefully shutting down Pimox worker VM ${vmid}...'
if sudo '${qm_bin}' shutdown '${vmid}' --timeout '${shutdown_timeout}'; then
exit 0
fi
if [ '${force_stop}' = 'true' ]; then
echo 'Graceful shutdown timed out; forcing stop for Pimox worker VM ${vmid}.'
sudo '${qm_bin}' stop '${vmid}'
else
echo 'Graceful shutdown timed out for Pimox worker VM ${vmid}. Set LAB_CLUSTER_STOP_FORCE=true to force stop.' >&2
exit 1
fi"
done
}
destroy_pimox_worker_vms() {
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local index
local vmid
if ! truthy "${LAB_NUKE_DESTROY_PIMOX_WORKERS:-true}"; then
echo "--> Leaving Pimox worker VMs intact because LAB_NUKE_DESTROY_PIMOX_WORKERS=${LAB_NUKE_DESTROY_PIMOX_WORKERS}."
return 0
fi
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
exit 1
fi
if ((worker_count == 0)); then
return 0
fi
echo "--> Destroying Pimox worker VMs on ${pimox_host}..."
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
continue
fi
vmid=$((worker_base_vmid + index - 1))
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then
echo 'Pimox worker VM ${vmid} does not exist; skipping.'
exit 0
fi
if sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'; then
echo 'Pimox VM ${vmid} is a template; refusing to destroy it as a worker.' >&2
exit 1
fi
sudo '${qm_bin}' stop '${vmid}' >/dev/null 2>&1 || true
sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
done
}
start_pimox_worker_vms() {
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local index
local vmid
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
echo "LAB_PIMOX_WORKER_COUNT must be an integer, got '${worker_count}'." >&2
exit 1
fi
if ((worker_count == 0)); then
return 0
fi
echo "--> Starting Pimox worker VMs on ${pimox_host}..."
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
echo "Skipping Pimox worker index ${index} because LAB_PIMOX_SKIP_WORKER_INDEXES=${worker_skip_indexes}."
continue
fi
vmid=$((worker_base_vmid + index - 1))
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
if ! sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1; then
echo 'Pimox worker VM ${vmid} does not exist; skipping.'
exit 0
fi
if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then
echo 'Pimox worker VM ${vmid} is already running.'
exit 0
fi
echo 'Starting Pimox worker VM ${vmid}...'
sudo '${qm_bin}' start '${vmid}'"
done
}
stop_cluster() {
local target
declare -a CLUSTER_WORKER_TARGETS=()
require_debian_server "stop-cluster"
cluster_worker_targets
echo "Stopping Kubernetes cluster runtime without destroying state..."
for target in "${CLUSTER_WORKER_TARGETS[@]}"; do
stop_remote_kubernetes_services "${target}"
done
stop_pimox_worker_vms
stop_local_kubernetes_services
echo "Kubernetes runtime stopped. OpenTofu state, kubeadm files, PV data, and VM disks were left intact."
}
start_cluster() {
local target
declare -a CLUSTER_WORKER_TARGETS=()
require_debian_server "start-cluster"
cluster_worker_targets
echo "Starting Kubernetes cluster runtime without rebuilding state..."
start_local_kubernetes_services
start_pimox_worker_vms
for target in "${CLUSTER_WORKER_TARGETS[@]}"; do
start_remote_kubernetes_services "${target}"
done
echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness."
}
kubernetes_api_reachable() {
kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1
}
cluster_control_plane_tracked() {
tofu_state_has_resource "bootstrap/cluster" "null_resource.kubeadm_control_plane"
}
wait_for_kubernetes_api() {
local timeout_seconds="${1:-180}"
local elapsed=0
until kubernetes_api_reachable; do
if ((elapsed >= timeout_seconds)); then
echo "Kubernetes API did not become reachable after ${timeout_seconds}s." >&2
return 1
fi
sleep 5
elapsed=$((elapsed + 5))
done
}
ensure_existing_cluster_started_for_up() {
if ! cluster_control_plane_tracked; then
echo "No tracked kubeadm control plane found; bootstrap/cluster will create one."
return 0
fi
if kubernetes_api_reachable; then
echo "Existing Kubernetes control plane is reachable."
return 0
fi
echo "Existing Kubernetes control plane is tracked but API is down; starting cluster runtime..."
start_cluster
wait_for_kubernetes_api "${LAB_CLUSTER_START_WAIT_SECONDS:-180}"
echo "Existing Kubernetes control plane is reachable after start."
}
status_section() {
printf '\n== %s ==\n' "$1"
}
status_run() {
local description="$1"
shift
printf '\n-- %s\n' "${description}"
if ! "$@"; then
printf 'status check failed: %s\n' "${description}" >&2
fi
}
status_http() {
local name="$1"
local url="$2"
if command -v curl >/dev/null 2>&1; then
printf '%-28s ' "${name}"
curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code} %{url_effective}\n' "${url}" ||
printf 'unreachable %s\n' "${url}"
else
printf '%-28s curl not installed\n' "${name}"
fi
}
status_http_ok() {
local url="$1"
local status
if ! command -v curl >/dev/null 2>&1; then
echo "curl not installed" >&2
return 1
fi
status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || {
echo "unreachable ${url}" >&2
return 1
}
case "${status}" in
2* | 3*)
return 0
;;
*)
echo "HTTP ${status} from ${url}" >&2
return 1
;;
esac
}
status_http_reachable() {
local url="$1"
local status
if ! command -v curl >/dev/null 2>&1; then
echo "curl not installed" >&2
return 1
fi
status="$(curl -k -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' "${url}")" || {
echo "unreachable ${url}" >&2
return 1
}
case "${status}" in
2* | 3* | 4*)
return 0
;;
*)
echo "HTTP ${status} from ${url}" >&2
return 1
;;
esac
}
status_gitea_public_repo_ok() {
local root_url="${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}"
local repo_owner="${LAB_GITEA_REPO_OWNER:-jv}"
local repo_name="${LAB_GITEA_REPO_NAME:-my-homelab-configs}"
status_http_ok "${root_url%/}/${repo_owner}/${repo_name}/"
}
status_cascade_check() {
local description="$1"
local output
shift
printf '%-36s ' "${description}"
if output="$("$@" 2>&1)"; then
printf 'ok\n'
if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
fi
return 0
fi
printf 'fail\n'
if [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed -n '1,8p' | sed 's/^/ /'
fi
return 1
}
status_cascade_warn() {
local description="$1"
local output
shift
printf '%-36s ' "${description}"
if output="$("$@" 2>&1)"; then
printf 'ok\n'
if truthy "${LAB_STATUS_SHOW_OK_OUTPUT:-false}" && [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
fi
return 0
fi
printf 'warn\n'
if [[ -n "${output}" ]]; then
printf '%s\n' "${output}" | sed -n '1,6p' | sed 's/^/ /'
fi
return 0
}
status_container_running() {
local container="$1"
if ! command -v docker >/dev/null 2>&1; then
echo "docker not installed" >&2
return 1
fi
if ! sudo docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null | grep -qx true; then
echo "container ${container} is not running" >&2
return 1
fi
}
status_kubernetes_api_ok() {
if ! command -v kubectl >/dev/null 2>&1; then
echo "kubectl not installed" >&2
return 1
fi
if [[ ! -s "${KUBECONFIG_PATH}" ]]; then
echo "kubeconfig missing: ${KUBECONFIG_PATH}" >&2
return 1
fi
kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null
}
status_kubernetes_nodes_ready() {
local nodes
local not_ready
nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes --no-headers 2>/dev/null)" || return 1
if [[ -z "${nodes}" ]]; then
echo "no Kubernetes nodes found" >&2
return 1
fi
not_ready="$(awk '$2 !~ /^Ready/ { print $1 ":" $2 }' <<<"${nodes}")"
if [[ -n "${not_ready}" ]]; then
echo "not ready: ${not_ready}" >&2
return 1
fi
}
status_kubernetes_deployment_ready() {
local namespace="$1"
local deployment="$2"
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" rollout status "deployment/${deployment}" --timeout=5s >/dev/null
}
status_no_problem_pods() {
local rows
rows="$(
kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers 2>/dev/null |
awk '$4 != "Running" && $4 != "Completed" { print $1 "/" $2 ":" $4 }'
)"
if [[ -n "${rows}" ]]; then
printf '%s\n' "${rows}" >&2
return 1
fi
}
status_registry_ok() {
status_http_ok "http://${LAB_REGISTRY_ENDPOINT:-192.168.100.73:30500}/v2/"
}
status_rpi_uptime_kuma_ok() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" \
"curl -fsS --max-time 5 'http://127.0.0.1:${uptime_kuma_port}/' >/dev/null"
}
status_pimox_workers_running() {
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local index
local vmid
local failures=0
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
echo "invalid worker count ${worker_count}" >&2
return 1
fi
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
continue
fi
vmid=$((worker_base_vmid + index - 1))
if ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then
echo "VM ${vmid} is not running" >&2
failures=$((failures + 1))
fi
done
((failures == 0))
}
status_cascade_report() {
local failures=0
status_section "Status Cascade"
status_cascade_check "inventory YAML" inventory_check || failures=$((failures + 1))
status_cascade_check "Debian Docker root" check_debian_docker_root || failures=$((failures + 1))
status_cascade_check "Debian Docker service" check_systemd_active docker || failures=$((failures + 1))
status_cascade_check "Debian containerd service" check_systemd_active containerd || failures=$((failures + 1))
status_cascade_check "Debian Tailscale IP" check_debian_tailscale_ip || failures=$((failures + 1))
status_section "Git And Local Services"
status_cascade_check "Gitea container" status_container_running "${GITEA_CONTAINER_NAME:-homelab-gitea}" || failures=$((failures + 1))
status_cascade_check "Gitea local HTTP" status_http_ok "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" || failures=$((failures + 1))
status_cascade_warn "Arr Radarr HTTP" status_http_ok "http://127.0.0.1:7878/"
status_cascade_warn "Arr Sonarr HTTP" status_http_ok "http://127.0.0.1:8989/"
status_cascade_warn "Arr Prowlarr HTTP" status_http_ok "http://127.0.0.1:9696/"
status_section "RPi Bootstrap And DNS"
status_cascade_check "RPi SSH" check_rpi_ssh || failures=$((failures + 1))
status_cascade_check "RPi Docker root state" check_rpi_docker_root_state || failures=$((failures + 1))
status_cascade_check "Pi-hole DNS" check_pihole_dns_query || failures=$((failures + 1))
status_cascade_warn "RPi Tailscale IP" check_rpi_tailscale_ip
status_cascade_warn "Uptime Kuma HTTP" status_rpi_uptime_kuma_ok
status_section "Pimox And Cluster"
status_cascade_check "Pimox storage" check_pimox_storage || failures=$((failures + 1))
status_cascade_check "Pimox workers running" status_pimox_workers_running || failures=$((failures + 1))
status_cascade_check "Kubernetes API" status_kubernetes_api_ok || failures=$((failures + 1))
status_cascade_check "Kubernetes nodes Ready" status_kubernetes_nodes_ready || failures=$((failures + 1))
status_cascade_warn "No problem pods" status_no_problem_pods
status_section "Platform And Apps"
status_cascade_check "Local registry" status_registry_ok || failures=$((failures + 1))
status_cascade_check "Traefik deployment" status_kubernetes_deployment_ready traefik traefik || failures=$((failures + 1))
status_cascade_check "Website deployment" status_kubernetes_deployment_ready website-production php-website-deployment || failures=$((failures + 1))
status_cascade_warn "Argo CD server" status_kubernetes_deployment_ready argocd argocd-server
status_section "Edge And Public URLs"
status_cascade_check "OCI edge SSH" check_edge_ssh || failures=$((failures + 1))
status_cascade_check "Traefik LoadBalancer HTTP" status_http_reachable "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/" || failures=$((failures + 1))
status_cascade_check "Website public URL" status_http_ok "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/" || failures=$((failures + 1))
status_cascade_check "Gitea public repo URL" status_gitea_public_repo_ok || failures=$((failures + 1))
if ((failures > 0)); then
printf '\nStatus cascade found %s blocking failure(s).\n' "${failures}" >&2
return 1
fi
printf '\nStatus cascade passed.\n'
}
backstage_brain_enabled() {
local provider="${LAB_AI_GATEWAY_PROVIDER:-ollama}"
local enabled="${LAB_BACKSTAGE_BRAIN_ENABLED:-false}"
[[ "${provider,,}" == "ollama" ]] && truthy "${enabled}"
}
backstage_brain_note() {
local topic="$1"
local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}"
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
local timeout="${LAB_AI_GATEWAY_TIMEOUT_SECONDS:-20}"
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
local context
local retrieved_context=""
if ! backstage_brain_enabled; then
return 0
fi
if ! command -v python3 >/dev/null 2>&1; then
return 0
fi
context="$(cat)"
if [[ -s "${index_dir}/index.json" ]]; then
retrieved_context="$("${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 3 "${topic} ${context}" 2>/dev/null || true)"
fi
if [[ -n "${retrieved_context}" ]]; then
context="${context}
Retrieved homelab knowledge:
${retrieved_context}"
fi
BACKSTAGE_CONTEXT="${context}" python3 - "${endpoint}" "${model}" "${timeout}" "${topic}" <<'PY' || true
import json
import os
import sys
import urllib.error
import urllib.request
endpoint, model, timeout, topic = sys.argv[1:5]
context = os.environ.get("BACKSTAGE_CONTEXT", "").strip()
try:
timeout_seconds = int(timeout)
except ValueError:
timeout_seconds = 20
prompt = f"""You are the backstage helper for a personal homelab script named jeannie.
Be concise and operational. Do not invent facts. Use only the supplied context.
Return at most 5 bullets. Focus on likely cause, next command, and risk.
Topic: {topic}
Context:
{context}
"""
payload = {
"model": model,
"prompt": prompt,
"stream": False,
"options": {
"temperature": 0.1,
"num_predict": 220,
},
}
try:
tags_request = urllib.request.Request(f"{endpoint.rstrip('/')}/api/tags")
with urllib.request.urlopen(tags_request, timeout=3) as response:
tags = json.loads(response.read().decode("utf-8"))
available = {item.get("name", "") for item in tags.get("models", [])}
if model not in available and f"{model}:latest" not in available:
print(f"\n== Backstage Notes ==\nOllama is reachable, but model '{model}' is not pulled. Run: ollama pull {model}")
raise SystemExit(0)
request = urllib.request.Request(
f"{endpoint.rstrip('/')}/api/generate",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
result = json.loads(response.read().decode("utf-8"))
except (OSError, TimeoutError, urllib.error.URLError, json.JSONDecodeError):
raise SystemExit(0)
note = (result.get("response") or "").strip()
if note:
print("\n== Backstage Notes ==")
print(note)
PY
}
status_systemd_units() {
local unit
for unit in "$@"; do
if systemctl list-unit-files --no-legend "${unit}.service" 2>/dev/null | grep -q .; then
printf '%-28s %s\n' "${unit}" "$(systemctl is-active "${unit}" 2>/dev/null || true)"
else
printf '%-28s not-installed\n' "${unit}"
fi
done
}
status_compose_stack() {
local name="$1"
local directory="$2"
printf '\n-- Docker Compose: %s (%s)\n' "${name}" "${directory}"
if [[ ! -d "${directory}" ]]; then
printf 'missing directory\n'
return 0
fi
if ! command -v docker >/dev/null 2>&1; then
printf 'docker not installed\n'
return 0
fi
if ! sudo docker compose version >/dev/null 2>&1; then
printf 'docker compose unavailable\n'
return 0
fi
(cd "${directory}" && sudo docker compose ps) || true
}
status_kubernetes() {
status_section "Kubernetes"
if ! command -v kubectl >/dev/null 2>&1; then
printf 'kubectl not installed\n'
return 0
fi
if [[ ! -s "${KUBECONFIG_PATH}" ]]; then
printf 'kubeconfig missing: %s\n' "${KUBECONFIG_PATH}"
return 0
fi
if ! kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1; then
printf 'API server not reachable through %s\n' "${KUBECONFIG_PATH}"
return 0
fi
status_run "nodes" kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o wide
status_kubernetes_problem_pods
status_run "traefik services" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n traefik get svc,pods -o wide
status_run "website pods" kubectl --kubeconfig "${KUBECONFIG_PATH}" -n website-production get pods -o wide
}
status_kubernetes_problem_pods() {
local rows
printf '\n-- pods not Running/Completed\n'
rows="$(
kubectl --kubeconfig "${KUBECONFIG_PATH}" get pods -A --no-headers -o wide |
awk '$4 != "Running" && $4 != "Completed" { print }'
)"
if [[ -n "${rows}" ]]; then
printf '%s\n' "${rows}"
else
printf 'none\n'
fi
}
status_pimox_workers() {
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local worker_count="${LAB_PIMOX_WORKER_COUNT:-$(pimox_worker_count_default)}"
local worker_base_vmid="${LAB_PIMOX_WORKER_BASE_VMID:-9010}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local index
local vmid
status_section "Pimox Workers"
if ! [[ "${worker_count}" =~ ^[0-9]+$ ]]; then
printf 'invalid LAB_PIMOX_WORKER_COUNT: %s\n' "${worker_count}"
return 0
fi
if ((worker_count == 0)); then
printf 'worker count is 0\n'
return 0
fi
for ((index = 1; index <= worker_count; index++)); do
if worker_index_is_skipped "${index}" "${worker_skip_indexes}"; then
printf 'worker index %s skipped\n' "${index}"
continue
fi
vmid=$((worker_base_vmid + index - 1))
printf 'VM %-6s ' "${vmid}"
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}'" 2>/dev/null ||
printf 'unreachable or missing\n'
done
}
status_rpi_services() {
local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}"
local rpi_user="${LAB_RPI_USER:-${LAB_RASPBERRY_USER:-jv}}"
local rpi_key="${LAB_RPI_SSH_KEY_PATH:-${LAB_RASPBERRY_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}}"
local install_dir="${LAB_RPI_SERVICES_INSTALL_DIR:-/opt/homelab-rpi-services}"
local docker_nvme_root="${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}"
local docker_fallback_root="${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}"
local pihole_container="${PIHOLE_CONTAINER_NAME:-homelab-pihole}"
local unbound_container="${UNBOUND_CONTAINER_NAME:-homelab-unbound}"
local uptime_kuma_port="${UPTIME_KUMA_PORT:-3001}"
status_section "RPi Services"
ssh -i "${rpi_key}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${rpi_user}@${rpi_host}" "
set +e
dns_query() {
server=\"\$1\"
port=\"\$2\"
name=\"\$3\"
python3 - \"\$server\" \"\$port\" \"\$name\" <<'PY'
import random
import socket
import struct
import sys
server, port, name = sys.argv[1], int(sys.argv[2]), sys.argv[3].rstrip('.')
query_id = random.randrange(0, 65536)
packet = struct.pack('!HHHHHH', query_id, 0x0100, 1, 0, 0, 0)
for label in name.split('.'):
packet += bytes([len(label)]) + label.encode('ascii')
packet += b'\x00' + struct.pack('!HH', 1, 1)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(5)
sock.sendto(packet, (server, port))
data, _ = sock.recvfrom(512)
if len(data) < 12:
raise SystemExit('short DNS response')
response_id, flags, _qd, an, _ns, _ar = struct.unpack('!HHHHHH', data[:12])
rcode = flags & 0x000f
if response_id != query_id:
raise SystemExit('mismatched DNS response id')
if rcode != 0:
raise SystemExit(f'DNS rcode {rcode}')
if an < 1:
raise SystemExit('DNS response had no answers')
print(f'ok answers={an}')
PY
}
check() {
label=\"\$1\"
shift
printf '%-34s ' \"\$label\"
output=\"\$("\$@" 2>&1)\"
status=\$?
if [ \"\$status\" -eq 0 ]; then
printf 'ok'
if [ -n \"\$output\" ]; then
printf ' - %s' \"\$(printf '%s' \"\$output\" | head -n 1)\"
fi
printf '\n'
else
printf 'fail - %s\n' \"\$(printf '%s' \"\$output\" | head -n 1)\"
fi
return 0
}
docker_root=\"\$(sudo docker info --format '{{.DockerRootDir}}' 2>/dev/null || true)\"
printf '%-34s %s\n' 'Docker root' \"\${docker_root:-unknown}\"
if [ \"\$docker_root\" = '${docker_nvme_root}' ]; then
if mountpoint -q '${docker_nvme_root}'; then
printf '%-34s ok - nvme mount active\n' 'Docker root class'
else
printf '%-34s warn - configured for nvme but mountpoint is missing\n' 'Docker root class'
fi
elif [ \"\$docker_root\" = '${docker_fallback_root}' ]; then
printf '%-34s ok - fallback root active\n' 'Docker root class'
else
printf '%-34s warn - unexpected docker root\n' 'Docker root class'
fi
if [ -d '${install_dir}' ]; then
cd '${install_dir}'
sudo docker compose ps || true
unbound_ip=\"\$(sudo docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' '${unbound_container}' 2>/dev/null || true)\"
check 'Pi-hole DNS query' dns_query 127.0.0.1 53 cloudflare.com
if [ -n \"\$unbound_ip\" ]; then
check 'Unbound DNS query' dns_query \"\$unbound_ip\" 53 cloudflare.com
else
printf '%-34s fail - container IP unavailable\n' 'Unbound DNS query'
fi
upstreams=\"\$(sudo docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' '${pihole_container}' 2>/dev/null | grep -E '^PIHOLE_DNS_=|^FTLCONF_dns_upstreams=' || true)\"
if printf '%s\n' \"\$upstreams\" | grep -Eq '1\\.1\\.1\\.1|9\\.9\\.9\\.9|8\\.8\\.8\\.8'; then
printf '%-34s ok - public fallback configured\n' 'Pi-hole fallback config'
else
printf '%-34s warn - no public fallback found in visible config\n' 'Pi-hole fallback config'
fi
check 'Fallback resolver 1.1.1.1' dns_query 1.1.1.1 53 cloudflare.com
check 'Fallback resolver 9.9.9.9' dns_query 9.9.9.9 53 cloudflare.com
check 'Uptime Kuma HTTP' curl -fsS --max-time 5 \"http://127.0.0.1:${uptime_kuma_port}/\"
else
echo 'RPi services install directory is missing: ${install_dir}'
fi" || printf 'unable to reach %s@%s\n' "${rpi_user}" "${rpi_host}"
}
status_report() {
local cascade_status=0
require_debian_server "status"
status_cascade_report || cascade_status=$?
if ! truthy "${LAB_STATUS_DETAILS:-false}"; then
return "${cascade_status}"
fi
status_section "Host"
printf 'hostname: %s\n' "$(hostname -f 2>/dev/null || hostname)"
printf 'date: %s\n' "$(date -Is)"
printf 'uptime: %s\n' "$(uptime -p 2>/dev/null || uptime)"
status_run "memory" free -h
status_run "disk" df -h / /data /data/openebs/local
status_section "Systemd"
status_systemd_units ssh docker containerd kubelet tailscaled homelab-gitea-runner
status_section "Docker"
if command -v docker >/dev/null 2>&1; then
status_run "docker containers" sudo docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
else
printf 'docker not installed\n'
fi
status_compose_stack "gitea" "/data/homelab-gitea"
status_compose_stack "arr-stack" "${REPO_ROOT}/infra/arr-stack"
status_kubernetes
status_pimox_workers
status_rpi_services
status_section "Tailscale"
if command -v tailscale >/dev/null 2>&1; then
status_run "tailscale ip" tailscale ip -4
status_run "tailscale peers" tailscale status
else
printf 'tailscale not installed\n'
fi
status_section "HTTP"
status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/"
status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}"
status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/"
status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/"
status_http "arr radarr" "http://127.0.0.1:7878/"
status_http "arr sonarr" "http://127.0.0.1:8989/"
status_http "arr prowlarr" "http://127.0.0.1:9696/"
return "${cascade_status}"
}
doctor_edge() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
local traefik_ip="${LAB_TRAEFIK_LB_IP:-192.168.100.240}"
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
require_debian_server "doctor-edge"
status_section "Doctor Edge"
status_http "website public" "${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/"
status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}"
printf '\n-- edge backend reachability\n'
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" "
set +e
printf '%-28s ' 'traefik backend'
curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable
printf '%-28s ' 'gitea backend'
curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable
printf '\n-- tailscale\n'
tailscale status || true
"; then
printf 'unable to reach edge host %s@%s\n' "${edge_user}" "${edge_host}"
fi
cat <<'EOF'
Next steps:
- If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster.
- If only Gitea is unreachable, run ./jeannie doctor-gitea.
- If both are unreachable from OCI, check Tailscale routes and bootstrap/edge.
- Runbook: docs/runbooks/edge-failures.md
EOF
backstage_brain_note "doctor-edge" <<EOF
Doctor command: doctor-edge
Public domain: ${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/
Public Gitea URL: ${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}
OCI edge SSH target: ${edge_user}@${edge_host}
Traefik backend from edge: http://${traefik_ip}:80/
Gitea backend from edge: http://${gitea_ts_ip}:${gitea_http_port}/
Known runbook: docs/runbooks/edge-failures.md
EOF
}
doctor_gitea() {
local edge_host="${LAB_EDGE_HOST:-132.145.170.74}"
local edge_user="${LAB_EDGE_USER:-ubuntu}"
local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:-100.85.138.30}"
local gitea_host="${LAB_GITEA_HOST:-192.168.100.73}"
local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}"
local gitea_ssh_port="${LAB_GITEA_SSH_PORT:-32222}"
require_debian_server "doctor-gitea"
status_section "Doctor Gitea"
status_http "gitea local" "http://127.0.0.1:${gitea_http_port}/"
status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}"
status_compose_stack "gitea" "/data/homelab-gitea"
status_run "gitea git remote" git ls-remote gitea main
printf '\n-- gitea SSH\n'
ssh -T -p "${gitea_ssh_port}" -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "git@${gitea_host}" || true
printf '\n-- edge to gitea backend\n'
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \
"curl -I --max-time 5 'http://${gitea_ts_ip}:${gitea_http_port}/'"; then
printf 'edge host cannot reach Gitea backend %s:%s\n' "${gitea_ts_ip}" "${gitea_http_port}"
fi
cat <<'EOF'
Next steps:
- If local Gitea is down, run ./jeannie deploy-gitea.
- If local works but public /git/ fails, check Tailscale and bootstrap/edge.
- If Git SSH fails, check the gitea remote and registered SSH key.
- Runbook: docs/runbooks/gitea-failures.md
EOF
backstage_brain_note "doctor-gitea" <<EOF
Doctor command: doctor-gitea
Gitea local URL: http://127.0.0.1:${gitea_http_port}/
Gitea LAN host: ${gitea_host}
Gitea Tailscale backend: http://${gitea_ts_ip}:${gitea_http_port}/
Gitea SSH port: ${gitea_ssh_port}
Gitea public URL: ${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:-https://lab2025.duckdns.org}/git/}
Known runbook: docs/runbooks/gitea-failures.md
EOF
}
doctor_rpi() {
require_debian_server "doctor-rpi"
status_rpi_services
cat <<'EOF'
Next steps:
- If Docker root is /var/lib/docker, the NVMe mount is unavailable or failed the writable check.
- If Pi-hole works but Unbound fails, DNS should still resolve through configured public fallbacks.
- If Pi-hole DNS fails, clients using the RPi as DNS will be affected.
- Reapply the stack with ./jeannie rpi-services.
EOF
backstage_brain_note "doctor-rpi" <<EOF
Doctor command: doctor-rpi
RPi LAN host: ${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}
RPi Tailscale IP: ${LAB_RPI_TAILSCALE_IP:-unknown}
Docker NVMe root: ${LAB_RPI_DOCKER_NVME_ROOT:-/nvme-storage/docker}
Docker fallback root: ${LAB_RPI_DOCKER_FALLBACK_ROOT:-/var/lib/docker}
Services: Pi-hole, Unbound, Uptime Kuma
EOF
}
doctor_cluster() {
require_debian_server "doctor-cluster"
status_section "Doctor Cluster"
status_systemd_units kubelet containerd docker
status_kubernetes
status_pimox_workers
status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:-192.168.100.240}/"
printf '\n-- local CRI containers\n'
if command -v crictl >/dev/null 2>&1; then
sudo crictl ps -a | head -50 || true
else
printf 'crictl not installed\n'
fi
cat <<'EOF'
Next steps:
- If API server is unreachable after a stop, run ./jeannie start-cluster.
- If nodes are NotReady, inspect kube-system pods and containerd/kubelet on that node.
- If Pimox workers are stopped, start them with ./jeannie start-cluster.
- If kubelet versions drift, run ./jeannie doctor-versions.
- Runbook: docs/runbooks/cluster-stop-start-failures.md
EOF
backstage_brain_note "doctor-cluster" <<EOF
Doctor command: doctor-cluster
Kubeconfig path: ${KUBECONFIG_PATH}
Traefik LoadBalancer IP: ${LAB_TRAEFIK_LB_IP:-192.168.100.240}
Pimox host: ${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}@${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}
Worker storage: ${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}
Known runbook: docs/runbooks/cluster-stop-start-failures.md
EOF
}
nuke() {
local target
declare -a CLUSTER_WORKER_TARGETS=()
require_debian_server "nuke"
if ! truthy "${LAB_NUKE_SKIP_CONFIRM:-false}" && [[ "${LAB_CONFIRM_NUKE:-}" != "homelab" ]]; then
cat >&2 <<'EOF'
nuke destroys Kubernetes state and Pimox worker VMs.
Rerun with LAB_CONFIRM_NUKE=homelab when that destructive action is intended.
EOF
exit 1
fi
echo "Brutally nuking the homelab infrastructure..."
cluster_worker_targets
echo "--> Terminating local OpenTofu tasks..."
killall tofu terraform 2>/dev/null || true
echo "--> Eviscerating local Kubernetes components..."
cleanup_node
sudo rm -f "${KUBECONFIG_PATH}"
for target in "${CLUSTER_WORKER_TARGETS[@]}"; do
echo "--> Eviscerating remote Kubernetes components (${target})..."
if ! ssh -o ConnectTimeout=5 "${target}" "bash -s" <<'EOF'
set -euo pipefail
cleanup_calico_links() {
ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true
sudo ip link delete vxlan.calico 2>/dev/null || true
sudo ip link delete tunl0 2>/dev/null || true
sudo ip link delete cni0 2>/dev/null || true
sudo ip link delete kube-ipvs0 2>/dev/null || true
ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true
}
cleanup_iptables() {
sudo iptables -F || true
sudo iptables -X || true
sudo iptables -t nat -F || true
sudo iptables -t nat -X || true
sudo iptables -t mangle -F || true
sudo iptables -t mangle -X || true
sudo iptables -t raw -F || true
sudo iptables -t raw -X || true
if command -v ipvsadm >/dev/null 2>&1; then
sudo ipvsadm --clear || true
fi
}
cleanup_calico_runtime_files() {
local path
for path in /run/calico /var/run/calico; do
if sudo test -e "${path}"; then
sudo find "${path}" -path '*/cgroup*' -prune -o -mindepth 1 -exec rm -rf -- {} + 2>/dev/null || true
sudo rmdir "${path}" 2>/dev/null || true
fi
done
}
restore_node_dns() {
sudo rm -f /etc/systemd/resolved.conf.d/homelab-k8s.conf
if sudo test -e /etc/resolv.conf.homelab-k8s-backup; then
sudo rm -f /etc/resolv.conf
sudo mv /etc/resolv.conf.homelab-k8s-backup /etc/resolv.conf
fi
sudo systemctl restart systemd-resolved 2>/dev/null || true
}
cleanup_mounts() {
if command -v findmnt >/dev/null 2>&1; then
local mount_root
while IFS= read -r mountpoint; do
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
done < <(
for mount_root in /var/lib/kubelet /var/lib/containerd /run/calico /run/calico/cgroup /var/run/calico /var/run/calico/cgroup; do
findmnt -Rno TARGET "${mount_root}" 2>/dev/null || true
done | sort -ru
)
fi
while IFS= read -r mountpoint; do
sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true
done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true)
sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true
}
sudo kubeadm reset --force || true
sudo systemctl stop kubelet 2>/dev/null || true
sudo systemctl stop containerd 2>/dev/null || true
sudo killall containerd-shim-runc-v2 2>/dev/null || true
cleanup_mounts
sudo rm -rf \
/etc/kubernetes/ \
/var/lib/etcd/ \
/var/lib/kubelet/ \
/var/lib/cni/ \
/etc/cni/net.d \
/run/flannel \
/var/lib/calico \
/var/log/calico \
/var/lib/containerd/* \
/run/containerd/* \
/etc/containerd/certs.d \
/etc/containerd/config.toml
cleanup_calico_runtime_files
sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam
cleanup_iptables
cleanup_calico_links
restore_node_dns
sudo mkdir -p /etc/containerd/certs.d
sudo systemctl reset-failed kubelet containerd 2>/dev/null || true
sudo systemctl start containerd 2>/dev/null || true
EOF
then
echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2
exit 1
fi
done
destroy_pimox_worker_vms
docker buildx rm lab-builder 2>/dev/null || true
docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true
rm -f "${BUILDX_CONFIG}" || true
echo "--> Backing up local OpenTofu tracking state files..."
backup_tofu_state
echo "--> Deleting OpenTofu tracking state files..."
rm -rf "${REPO_ROOT}"/bootstrap/cluster/terraform.tfstate*
rm -f "${REPO_ROOT}"/bootstrap/cluster/.terraform.tfstate.lock.info
rm -rf "${REPO_ROOT}"/bootstrap/cluster/.terraform/
rm -rf "${REPO_ROOT}"/bootstrap/platform/terraform.tfstate*
rm -f "${REPO_ROOT}"/bootstrap/platform/.terraform.tfstate.lock.info
rm -rf "${REPO_ROOT}"/bootstrap/platform/.terraform/
rm -rf "${REPO_ROOT}"/bootstrap/apps/terraform.tfstate*
rm -f "${REPO_ROOT}"/bootstrap/apps/.terraform.tfstate.lock.info
rm -rf "${REPO_ROOT}"/bootstrap/apps/.terraform/
rm -rf "${REPO_ROOT}"/bootstrap/edge/terraform.tfstate*
rm -f "${REPO_ROOT}"/bootstrap/edge/.terraform.tfstate.lock.info
rm -rf "${REPO_ROOT}"/bootstrap/edge/.terraform/
rm -f "${REPO_ROOT}/.lab/pimox-workers.tsv" "${REPO_ROOT}/.lab/cluster-workers.auto.tfvars.json"
echo "Destruction complete. Retained data under /data/openebs/local was left intact."
}
ensure_sops_age_tools() {
local missing_packages=()
if ! command -v age-keygen >/dev/null 2>&1; then
missing_packages+=(age)
fi
if ! command -v sops >/dev/null 2>&1; then
missing_packages+=(sops)
fi
if ((${#missing_packages[@]} > 0)); then
echo "Installing missing secret-management tools: ${missing_packages[*]}"
sudo apt-get update
sudo apt-get install -y --no-install-recommends "${missing_packages[@]}"
fi
if ! command -v age-keygen >/dev/null 2>&1; then
echo "age-keygen is still unavailable after package installation." >&2
exit 1
fi
if ! command -v sops >/dev/null 2>&1; then
echo "sops is still unavailable after package installation." >&2
exit 1
fi
}
sops_age_key_file() {
printf '%s\n' "${SOPS_AGE_KEY_FILE:-${HOME}/.config/sops/age/keys.txt}"
}
sops_age_recipient() {
local key_file="$1"
awk -F': ' '/^# public key:/ { print $2; exit }' "${key_file}"
}
secrets_init() {
local key_file
local key_dir
local recipient
local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}"
local example_file="${REPO_ROOT}/.sops.yaml.example"
require_debian_server "secrets-init"
ensure_sops_age_tools
key_file="$(sops_age_key_file)"
key_dir="$(dirname "${key_file}")"
mkdir -p "${key_dir}"
chmod 700 "${key_dir}"
if [[ ! -s "${key_file}" ]]; then
echo "Generating age identity at ${key_file}..."
age-keygen -o "${key_file}"
chmod 600 "${key_file}"
else
echo "Using existing age identity at ${key_file}."
chmod 600 "${key_file}"
fi
recipient="$(sops_age_recipient "${key_file}")"
if [[ -z "${recipient}" ]]; then
echo "Could not read the public recipient from ${key_file}." >&2
exit 1
fi
if [[ -e "${config_file}" ]]; then
if grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then
echo "${config_file} still contains the placeholder age recipient." >&2
exit 1
fi
echo "SOPS config already exists: ${config_file}"
else
if [[ ! -s "${example_file}" ]]; then
echo "Missing ${example_file}" >&2
exit 1
fi
sed "s/age1replacewithyourpublicrecipient/${recipient}/g" "${example_file}" >"${config_file}"
echo "Wrote ${config_file} with Debian host age recipient ${recipient}."
fi
echo "Private age key: ${key_file}"
echo "Public age recipient: ${recipient}"
echo "Review and commit ${config_file}; never commit ${key_file}."
}
secrets_check_file() {
local file="$1"
local key_file="$2"
case "${file}" in
*.json)
grep -q '"sops"' "${file}" || {
echo "${file} does not look SOPS-encrypted." >&2
return 1
}
;;
*)
grep -q '^sops:' "${file}" || {
echo "${file} does not look SOPS-encrypted." >&2
return 1
}
;;
esac
if [[ -s "${key_file}" ]]; then
SOPS_AGE_KEY_FILE="${key_file}" sops -d "${file}" >/dev/null
fi
}
secrets_check() {
local config_file="${SOPS_CONFIG:-${REPO_ROOT}/.sops.yaml}"
local key_file
local recipient=""
local failures=0
local found_files=0
local file
key_file="$(sops_age_key_file)"
if [[ ! -s "${config_file}" ]]; then
echo "Missing ${config_file}. Run ./jeannie secrets-init on the Debian host." >&2
failures=$((failures + 1))
elif grep -q 'age1replacewithyourpublicrecipient' "${config_file}"; then
echo "${config_file} still contains the placeholder age recipient." >&2
failures=$((failures + 1))
fi
if [[ -s "${key_file}" ]]; then
recipient="$(sops_age_recipient "${key_file}")"
if [[ -n "${recipient}" && -s "${config_file}" ]] && ! grep -q "${recipient}" "${config_file}"; then
echo "${config_file} does not include this host's age recipient ${recipient}." >&2
failures=$((failures + 1))
fi
else
echo "No local age key found at ${key_file}; encrypted file structure will be checked without decrypting."
fi
if command -v git >/dev/null 2>&1; then
while IFS= read -r file; do
[[ -n "${file}" ]] || continue
found_files=$((found_files + 1))
secrets_check_file "${REPO_ROOT}/${file}" "${key_file}" || failures=$((failures + 1))
done < <(git -C "${REPO_ROOT}" ls-files '*.secret.yaml' '*.secret.yml' '*.secret.json' '*.enc.yaml' '*.enc.yml' '*.enc.json')
fi
if ((failures > 0)); then
echo "Secret checks failed with ${failures} issue(s)." >&2
exit 1
fi
if ((found_files == 0)); then
echo "SOPS config checks passed. No encrypted secret files are committed yet."
else
echo "SOPS config checks passed for ${found_files} encrypted secret file(s)."
fi
}
tailnet_policy_check() {
"${REPO_ROOT}/scripts/validate-tailnet-policy"
}
ai_index() {
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
require_debian_server "ai-index"
ensure_python3
if [[ "${index_dir}" == /data/* ]]; then
sudo mkdir -p "${index_dir}"
sudo chown "${USER}:$(id -gn)" "${index_dir}"
fi
"${REPO_ROOT}/scripts/build-homelab-ai-index" --index-dir "${index_dir}"
}
ai_check() {
local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}"
local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}"
local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}"
local failures=0
require_debian_server "ai-check"
status_section "AI Knowledge"
if command -v python3 >/dev/null 2>&1; then
printf '%-28s ok\n' "python3"
else
printf '%-28s missing\n' "python3"
failures=$((failures + 1))
fi
if [[ -s "${index_dir}/index.json" ]]; then
printf '%-28s %s\n' "knowledge index" "${index_dir}/index.json"
else
printf '%-28s missing - run ./jeannie ai-index\n' "knowledge index"
failures=$((failures + 1))
fi
if "${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 2 "gitea edge traefik" >/dev/null 2>&1; then
printf '%-28s ok\n' "retrieval smoke test"
else
printf '%-28s failed\n' "retrieval smoke test"
failures=$((failures + 1))
fi
if backstage_brain_enabled; then
if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then
printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}"
if python3 - "${endpoint}" "${model}" <<'PY'
import json
import sys
import urllib.request
endpoint, model = sys.argv[1:3]
with urllib.request.urlopen(f"{endpoint.rstrip('/')}/api/tags", timeout=5) as response:
tags = json.loads(response.read().decode("utf-8"))
models = {item.get("name", "") for item in tags.get("models", [])}
raise SystemExit(0 if model in models or f"{model}:latest" in models else 1)
PY
then
printf '%-28s ok - %s\n' "ollama model" "${model}"
else
printf '%-28s missing - run: ollama pull %s\n' "ollama model" "${model}"
failures=$((failures + 1))
fi
else
printf '%-28s unreachable - %s\n' "ollama endpoint" "${endpoint}"
failures=$((failures + 1))
fi
else
printf '%-28s disabled\n' "backstage helper"
fi
if ((failures > 0)); then
echo "AI check failed with ${failures} issue(s)." >&2
exit 1
fi
echo "AI check passed."
}
case "${1:-}" in
up)
up
;;
plan)
plan_homelab "${2:-all}"
;;
rebuild-cluster)
rebuild_cluster
;;
stop-cluster)
stop_cluster
;;
start-cluster)
start_cluster
;;
status)
status_report
;;
apps)
require_debian_server "apps"
jeannie_log_start "apps"
jeannie_step_plan 1
run_step "Applications" apps
echo "Log: ${JEANNIE_LOG_FILE}"
;;
website-translation-model)
website_translation_model
;;
website-ollama-listen)
website_ollama_listen
;;
ollama-setup)
ollama_setup
;;
deploy-gitea)
deploy_gitea
;;
rpi-services)
deploy_rpi_services
;;
bootstrap-gitea-repo)
bootstrap_gitea_repo
;;
backup-gitea)
backup_gitea
;;
drill-gitea-restore)
drill_gitea_restore
;;
install-gitea-runner)
install_gitea_runner "${2:-}"
;;
move-prometheus-stack-workers)
move_prometheus_stack_workers
;;
doctor-versions)
doctor_versions
;;
doctor-edge)
doctor_edge
;;
doctor-gitea)
doctor_gitea
;;
doctor-rpi)
doctor_rpi
;;
doctor-cluster)
doctor_cluster
;;
preflight)
homelab_preflight
;;
doctor-preapply)
doctor_preapply
;;
inventory-check)
inventory_check
;;
state-backup)
backup_tofu_state
;;
fix-debian-docker-root)
fix_debian_docker_root
;;
secrets-init)
secrets_init
;;
secrets-check)
secrets_check
;;
tailnet-policy-check)
tailnet_policy_check
;;
ai-index)
ai_index
;;
ai-check)
ai_check
;;
openwrt)
openwrt
;;
nuke)
require_debian_server "nuke"
jeannie_log_start "nuke"
jeannie_step_plan 1
run_step "Nuke homelab cluster state" nuke
echo "Log: ${JEANNIE_LOG_FILE}"
;;
*)
echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|doctor-preapply|inventory-check|state-backup|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}"
exit 1
;;
esac
Reference in New Issue View Git Blame Copy Permalink