|
|
|
|
@ -1,4 +1,5 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
# shellcheck disable=SC2034,SC2043,SC2140
|
|
|
|
|
|
|
|
|
|
load_homelab_inventory_defaults() {
|
|
|
|
|
local inventory_file="${HOMELAB_INVENTORY_FILE:-${REPO_ROOT}/homelab.yml}"
|
|
|
|
|
@ -26,7 +27,7 @@ mapping = {
|
|
|
|
|
"hosts.debian.lan_ip": "LAB_DEBIAN_LAN_IP",
|
|
|
|
|
"hosts.debian.tailscale_ip": "LAB_DEBIAN_TAILSCALE_IP",
|
|
|
|
|
"hosts.debian.docker_root": "LAB_DEBIAN_DOCKER_ROOT",
|
|
|
|
|
"hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH_PATH",
|
|
|
|
|
"hosts.debian.kubeconfig": "LAB_KUBECONFIG_PATH",
|
|
|
|
|
"hosts.rpi4.user": "LAB_RPI_USER",
|
|
|
|
|
"hosts.rpi4.lan_ip": "LAB_RPI_HOST",
|
|
|
|
|
"hosts.rpi4.tailscale_ip": "LAB_RPI_TAILSCALE_IP",
|
|
|
|
|
@ -157,7 +158,7 @@ export_if_unset() {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export_homelab_inventory_tf_vars() {
|
|
|
|
|
export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH_PATH:-}"
|
|
|
|
|
export_if_unset TF_VAR_kubeconfig_path "${LAB_KUBECONFIG_PATH:-}"
|
|
|
|
|
export_if_unset TF_VAR_control_plane_endpoint "${LAB_DEBIAN_LAN_IP:-}"
|
|
|
|
|
export_if_unset TF_VAR_registry_endpoint "${LAB_REGISTRY_ENDPOINT:-}"
|
|
|
|
|
export_if_unset TF_VAR_provisioning_host "${LAB_DEBIAN_LAN_IP:-}"
|
|
|
|
|
@ -216,7 +217,7 @@ helm_release_secret_exists() {
|
|
|
|
|
local release_name="$2"
|
|
|
|
|
local secret_name
|
|
|
|
|
|
|
|
|
|
secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get secrets \
|
|
|
|
|
secret_name="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get secrets \
|
|
|
|
|
-l "owner=helm,name=${release_name}" \
|
|
|
|
|
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
|
|
|
|
|
|
|
|
|
|
@ -229,11 +230,11 @@ kubernetes_resource_exists() {
|
|
|
|
|
local resource_name="$3"
|
|
|
|
|
|
|
|
|
|
if [[ -n "${namespace}" ]]; then
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
|
|
|
return $?
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get "${resource_kind}" "${resource_name}" >/dev/null 2>&1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
adopt_tofu_helm_release() {
|
|
|
|
|
@ -405,7 +406,7 @@ ensure_homelab_node_labels() {
|
|
|
|
|
[[ -n "${node}" ]] || continue
|
|
|
|
|
|
|
|
|
|
if [[ "${node}" == "${control_plane_node}" ]]; then
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
|
|
|
homelab.dev/node-role=control-plane \
|
|
|
|
|
homelab.dev/storage=local \
|
|
|
|
|
homelab.dev/workload-class=control-plane \
|
|
|
|
|
@ -413,26 +414,26 @@ ensure_homelab_node_labels() {
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
|
|
|
node-role.kubernetes.io/worker=worker \
|
|
|
|
|
--overwrite
|
|
|
|
|
|
|
|
|
|
if [[ "${node}" == pimox-worker-* ]]; then
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
|
|
|
homelab.dev/node-role=app \
|
|
|
|
|
homelab.dev/storage=ssd \
|
|
|
|
|
homelab.dev/workload-class=platform \
|
|
|
|
|
--overwrite
|
|
|
|
|
elif [[ "${node}" == "${raspberry_node}" ]]; then
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" label node "${node}" \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \
|
|
|
|
|
homelab.dev/node-role=edge-app \
|
|
|
|
|
homelab.dev/storage=local \
|
|
|
|
|
homelab.dev/workload-class=edge \
|
|
|
|
|
--overwrite
|
|
|
|
|
fi
|
|
|
|
|
done < <(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
|
|
|
done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
|
|
|
|
|
|
|
|
|
|
target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes -l "${prometheus_selector}" -o name)"
|
|
|
|
|
target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)"
|
|
|
|
|
if [[ -z "${target_nodes}" ]]; then
|
|
|
|
|
echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2
|
|
|
|
|
exit 1
|
|
|
|
|
@ -445,22 +446,22 @@ delete_prometheus_stack_storage() {
|
|
|
|
|
local pvc_names
|
|
|
|
|
local pv_names
|
|
|
|
|
|
|
|
|
|
pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null |
|
|
|
|
|
pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null |
|
|
|
|
|
awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')"
|
|
|
|
|
pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get pv \
|
|
|
|
|
pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \
|
|
|
|
|
-o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null |
|
|
|
|
|
awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')"
|
|
|
|
|
|
|
|
|
|
if [[ -n "${pvc_names}" ]]; then
|
|
|
|
|
echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..."
|
|
|
|
|
printf '%s\n' "${pvc_names}" |
|
|
|
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s
|
|
|
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
if [[ -n "${pv_names}" ]]; then
|
|
|
|
|
echo "Deleting old prometheus-stack retained PV objects..."
|
|
|
|
|
printf '%s\n' "${pv_names}" |
|
|
|
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" delete pv --wait=false
|
|
|
|
|
xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
@ -579,7 +580,7 @@ EOF
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH_PATH}}"
|
|
|
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
|
|
|
|
|
export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"
|
|
|
|
|
|
|
|
|
|
echo "Moving prometheus-stack off the control plane. Existing prometheus-stack PVC data will be deleted."
|
|
|
|
|
@ -589,7 +590,7 @@ EOF
|
|
|
|
|
tofu -chdir="${REPO_ROOT}/${stack}" destroy -target=helm_release.prometheus_stack "${approve_args[@]}"
|
|
|
|
|
delete_prometheus_stack_storage "${namespace}"
|
|
|
|
|
tofu -chdir="${REPO_ROOT}/${stack}" apply "${approve_args[@]}"
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" -n "${namespace}" get pods -o wide
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pods -o wide
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
doctor_versions_report() {
|
|
|
|
|
@ -604,12 +605,12 @@ doctor_versions_report() {
|
|
|
|
|
ensure_python3
|
|
|
|
|
|
|
|
|
|
api_version="$(
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" version -o json |
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
|
|
|
|
|
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
|
|
|
|
|
)"
|
|
|
|
|
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
|
|
|
|
|
node_rows="$(
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
|
|
|
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
|
|
|
|
|
)"
|
|
|
|
|
|
|
|
|
|
@ -690,12 +691,12 @@ doctor_versions_compact() {
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
api_version="$(
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" version -o json |
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" version -o json |
|
|
|
|
|
python3 -c 'import json, sys; print(json.load(sys.stdin)["serverVersion"]["gitVersion"])'
|
|
|
|
|
)"
|
|
|
|
|
api_minor="$(printf '%s\n' "${api_version}" | awk -F. '{gsub(/^v/, "", $1); print $1 "." $2}')"
|
|
|
|
|
node_rows="$(
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH_PATH}" get nodes \
|
|
|
|
|
kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes \
|
|
|
|
|
-o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\t"}{.status.nodeInfo.containerRuntimeVersion}{"\n"}{end}'
|
|
|
|
|
)"
|
|
|
|
|
|
|
|
|
|
@ -1568,10 +1569,6 @@ fi
|
|
|
|
|
sudo taskset -c \"\$cpuset\" bash \"\$start_script\""
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cleanup() {
|
|
|
|
|
rm -f \"\$start_script\"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pimox_shutdown_vm_gracefully() {
|
|
|
|
|
local host="$1"
|
|
|
|
|
local user="$2"
|
|
|
|
|
@ -2913,16 +2910,6 @@ fi
|
|
|
|
|
EOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
cleanup() {
|
|
|
|
|
if mountpoint -q "\$mnt_dir" 2>/dev/null; then
|
|
|
|
|
sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true
|
|
|
|
|
fi
|
|
|
|
|
if [ -n "\$loopdev" ]; then
|
|
|
|
|
sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true
|
|
|
|
|
fi
|
|
|
|
|
rm -rf "\$tmp_dir"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
openwrt() {
|
|
|
|
|
require_debian_server "openwrt"
|
|
|
|
|
|
|
|
|
|
@ -3766,38 +3753,6 @@ sudo docker compose ps
|
|
|
|
|
install_gitea_backup_timer
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
install_missing_packages() {
|
|
|
|
|
missing_packages=''
|
|
|
|
|
for package in \"\$@\"; do
|
|
|
|
|
if ! dpkg-query -W -f='\${Status}' \"\$package\" 2>/dev/null | grep -q 'install ok installed'; then
|
|
|
|
|
missing_packages=\"\$missing_packages \$package\"
|
|
|
|
|
fi
|
|
|
|
|
done
|
|
|
|
|
if [ -n \"\$missing_packages\" ]; then
|
|
|
|
|
sudo apt-get update
|
|
|
|
|
sudo apt-get install -y --no-install-recommends \$missing_packages
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
repair_docker_iptables() {
|
|
|
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
echo 'Docker NAT chain is missing on the Gitea host; restarting Docker once to restore iptables state...'
|
|
|
|
|
sudo systemctl restart docker
|
|
|
|
|
sleep 3
|
|
|
|
|
|
|
|
|
|
if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
echo 'Docker NAT chain is still missing after restarting Docker.' >&2
|
|
|
|
|
sudo iptables -t nat -S >&2 || true
|
|
|
|
|
sudo systemctl status docker --no-pager -l >&2 || true
|
|
|
|
|
exit 1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
install_missing_debian_packages() {
|
|
|
|
|
local missing_packages=()
|
|
|
|
|
local package
|
|
|
|
|
@ -5284,7 +5239,7 @@ apps() {
|
|
|
|
|
website_image_ref="${registry_endpoint}/php-website:${WEBSITE_IMAGE_TAG:-$(website_image_tag "${website_source_hash}")}"
|
|
|
|
|
export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}"
|
|
|
|
|
export TF_VAR_website_image_ref="${TF_VAR_website_image_ref:-${website_image_ref}}"
|
|
|
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH_PATH}}"
|
|
|
|
|
export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
|
|
|
|
|
export KUBECONFIG_PATH="${TF_VAR_kubeconfig_path}"
|
|
|
|
|
|
|
|
|
|
require_argocd_application_controller
|
|
|
|
|
@ -5704,6 +5659,15 @@ PY
|
|
|
|
|
return "${status}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
graphiphy_map() {
|
|
|
|
|
if ! command -v graphiphy >/dev/null 2>&1; then
|
|
|
|
|
echo "graphiphy is not installed; skipping repo map generation."
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
graphiphy "${REPO_ROOT}"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
up_task() {
|
|
|
|
|
require_debian_server "__up-task"
|
|
|
|
|
|
|
|
|
|
|