#!/usr/bin/env bash set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${REPO_ROOT}/lib/jeannie/core.sh" source "${REPO_ROOT}/lib/jeannie/env.sh" source "${REPO_ROOT}/lib/jeannie/toolchain.sh" source "${REPO_ROOT}/lib/jeannie/pimox.sh" BUILDX_CONFIG="/tmp/buildx-config.toml" HOMELAB_STATE_DIR="${HOMELAB_STATE_DIR:-${XDG_DATA_HOME:-${HOME}/.local/share}/homelab}" COSIGN_KEY_PREFIX="${COSIGN_KEY_PREFIX:-${HOMELAB_STATE_DIR}/cosign/cosign}" COSIGN_KEY_PATH="${COSIGN_KEY_PATH:-${COSIGN_KEY_PREFIX}.key}" COSIGN_PUBLIC_KEY_PATH="${COSIGN_PUBLIC_KEY_PATH:-${COSIGN_KEY_PREFIX}.pub}" COSIGN_PASSWORD_FILE="${COSIGN_PASSWORD_FILE:-${HOMELAB_STATE_DIR}/cosign/cosign.password}" COSIGN_VERSION="${COSIGN_VERSION:-2.6.3}" COSIGN_BIN="${COSIGN_BIN:-}" HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-homelab-cosign-public-key}" HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}" JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}" JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}" JEANNIE_ERROR_LOG_FILE="${JEANNIE_ERROR_LOG_FILE:-}" JEANNIE_STEP_INDEX=0 JEANNIE_STEP_TOTAL=0 # shellcheck disable=SC1091 source "${REPO_ROOT}/scripts/report-ui" trap 'rm -f "${BUILDX_CONFIG}"' EXIT load_homelab_inventory_defaults export_homelab_inventory_tf_vars KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-${LAB_KUBECONFIG_PATH:-/home/jv/.kube/config}}}" adopt_platform_existing_resources() { local stack="bootstrap/platform" adopt_tofu_helm_release "${stack}" "helm_release.calico_crds" "tigera-operator" "calico-crds" adopt_tofu_helm_release "${stack}" "helm_release.calico" "tigera-operator" "calico" adopt_tofu_helm_release "${stack}" "helm_release.openebs" "openebs" "openebs" adopt_tofu_helm_release "${stack}" "helm_release.metallb[\"enabled\"]" "metallb-system" "metallb" adopt_tofu_helm_release "${stack}" "helm_release.traefik[\"enabled\"]" "traefik" "traefik" adopt_tofu_helm_release "${stack}" "helm_release.argocd" "argocd" "argocd" adopt_tofu_helm_release "${stack}" "helm_release.kyverno" "kyverno" "kyverno" adopt_tofu_helm_release "${stack}" "helm_release.kyverno_policies" "kyverno" "kyverno-policies" adopt_tofu_helm_release "${stack}" "helm_release.loki" "monitoring" "loki" adopt_tofu_helm_release "${stack}" "helm_release.promtail" "monitoring" "promtail" adopt_tofu_helm_release "${stack}" "helm_release.prometheus_stack" "monitoring" "prometheus-stack" adopt_tofu_kubernetes_resource \ "${stack}" \ "kubernetes_storage_class_v1.openebs_hostpath_retain" \ "" \ "storageclass" \ "openebs-hostpath-retain" \ "openebs-hostpath-retain" adopt_tofu_kubernetes_resource \ "${stack}" \ "kubernetes_namespace_v1.monitoring" \ "" \ "namespace" \ "monitoring" \ "monitoring" } adopt_apps_existing_resources() { local stack="bootstrap/apps" local namespace="${TF_VAR_argocd_namespace:-argocd}" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["container-registry"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "container-registry" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["website-production"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "website-production" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["demos-static"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "demos-static" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["heimdall"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "heimdall" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["n8n"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "n8n" adopt_tofu_kubernetes_manifest \ "${stack}" \ 'kubernetes_manifest.argocd_application["supply-chain-policy"]' \ "${namespace}" \ "applications.argoproj.io" \ "argoproj.io/v1alpha1" \ "Application" \ "supply-chain-policy" } ensure_homelab_node_labels() { local control_plane_node="${LAB_CONTROL_PLANE_NODE_NAME:-debian}" local raspberry_node="${LAB_RASPBERRY_NODE_NAME:-raspberry}" local prometheus_selector="homelab.dev/node-role=app,homelab.dev/storage=ssd" local node local target_nodes echo "Applying homelab labels to existing Kubernetes nodes..." while IFS= read -r node; do [[ -n "${node}" ]] || continue if [[ "${node}" == "${control_plane_node}" ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=control-plane \ homelab.dev/storage=local \ homelab.dev/workload-class=control-plane \ --overwrite continue fi kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ node-role.kubernetes.io/worker=worker \ --overwrite if [[ "${node}" == pimox-worker-* ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=app \ homelab.dev/storage=ssd \ homelab.dev/workload-class=platform \ --overwrite elif [[ "${node}" == "${raspberry_node}" ]]; then kubectl --kubeconfig "${KUBECONFIG_PATH}" label node "${node}" \ homelab.dev/node-role=edge-app \ homelab.dev/storage=local \ homelab.dev/workload-class=edge \ --overwrite fi done < <(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') target_nodes="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get nodes -l "${prometheus_selector}" -o name)" if [[ -z "${target_nodes}" ]]; then echo "No nodes match ${prometheus_selector}; refusing to move prometheus-stack." >&2 exit 1 fi } delete_prometheus_stack_storage() { local namespace="${1:-monitoring}" local pattern='(^|-)prometheus-stack-(prometheus|alertmanager|grafana)(-|$)|^prometheus-prometheus-stack|^alertmanager-prometheus-stack|^storage-prometheus-stack-grafana' local pvc_names local pv_names pvc_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" get pvc -o name 2>/dev/null | awk -F/ -v pattern="${pattern}" '$2 ~ pattern {print $2}')" pv_names="$(kubectl --kubeconfig "${KUBECONFIG_PATH}" get pv \ -o jsonpath='{range .items[?(@.spec.claimRef.namespace=="'"${namespace}"'")]}{.metadata.name}{"\t"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null | awk -v pattern="${pattern}" '$2 ~ pattern {print $1}')" if [[ -n "${pvc_names}" ]]; then echo "Deleting old prometheus-stack PVCs in ${namespace}; saved Prometheus, Alertmanager, and Grafana data will be discarded..." printf '%s\n' "${pvc_names}" | xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" -n "${namespace}" delete pvc --wait=true --timeout=180s fi if [[ -n "${pv_names}" ]]; then echo "Deleting old prometheus-stack retained PV objects..." printf '%s\n' "${pv_names}" | xargs -r kubectl --kubeconfig "${KUBECONFIG_PATH}" delete pv --wait=false fi } run_tofu_stack() { local stack="$1" local auto_approve="${L ... [OUTPUT TRUNCATED - 267,570 chars omitted out of 317,495 total] ... s not installed. Run ./jeannie secrets-init first." >&2 return 1 fi key_file="$(sops_age_key_file)" if [[ ! -s "${key_file}" ]]; then echo "Missing SOPS age key at ${key_file}. Run ./jeannie secrets-init, or copy the age key to this machine." >&2 return 1 fi out="$(SOPS_AGE_KEY_FILE="${key_file}" sops --decrypt "${file}")" || { echo "SOPS decryption failed for ${file}." >&2 return 1 } # Export every bare `KEY: value` line (ignore comments/blank). local line key val syn="0" while IFS= read -r line; do [[ "${line}" =~ ^[A-Za-z_][A-Za-z0-9_]*: ]] || continue key="${line%%:*}" val="${line#*: }" # Strip surrounding double quotes, if sops kept them. val="${val%\"}"; val="${val#\"}" export "${key}=${val}" syn="$((syn + 1))" done <<<"${out}" if ((syn == 0)); then echo "No KEY: value pairs found in decrypted ${file}." >&2 return 1 fi } tailnet_policy_check() { "${REPO_ROOT}/scripts/validate-tailnet-policy" } ai_index() { local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" require_debian_server "ai-index" ensure_python3 if [[ "${index_dir}" == /data/* ]]; then sudo mkdir -p "${index_dir}" sudo chown "${USER}:$(id -gn)" "${index_dir}" fi "${REPO_ROOT}/scripts/build-homelab-ai-index" --index-dir "${index_dir}" } ai_check() { local index_dir="${LAB_AI_KNOWLEDGE_INDEX_DIR:-/data/homelab-ai/index}" local endpoint="${LAB_AI_GATEWAY_URL:-${LAB_OLLAMA_URL:-http://127.0.0.1:11434}}" local model="${LAB_AI_GATEWAY_MODEL:-qwen2.5:0.5b}" local failures=0 require_debian_server "ai-check" status_section "AI Knowledge" if command -v python3 >/dev/null 2>&1; then printf '%-28s ok\n' "python3" else printf '%-28s missing\n' "python3" failures=$((failures + 1)) fi if [[ -s "${index_dir}/index.json" ]]; then printf '%-28s %s\n' "knowledge index" "${index_dir}/index.json" else printf '%-28s missing - run ./jeannie ai-index\n' "knowledge index" failures=$((failures + 1)) fi if "${REPO_ROOT}/scripts/query-homelab-ai-index" --index-dir "${index_dir}" --context-only --limit 2 "gitea edge traefik" >/dev/null 2>&1; then printf '%-28s ok\n' "retrieval smoke test" else printf '%-28s failed\n' "retrieval smoke test" failures=$((failures + 1)) fi if backstage_brain_enabled; then if curl -fsS --max-time 5 "${endpoint%/}/api/tags" >/dev/null 2>&1; then printf '%-28s ok - %s\n' "ollama endpoint" "${endpoint}" if python3 "${REPO_ROOT}/scripts/ai-model-check.py" "${endpoint}" "${model}" then printf '%-28s ok - %s\n' "ollama model" "${model}" else printf '%-28s missing - run: ollama pull %s\n' "ollama model" "${model}" failures=$((failures + 1)) fi else printf '%-28s unreachable - %s\n' "ollama endpoint" "${endpoint}" failures=$((failures + 1)) fi else printf '%-28s disabled\n' "backstage helper" fi if ((failures > 0)); then echo "AI check failed with ${failures} issue(s)." >&2 exit 1 fi echo "AI check passed." } ask_homelab() { require_debian_server "ask" "${REPO_ROOT}/scripts/ask" "${@:2}" } ai_evals() { "${REPO_ROOT}/scripts/ai-evals" "${@:2}" } impact() { "${REPO_ROOT}/scripts/impact" "${@:2}" } review_last_change() { "${REPO_ROOT}/scripts/review-last-change" "${@:2}" } security_scan() { require_debian_server "security-scan" "${REPO_ROOT}/scripts/security-scan" all } security_prepare() { require_debian_server "security-prepare" "${REPO_ROOT}/scripts/security-scan" prepare } security_zap() { require_debian_server "security-zap" "${REPO_ROOT}/scripts/security-scan" zap } security_k8s() { require_debian_server "security-k8s" "${REPO_ROOT}/scripts/security-scan" kube-bench } security_host() { require_debian_server "security-host" "${REPO_ROOT}/scripts/security-scan" host } security_trivy() { require_debian_server "security-trivy" "${REPO_ROOT}/scripts/security-scan" trivy } security_secrets() { require_debian_server "security-secrets" "${REPO_ROOT}/scripts/security-scan" secrets } security_nuclei() { require_debian_server "security-nuclei" "${REPO_ROOT}/scripts/security-scan" nuclei } security_web() { require_debian_server "security-web" "${REPO_ROOT}/scripts/security-scan" web } security_logs() { require_debian_server "security-logs" "${REPO_ROOT}/scripts/security-logs" } security_runtime() { require_debian_server "security-runtime" if ! command -v kubectl >/dev/null 2>&1; then echo "kubectl is required for security-runtime." >&2 exit 1 fi kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system rollout status daemonset/tetragon --timeout=30s kubectl --kubeconfig "${KUBECONFIG_PATH}" -n kube-system get pods -l app.kubernetes.io/name=tetragon -o wide cat <<'EOF' Watch recent Tetragon events: kubectl -n kube-system logs -l app.kubernetes.io/name=tetragon -c export-stdout --tail=100 -f Useful practice trigger: kubectl -n security-lab exec deploy/juice-shop -- sh -c 'id; uname -a' EOF }