Add compact step logging for noisy commands

jeannie

@@ -12,6 +12,10 @@ COSIGN_VERSION="${COSIGN_VERSION:-2.6.3}"
 COSIGN_BIN="${COSIGN_BIN:-}"
 HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-homelab-cosign-public-key}"
 HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}"
+JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}"
+JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}"
+JEANNIE_STEP_INDEX=0
+JEANNIE_STEP_TOTAL=0
 
 trap 'rm -f "${BUILDX_CONFIG}"' EXIT
 
@@ -614,6 +618,80 @@ disabled_value() {
     esac
 }
 
+jeannie_log_start() {
+    local command_name="$1"
+    local timestamp
+
+    if [[ -n "${JEANNIE_LOG_FILE}" ]]; then
+        return 0
+    fi
+
+    timestamp="$(date +%Y%m%d-%H%M%S)"
+    mkdir -p "${JEANNIE_LOG_DIR}"
+    JEANNIE_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.log"
+    : >"${JEANNIE_LOG_FILE}"
+    printf 'Log: %s\n' "${JEANNIE_LOG_FILE}"
+}
+
+jeannie_step_plan() {
+    JEANNIE_STEP_INDEX=0
+    JEANNIE_STEP_TOTAL="$1"
+}
+
+jeannie_verbose_enabled() {
+    truthy "${JEANNIE_VERBOSE:-false}"
+}
+
+run_step() {
+    local description="$1"
+    local step_log
+    local status
+    shift
+
+    JEANNIE_STEP_INDEX=$((JEANNIE_STEP_INDEX + 1))
+    if ((JEANNIE_STEP_TOTAL > 0)); then
+        printf '[%d/%d] %-42s ' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}"
+    else
+        printf '[%d] %-46s ' "${JEANNIE_STEP_INDEX}" "${description}"
+    fi
+
+    step_log="$(mktemp)"
+    {
+        printf '\n== [%d/%d] %s ==\n' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}"
+        printf 'Started: %s\n' "$(date -Is)"
+    } >>"${JEANNIE_LOG_FILE}"
+
+    if jeannie_verbose_enabled; then
+        set +e
+        "$@" 2>&1 | tee "${step_log}"
+        status=${PIPESTATUS[0]}
+        set -e
+    else
+        set +e
+        "$@" >"${step_log}" 2>&1
+        status=$?
+        set -e
+    fi
+
+    cat "${step_log}" >>"${JEANNIE_LOG_FILE}"
+    printf 'Finished: %s\n' "$(date -Is)" >>"${JEANNIE_LOG_FILE}"
+
+    if ((status == 0)); then
+        printf 'ok\n'
+        rm -f "${step_log}"
+        return 0
+    fi
+
+    printf 'failed\n\n'
+    printf 'Step failed: %s\n' "${description}" >&2
+    printf 'Exit status: %s\n' "${status}" >&2
+    printf 'Full step output:\n' >&2
+    sed 's/^/    /' "${step_log}" >&2
+    printf '\nFull log: %s\n' "${JEANNIE_LOG_FILE}" >&2
+    rm -f "${step_log}"
+    return "${status}"
+}
+
 worker_index_is_skipped() {
     local index="$1"
     local skip_indexes="$2"
@@ -3871,29 +3949,36 @@ apps() {
     echo "Application deployment successfully completed."
 }
 
+ensure_cluster_worker_var_file() {
+    if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
+        prepare_cluster_worker_var_file true
+    fi
+}
+
 up() {
     require_debian_server "up"
 
     echo "Deploying the homelab infrastructure..."
+    jeannie_log_start "up"
+    jeannie_step_plan 14
 
-    homelab_preflight early
+    run_step "Early preflight" homelab_preflight early
-    deploy_gitea
+    run_step "Gitea deploy" deploy_gitea
-    bootstrap_gitea_repo
+    run_step "Gitea repo bootstrap" bootstrap_gitea_repo
-    deploy_rpi_services
+    run_step "RPi services" deploy_rpi_services
-    homelab_preflight full
+    run_step "Full preflight" homelab_preflight full
-    run_pimox_pipeline
+    run_step "Pimox provisioning and workers" run_pimox_pipeline
-    run_openwrt_pipeline
+    run_step "OpenWrt VM" run_openwrt_pipeline
-    if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
+    run_step "Worker var file" ensure_cluster_worker_var_file
-        prepare_cluster_worker_var_file true
+    run_step "Start existing cluster if stopped" ensure_existing_cluster_started_for_up
-    fi
+    run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
-    ensure_existing_cluster_started_for_up
+    run_step "Version report" doctor_versions_report
-    run_tofu_stack "bootstrap/cluster"
+    run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
-    doctor_versions_report
+    run_step "Applications" apps
-    run_tofu_stack "bootstrap/platform"
+    run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"
-    apps
-    run_tofu_stack "bootstrap/edge"
 
     echo "Deployment successfully completed."
+    echo "Log: ${JEANNIE_LOG_FILE}"
 }
 
 tofu_stack_from_plan_target() {
@@ -3964,21 +4049,22 @@ rebuild_cluster() {
     export TF_VAR_force_worker_rejoin="${TF_VAR_force_worker_rejoin:-true}"
 
     echo "Rebuilding the Kubernetes cluster without touching external Gitea..."
+    jeannie_log_start "rebuild-cluster"
+    jeannie_step_plan 10
 
-    homelab_preflight
+    run_step "Preflight" homelab_preflight
-    nuke
+    run_step "Nuke existing cluster state" nuke
-    run_pimox_pipeline
+    run_step "Pimox provisioning and workers" run_pimox_pipeline
-    run_openwrt_pipeline
+    run_step "OpenWrt VM" run_openwrt_pipeline
-    if [[ -z "${LAB_CLUSTER_VAR_FILE:-}" ]]; then
+    run_step "Worker var file" ensure_cluster_worker_var_file
-        prepare_cluster_worker_var_file true
+    run_step "Cluster OpenTofu apply" run_tofu_stack "bootstrap/cluster"
-    fi
+    run_step "Version report" doctor_versions_report
-    run_tofu_stack "bootstrap/cluster"
+    run_step "Platform OpenTofu apply" run_tofu_stack "bootstrap/platform"
-    doctor_versions_report
+    run_step "Applications" apps
-    run_tofu_stack "bootstrap/platform"
+    run_step "Edge OpenTofu apply" run_tofu_stack "bootstrap/edge"
-    apps
-    run_tofu_stack "bootstrap/edge"
 
     echo "Cluster rebuild successfully completed."
+    echo "Log: ${JEANNIE_LOG_FILE}"
 }
 
 cluster_worker_targets() {
@@ -5250,7 +5336,11 @@ case "${1:-}" in
         status_report
         ;;
     apps)
-        apps
+        require_debian_server "apps"
+        jeannie_log_start "apps"
+        jeannie_step_plan 1
+        run_step "Applications" apps
+        echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     website-translation-model)
         website_translation_model
@@ -5325,7 +5415,11 @@ case "${1:-}" in
         openwrt
         ;;
     nuke)
-        nuke
+        require_debian_server "nuke"
+        jeannie_log_start "nuke"
+        jeannie_step_plan 1
+        run_step "Nuke homelab cluster state" nuke
+        echo "Log: ${JEANNIE_LOG_FILE}"
         ;;
     *)
         echo "Usage: $0 {up|plan [all|provisioning|cluster|platform|apps|edge]|rebuild-cluster|stop-cluster|start-cluster|status|apps|website-translation-model|website-ollama-listen|ollama-setup|deploy-gitea|rpi-services|bootstrap-gitea-repo|backup-gitea|drill-gitea-restore|install-gitea-runner|move-prometheus-stack-workers|doctor-versions|doctor-edge|doctor-gitea|doctor-rpi|doctor-cluster|preflight|inventory-check|fix-debian-docker-root|secrets-init|secrets-check|tailnet-policy-check|ai-index|ai-check|openwrt|nuke}"