From a47f4b56e954c2078d70e08ca61bfdfdf56b04c1 Mon Sep 17 00:00:00 2001 From: "juvenal.diaz" Date: Wed, 22 Jul 2026 21:43:36 -0600 Subject: [PATCH] Using dedicated known host file for ephemeral pimox workers --- bootstrap/provisioning/main.tf | 37 +++++++++++++++++++++++++++++++--- jeannie | 10 +++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index ed99ee4..1f4d34e 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -9,6 +9,7 @@ terraform { } locals { + repo_root = abspath("${path.module}/../..") tftp_root = "${var.provisioning_install_dir}/tftp" http_root = "${var.provisioning_install_dir}/html" @@ -257,6 +258,7 @@ resource "null_resource" "pimox_template_vm_seal" { guest_host = var.pimox_template_build_host guest_user = var.pimox_template_build_user guest_key_path = var.pimox_template_build_ssh_key_path + known_hosts_file = "${local.repo_root}/.lab/pimox-template-known_hosts" seal_version = "5" create_id = null_resource.pimox_template_vm_create[0].id config_hash = local.config_hash @@ -279,6 +281,7 @@ pimox_qm_bin="${self.triggers.pimox_qm_bin}" guest_host="${self.triggers.guest_host}" guest_user="${self.triggers.guest_user}" guest_key="${self.triggers.guest_key_path}" +known_hosts_file="${self.triggers.known_hosts_file}" timeout_seconds="${self.triggers.timeout_seconds}" ssh_timeout_seconds="${self.triggers.ssh_timeout_seconds}" guest_ip_prefix="${self.triggers.guest_ip_prefix}" @@ -294,7 +297,23 @@ ssh_pimox() { } ssh_guest() { - ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "$guest_user@$guest_host" "$@" + ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$guest_user@$guest_host" "$@" +} + +ssh_guest_ready() { + local ssh_output + if ssh_output="$(ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh" 2>&1)"; then + return 0 + fi + if [[ "$ssh_output" == *"Host key verification failed."* ]] || [[ "$ssh_output" == *"REMOTE HOST IDENTIFICATION HAS CHANGED"* ]]; then + echo "Host key verification failed for template-build VM $vmid at $guest_host using $known_hosts_file." >&2 + echo "Jeannie uses a dedicated known-hosts file for ephemeral template VMs; the stale entry should be safe to remove." >&2 + echo "Suggested fix: ssh-keygen -f '$known_hosts_file' -R '$guest_host'" >&2 + echo "$ssh_output" >&2 + exit 1 + fi + printf '%s\n' "$ssh_output" >&2 + return 1 } guest_ip_from_agent() { @@ -343,9 +362,14 @@ if ! [[ "$ssh_timeout_seconds" =~ ^[0-9]+$ ]] || [ "$ssh_timeout_seconds" -le 0 exit 1 fi +mkdir -p "$(dirname "$known_hosts_file")" +touch "$known_hosts_file" +chmod 0600 "$known_hosts_file" + deadline=$((SECONDS + timeout_seconds)) ssh_deadline=0 next_log=$SECONDS +last_known_hosts_ip="" while (( SECONDS < deadline )); do agent_ip="$(guest_ip_from_agent || true)" if [ -n "$agent_ip" ]; then @@ -358,7 +382,14 @@ while (( SECONDS < deadline )); do ssh_deadline=$((SECONDS + ssh_timeout_seconds)) fi fi - if [ -n "$guest_host" ] && ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then + if [ -n "$guest_host" ] && (( ssh_deadline == 0 )); then + ssh_deadline=$((SECONDS + ssh_timeout_seconds)) + fi + if [ -n "$guest_host" ] && [ "$last_known_hosts_ip" != "$guest_host" ]; then + ssh-keygen -R "$guest_host" -f "$known_hosts_file" >/dev/null 2>&1 || true + last_known_hosts_ip="$guest_host" + fi + if [ -n "$guest_host" ] && ssh_guest_ready; then break fi if [ -n "$guest_host" ] && (( ssh_deadline > 0 && SECONDS >= ssh_deadline )); then @@ -382,7 +413,7 @@ if [ -z "$guest_host" ]; then exit 1 fi -if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then +if ! ssh_guest_ready; then echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2 exit 1 fi diff --git a/jeannie b/jeannie index a3c4a4e..e1c0b3b 100755 --- a/jeannie +++ b/jeannie @@ -14,6 +14,7 @@ HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-home HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}" JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}" JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}" +JEANNIE_ERROR_LOG_FILE="${JEANNIE_ERROR_LOG_FILE:-}" JEANNIE_STEP_INDEX=0 JEANNIE_STEP_TOTAL=0 @@ -808,8 +809,11 @@ jeannie_log_start() { timestamp="$(date +%Y%m%d-%H%M%S)" mkdir -p "${JEANNIE_LOG_DIR}" JEANNIE_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.log" + JEANNIE_ERROR_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.error.log" : >"${JEANNIE_LOG_FILE}" + : >"${JEANNIE_ERROR_LOG_FILE}" printf 'Log: %s\n' "${JEANNIE_LOG_FILE}" + printf 'Error log: %s\n' "${JEANNIE_ERROR_LOG_FILE}" } jeannie_step_plan() { @@ -955,11 +959,17 @@ run_step() { printf '\r' jeannie_step_prefix "${description}" printf 'failed\033[K\n\n' + { + printf '\n== FAILED [%d/%d] %s ==\n' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}" + printf 'Finished: %s\n' "$(date -Is)" + sed 's/^/ /' "${step_log}" + } >>"${JEANNIE_ERROR_LOG_FILE}" printf 'Step failed: %s\n' "${description}" >&2 printf 'Exit status: %s\n' "${status}" >&2 printf 'Full step output:\n' >&2 sed 's/^/ /' "${step_log}" >&2 printf '\nFull log: %s\n' "${JEANNIE_LOG_FILE}" >&2 + printf 'Error log: %s\n' "${JEANNIE_ERROR_LOG_FILE}" >&2 rm -f "${step_log}" "${status_file}" return "${status}" }