Using dedicated known host file for ephemeral pimox workers

bootstrap/provisioning/main.tf

@@ -9,6 +9,7 @@ terraform {
 }
 
 locals {
+  repo_root = abspath("${path.module}/../..")
   tftp_root = "${var.provisioning_install_dir}/tftp"
   http_root = "${var.provisioning_install_dir}/html"
 
@@ -257,6 +258,7 @@ resource "null_resource" "pimox_template_vm_seal" {
     guest_host      = var.pimox_template_build_host
     guest_user      = var.pimox_template_build_user
     guest_key_path  = var.pimox_template_build_ssh_key_path
+    known_hosts_file = "${local.repo_root}/.lab/pimox-template-known_hosts"
     seal_version    = "5"
     create_id       = null_resource.pimox_template_vm_create[0].id
     config_hash     = local.config_hash
@@ -279,6 +281,7 @@ pimox_qm_bin="${self.triggers.pimox_qm_bin}"
 guest_host="${self.triggers.guest_host}"
 guest_user="${self.triggers.guest_user}"
 guest_key="${self.triggers.guest_key_path}"
+known_hosts_file="${self.triggers.known_hosts_file}"
 timeout_seconds="${self.triggers.timeout_seconds}"
 ssh_timeout_seconds="${self.triggers.ssh_timeout_seconds}"
 guest_ip_prefix="${self.triggers.guest_ip_prefix}"
@@ -294,7 +297,23 @@ ssh_pimox() {
 }
 
 ssh_guest() {
-  ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new "$guest_user@$guest_host" "$@"
+  ssh -i "$guest_key" -o BatchMode=yes -o ConnectTimeout=8 -o IdentitiesOnly=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$known_hosts_file" "$guest_user@$guest_host" "$@"
+}
+
+ssh_guest_ready() {
+  local ssh_output
+  if ssh_output="$(ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh" 2>&1)"; then
+    return 0
+  fi
+  if [[ "$ssh_output" == *"Host key verification failed."* ]] || [[ "$ssh_output" == *"REMOTE HOST IDENTIFICATION HAS CHANGED"* ]]; then
+    echo "Host key verification failed for template-build VM $vmid at $guest_host using $known_hosts_file." >&2
+    echo "Jeannie uses a dedicated known-hosts file for ephemeral template VMs; the stale entry should be safe to remove." >&2
+    echo "Suggested fix: ssh-keygen -f '$known_hosts_file' -R '$guest_host'" >&2
+    echo "$ssh_output" >&2
+    exit 1
+  fi
+  printf '%s\n' "$ssh_output" >&2
+  return 1
 }
 
 guest_ip_from_agent() {
@@ -343,9 +362,14 @@ if ! [[ "$ssh_timeout_seconds" =~ ^[0-9]+$ ]] || [ "$ssh_timeout_seconds" -le 0
   exit 1
 fi
 
+mkdir -p "$(dirname "$known_hosts_file")"
+touch "$known_hosts_file"
+chmod 0600 "$known_hosts_file"
+
 deadline=$((SECONDS + timeout_seconds))
 ssh_deadline=0
 next_log=$SECONDS
+last_known_hosts_ip=""
 while (( SECONDS < deadline )); do
   agent_ip="$(guest_ip_from_agent || true)"
   if [ -n "$agent_ip" ]; then
@@ -358,7 +382,14 @@ while (( SECONDS < deadline )); do
       ssh_deadline=$((SECONDS + ssh_timeout_seconds))
     fi
   fi
-  if [ -n "$guest_host" ] && ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
+  if [ -n "$guest_host" ] && (( ssh_deadline == 0 )); then
+    ssh_deadline=$((SECONDS + ssh_timeout_seconds))
+  fi
+  if [ -n "$guest_host" ] && [ "$last_known_hosts_ip" != "$guest_host" ]; then
+    ssh-keygen -R "$guest_host" -f "$known_hosts_file" >/dev/null 2>&1 || true
+    last_known_hosts_ip="$guest_host"
+  fi
+  if [ -n "$guest_host" ] && ssh_guest_ready; then
     break
   fi
   if [ -n "$guest_host" ] && (( ssh_deadline > 0 && SECONDS >= ssh_deadline )); then
@@ -382,7 +413,7 @@ if [ -z "$guest_host" ]; then
   exit 1
 fi
 
-if ! ssh_guest "test -x /usr/local/sbin/homelab-prepare-template.sh"; then
+if ! ssh_guest_ready; then
   echo "Timed out waiting for SSH on template-build VM $vmid at $guest_host" >&2
   exit 1
 fi

jeannie

@@ -14,6 +14,7 @@ HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP="${HOMELAB_COSIGN_PUBLIC_KEY_CONFIGMAP:-home
 HOMELAB_SBOM_PREDICATE_TYPE="${HOMELAB_SBOM_PREDICATE_TYPE:-https://spdx.dev/Document}"
 JEANNIE_LOG_DIR="${JEANNIE_LOG_DIR:-${HOMELAB_STATE_DIR}/logs}"
 JEANNIE_LOG_FILE="${JEANNIE_LOG_FILE:-}"
+JEANNIE_ERROR_LOG_FILE="${JEANNIE_ERROR_LOG_FILE:-}"
 JEANNIE_STEP_INDEX=0
 JEANNIE_STEP_TOTAL=0
 
@@ -808,8 +809,11 @@ jeannie_log_start() {
     timestamp="$(date +%Y%m%d-%H%M%S)"
     mkdir -p "${JEANNIE_LOG_DIR}"
     JEANNIE_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.log"
+    JEANNIE_ERROR_LOG_FILE="${JEANNIE_LOG_DIR}/jeannie-${command_name}-${timestamp}.error.log"
     : >"${JEANNIE_LOG_FILE}"
+    : >"${JEANNIE_ERROR_LOG_FILE}"
     printf 'Log: %s\n' "${JEANNIE_LOG_FILE}"
+    printf 'Error log: %s\n' "${JEANNIE_ERROR_LOG_FILE}"
 }
 
 jeannie_step_plan() {
@@ -955,11 +959,17 @@ run_step() {
     printf '\r'
     jeannie_step_prefix "${description}"
     printf 'failed\033[K\n\n'
+    {
+        printf '\n== FAILED [%d/%d] %s ==\n' "${JEANNIE_STEP_INDEX}" "${JEANNIE_STEP_TOTAL}" "${description}"
+        printf 'Finished: %s\n' "$(date -Is)"
+        sed 's/^/    /' "${step_log}"
+    } >>"${JEANNIE_ERROR_LOG_FILE}"
     printf 'Step failed: %s\n' "${description}" >&2
     printf 'Exit status: %s\n' "${status}" >&2
     printf 'Full step output:\n' >&2
     sed 's/^/    /' "${step_log}" >&2
     printf '\nFull log: %s\n' "${JEANNIE_LOG_FILE}" >&2
+    printf 'Error log: %s\n' "${JEANNIE_ERROR_LOG_FILE}" >&2
     rm -f "${step_log}" "${status_file}"
     return "${status}"
 }