Repair Pimox worker SSH readiness

jeannie

@@ -1076,7 +1076,18 @@ sudo '${qm_bin}' status '${vmid}'
 echo 'Pimox VM ${vmid} config summary:'
 sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
 echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
-sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" >&2 || true
+sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true
+echo 'Pimox VM ${vmid} guest SSH diagnostics:'
+sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc 'set +e
+echo "-- units --"
+systemctl is-active ssh sshd 2>/dev/null || true
+echo "-- listeners --"
+ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true
+echo "-- sshd config test --"
+sudo sshd -t 2>&1 || true
+echo "-- recent ssh logs --"
+journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
+' || true" >&2 || true
 }
 
 pimox_guest_exec_exitcode() {
@@ -1099,6 +1110,73 @@ except Exception:
 PY
 }
 
+pimox_worker_guest_ssh_repair() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local vmid="$4"
+    local qm_bin="$5"
+    local script
+    local encoded_script
+    local repair_output
+    local repair_exitcode
+
+    script="$(cat <<'EOF'
+set -eu
+ssh_unit=""
+if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
+  ssh_unit=ssh.service
+elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then
+  ssh_unit=sshd.service
+else
+  echo "Neither ssh.service nor sshd.service exists in the guest" >&2
+  exit 1
+fi
+
+sudo ssh-keygen -A
+sudo install -d -m 0755 /run/sshd
+sudo mkdir -p /etc/ssh/sshd_config.d
+sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
+Port 22
+ListenAddress 0.0.0.0
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitRootLogin no
+UsePAM yes
+SSHD_CONFIG
+
+sudo sshd -t
+sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
+sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
+sudo systemctl enable "$ssh_unit" >/dev/null
+sudo systemctl restart "$ssh_unit"
+sleep 2
+sudo systemctl is-active "$ssh_unit"
+if command -v ss >/dev/null 2>&1; then
+  ss -ltnp 2>/dev/null || ss -ltn
+  ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
+fi
+EOF
+)"
+    encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
+    if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
+        echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2
+        printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
+        return 1
+    fi
+
+    printf '%s\n' "${repair_output}" | sed 's/^/    guest-ssh-repair: /' >&2
+    if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then
+        return 0
+    fi
+    echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2
+    if [[ -n "${repair_exitcode:-}" ]]; then
+        echo "Guest exit code: ${repair_exitcode}" >&2
+    fi
+    return 1
+}
+
 wait_for_pimox_guest_ssh() {
     local host="$1"
     local user="$2"
@@ -1124,6 +1202,9 @@ wait_for_pimox_guest_ssh() {
     local ssh_deadline=0
     local ssh_output
     local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}"
+    local ssh_refused_since=0
+    local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}"
+    local ssh_repair_attempted=false
 
     ip_filter_description="matching prefix ${ip_prefix}"
     if [[ -z "${ip_prefix}" ]]; then
@@ -1133,6 +1214,10 @@ wait_for_pimox_guest_ssh() {
         echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
         return 1
     fi
+    if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then
+        echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2
+        return 1
+    fi
     if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
         echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
         return 1
@@ -1181,6 +1266,23 @@ wait_for_pimox_guest_ssh() {
                 return 0
             fi
             last_ssh_output="${ssh_output}"
+            if [[ "${ssh_output}" == *"Connection refused"* ]]; then
+                if ((ssh_refused_since == 0)); then
+                    ssh_refused_since="${SECONDS}"
+                fi
+                if [[ "${ssh_repair_attempted}" == "false" ]]; then
+                    echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2
+                    pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true
+                    ssh_repair_attempted=true
+                elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then
+                    echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2
+                    echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2
+                    pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
+                    return 1
+                fi
+            else
+                ssh_refused_since=0
+            fi
         fi
 
         if ((SECONDS >= next_log)); then
@@ -1360,15 +1462,36 @@ sudo ip addr add ${worker_ip} dev ${interface_name}
 sudo ip route replace default via ${gateway} dev ${interface_name}
 printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null
 sudo ifup --force ${interface_name} 2>/dev/null || true
-sudo systemctl restart ssh 2>/dev/null || true
+sudo ssh-keygen -A
+sudo install -d -m 0755 /run/sshd
+sudo mkdir -p /etc/ssh/sshd_config.d
+sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
+Port 22
+ListenAddress 0.0.0.0
+PubkeyAuthentication yes
+PasswordAuthentication no
+KbdInteractiveAuthentication no
+PermitRootLogin no
+UsePAM yes
+SSHD_CONFIG
+sudo sshd -t
+sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
+sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
+ssh_unit=ssh.service
+if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
+  ssh_unit=sshd.service
+fi
+sudo systemctl enable "\$ssh_unit" >/dev/null
+sudo systemctl restart "\$ssh_unit"
 ip -br addr show ${interface_name}
 ip route
 ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
 ${lan_probe_block}
 echo 'guest-check: ssh service'
-sudo systemctl is-active ssh
+sudo systemctl is-active "\$ssh_unit"
 if command -v ss >/dev/null 2>&1; then
-  ss -ltn | grep -Eq '(^|[[:space:]])0\.0\.0\.0:22[[:space:]]|(^|[[:space:]])\[::\]:22[[:space:]]|(^|[[:space:]])\*:22[[:space:]]'
+  ss -ltnp 2>/dev/null || ss -ltn
+  ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
 fi
 EOF
 )"