From 06941a8454172dc6575792e6629ead12faecb556 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Wed, 1 Jul 2026 19:33:23 -0600 Subject: [PATCH] Repair Pimox worker SSH readiness --- jeannie | 131 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 4 deletions(-) diff --git a/jeannie b/jeannie index d7811e9..46fe009 100755 --- a/jeannie +++ b/jeannie @@ -1076,7 +1076,18 @@ sudo '${qm_bin}' status '${vmid}' echo 'Pimox VM ${vmid} config summary:' sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:' -sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" >&2 || true +sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true +echo 'Pimox VM ${vmid} guest SSH diagnostics:' +sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc 'set +e +echo "-- units --" +systemctl is-active ssh sshd 2>/dev/null || true +echo "-- listeners --" +ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true +echo "-- sshd config test --" +sudo sshd -t 2>&1 || true +echo "-- recent ssh logs --" +journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true +' || true" >&2 || true } pimox_guest_exec_exitcode() { @@ -1099,6 +1110,73 @@ except Exception: PY } +pimox_worker_guest_ssh_repair() { + local host="$1" + local user="$2" + local key_path="$3" + local vmid="$4" + local qm_bin="$5" + local script + local encoded_script + local repair_output + local repair_exitcode + + script="$(cat <<'EOF' +set -eu +ssh_unit="" +if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then + ssh_unit=ssh.service +elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then + ssh_unit=sshd.service +else + echo "Neither ssh.service nor sshd.service exists in the guest" >&2 + exit 1 +fi + +sudo ssh-keygen -A +sudo install -d -m 0755 /run/sshd +sudo mkdir -p /etc/ssh/sshd_config.d +sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' +Port 22 +ListenAddress 0.0.0.0 +PubkeyAuthentication yes +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin no +UsePAM yes +SSHD_CONFIG + +sudo sshd -t +sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true +sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true +sudo systemctl enable "$ssh_unit" >/dev/null +sudo systemctl restart "$ssh_unit" +sleep 2 +sudo systemctl is-active "$ssh_unit" +if command -v ss >/dev/null 2>&1; then + ss -ltnp 2>/dev/null || ss -ltn + ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' +fi +EOF +)" + encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')" + if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then + echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2 + printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 + return 1 + fi + + printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2 + if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then + return 0 + fi + echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2 + if [[ -n "${repair_exitcode:-}" ]]; then + echo "Guest exit code: ${repair_exitcode}" >&2 + fi + return 1 +} + wait_for_pimox_guest_ssh() { local host="$1" local user="$2" @@ -1124,6 +1202,9 @@ wait_for_pimox_guest_ssh() { local ssh_deadline=0 local ssh_output local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}" + local ssh_refused_since=0 + local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}" + local ssh_repair_attempted=false ip_filter_description="matching prefix ${ip_prefix}" if [[ -z "${ip_prefix}" ]]; then @@ -1133,6 +1214,10 @@ wait_for_pimox_guest_ssh() { echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2 return 1 fi + if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then + echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2 + return 1 + fi if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2 return 1 @@ -1181,6 +1266,23 @@ wait_for_pimox_guest_ssh() { return 0 fi last_ssh_output="${ssh_output}" + if [[ "${ssh_output}" == *"Connection refused"* ]]; then + if ((ssh_refused_since == 0)); then + ssh_refused_since="${SECONDS}" + fi + if [[ "${ssh_repair_attempted}" == "false" ]]; then + echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2 + pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true + ssh_repair_attempted=true + elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then + echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2 + echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2 + pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" + return 1 + fi + else + ssh_refused_since=0 + fi fi if ((SECONDS >= next_log)); then @@ -1360,15 +1462,36 @@ sudo ip addr add ${worker_ip} dev ${interface_name} sudo ip route replace default via ${gateway} dev ${interface_name} printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null sudo ifup --force ${interface_name} 2>/dev/null || true -sudo systemctl restart ssh 2>/dev/null || true +sudo ssh-keygen -A +sudo install -d -m 0755 /run/sshd +sudo mkdir -p /etc/ssh/sshd_config.d +sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG' +Port 22 +ListenAddress 0.0.0.0 +PubkeyAuthentication yes +PasswordAuthentication no +KbdInteractiveAuthentication no +PermitRootLogin no +UsePAM yes +SSHD_CONFIG +sudo sshd -t +sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true +sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true +ssh_unit=ssh.service +if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then + ssh_unit=sshd.service +fi +sudo systemctl enable "\$ssh_unit" >/dev/null +sudo systemctl restart "\$ssh_unit" ip -br addr show ${interface_name} ip route ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/' ${lan_probe_block} echo 'guest-check: ssh service' -sudo systemctl is-active ssh +sudo systemctl is-active "\$ssh_unit" if command -v ss >/dev/null 2>&1; then - ss -ltn | grep -Eq '(^|[[:space:]])0\.0\.0\.0:22[[:space:]]|(^|[[:space:]])\[::\]:22[[:space:]]|(^|[[:space:]])\*:22[[:space:]]' + ss -ltnp 2>/dev/null || ss -ltn + ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }' fi EOF )"