Repair Pimox worker SSH readiness
This commit is contained in:
juvdiaz
parent
349ffd006e
commit
06941a8454
131
jeannie
131
jeannie
|
|
@ -1076,7 +1076,18 @@ sudo '${qm_bin}' status '${vmid}'
|
|||
echo 'Pimox VM ${vmid} config summary:'
|
||||
sudo '${qm_bin}' config '${vmid}' | grep -E '^(agent|boot|net0|scsi0|virtio0|sata0|ide0|ide2|efidisk0):' || true
|
||||
echo 'Pimox VM ${vmid} guest-agent network-get-interfaces:'
|
||||
sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces" >&2 || true
|
||||
sudo '${qm_bin}' guest cmd '${vmid}' network-get-interfaces || true
|
||||
echo 'Pimox VM ${vmid} guest SSH diagnostics:'
|
||||
sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc 'set +e
|
||||
echo "-- units --"
|
||||
systemctl is-active ssh sshd 2>/dev/null || true
|
||||
echo "-- listeners --"
|
||||
ss -ltnp 2>/dev/null | grep :22 || ss -ltn 2>/dev/null | grep :22 || true
|
||||
echo "-- sshd config test --"
|
||||
sudo sshd -t 2>&1 || true
|
||||
echo "-- recent ssh logs --"
|
||||
journalctl -u ssh -u sshd --no-pager -n 40 2>/dev/null || true
|
||||
' || true" >&2 || true
|
||||
}
|
||||
|
||||
pimox_guest_exec_exitcode() {
|
||||
|
|
@ -1099,6 +1110,73 @@ except Exception:
|
|||
PY
|
||||
}
|
||||
|
||||
pimox_worker_guest_ssh_repair() {
|
||||
local host="$1"
|
||||
local user="$2"
|
||||
local key_path="$3"
|
||||
local vmid="$4"
|
||||
local qm_bin="$5"
|
||||
local script
|
||||
local encoded_script
|
||||
local repair_output
|
||||
local repair_exitcode
|
||||
|
||||
script="$(cat <<'EOF'
|
||||
set -eu
|
||||
ssh_unit=""
|
||||
if systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
|
||||
ssh_unit=ssh.service
|
||||
elif systemctl list-unit-files sshd.service 2>/dev/null | grep -q '^sshd[.]service'; then
|
||||
ssh_unit=sshd.service
|
||||
else
|
||||
echo "Neither ssh.service nor sshd.service exists in the guest" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sudo ssh-keygen -A
|
||||
sudo install -d -m 0755 /run/sshd
|
||||
sudo mkdir -p /etc/ssh/sshd_config.d
|
||||
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
|
||||
Port 22
|
||||
ListenAddress 0.0.0.0
|
||||
PubkeyAuthentication yes
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
PermitRootLogin no
|
||||
UsePAM yes
|
||||
SSHD_CONFIG
|
||||
|
||||
sudo sshd -t
|
||||
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
|
||||
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
|
||||
sudo systemctl enable "$ssh_unit" >/dev/null
|
||||
sudo systemctl restart "$ssh_unit"
|
||||
sleep 2
|
||||
sudo systemctl is-active "$ssh_unit"
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltnp 2>/dev/null || ss -ltn
|
||||
ss -ltn | awk '$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
|
||||
fi
|
||||
EOF
|
||||
)"
|
||||
encoded_script="$(printf '%s' "${script}" | base64 | tr -d '\n')"
|
||||
if ! repair_output="$(pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' guest exec '${vmid}' -- bash -lc \"printf '%s' '${encoded_script}' | base64 -d | sudo bash\"" 2>&1)"; then
|
||||
echo "Could not run guest SSH repair through qemu-guest-agent for VM ${vmid}." >&2
|
||||
printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "${repair_output}" | sed 's/^/ guest-ssh-repair: /' >&2
|
||||
if repair_exitcode="$(pimox_guest_exec_exitcode "${repair_output}")" && [[ "${repair_exitcode}" == "0" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Guest SSH repair failed inside Pimox VM ${vmid}." >&2
|
||||
if [[ -n "${repair_exitcode:-}" ]]; then
|
||||
echo "Guest exit code: ${repair_exitcode}" >&2
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_pimox_guest_ssh() {
|
||||
local host="$1"
|
||||
local user="$2"
|
||||
|
|
@ -1124,6 +1202,9 @@ wait_for_pimox_guest_ssh() {
|
|||
local ssh_deadline=0
|
||||
local ssh_output
|
||||
local ssh_timeout_seconds="${LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS:-600}"
|
||||
local ssh_refused_since=0
|
||||
local ssh_refused_timeout_seconds="${LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS:-120}"
|
||||
local ssh_repair_attempted=false
|
||||
|
||||
ip_filter_description="matching prefix ${ip_prefix}"
|
||||
if [[ -z "${ip_prefix}" ]]; then
|
||||
|
|
@ -1133,6 +1214,10 @@ wait_for_pimox_guest_ssh() {
|
|||
echo "LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS must be a positive integer." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! [[ "${ssh_refused_timeout_seconds}" =~ ^[0-9]+$ ]] || ((ssh_refused_timeout_seconds == 0)); then
|
||||
echo "LAB_PIMOX_GUEST_SSH_REFUSED_TIMEOUT_SECONDS must be a positive integer." >&2
|
||||
return 1
|
||||
fi
|
||||
if ! truthy "${require_host_probe}" && ! disabled_value "${require_host_probe}"; then
|
||||
echo "LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE must be true or false." >&2
|
||||
return 1
|
||||
|
|
@ -1181,6 +1266,23 @@ wait_for_pimox_guest_ssh() {
|
|||
return 0
|
||||
fi
|
||||
last_ssh_output="${ssh_output}"
|
||||
if [[ "${ssh_output}" == *"Connection refused"* ]]; then
|
||||
if ((ssh_refused_since == 0)); then
|
||||
ssh_refused_since="${SECONDS}"
|
||||
fi
|
||||
if [[ "${ssh_repair_attempted}" == "false" ]]; then
|
||||
echo "SSH on worker VM ${vmid} at ${guest_ip} is refusing connections; attempting qemu-guest-agent SSH repair..." >&2
|
||||
pimox_worker_guest_ssh_repair "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}" || true
|
||||
ssh_repair_attempted=true
|
||||
elif ((SECONDS - ssh_refused_since >= ssh_refused_timeout_seconds)); then
|
||||
echo "Worker VM ${vmid} at ${guest_ip} is reachable but still refuses SSH after ${ssh_refused_timeout_seconds}s." >&2
|
||||
echo "This usually means sshd is missing, broken, or not listening on IPv4 inside the worker template." >&2
|
||||
pimox_worker_vm_debug "${host}" "${user}" "${key_path}" "${vmid}" "${qm_bin}"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
ssh_refused_since=0
|
||||
fi
|
||||
fi
|
||||
|
||||
if ((SECONDS >= next_log)); then
|
||||
|
|
@ -1360,15 +1462,36 @@ sudo ip addr add ${worker_ip} dev ${interface_name}
|
|||
sudo ip route replace default via ${gateway} dev ${interface_name}
|
||||
printf '%s\n' ${dns_servers} | awk '{ for (i = 1; i <= NF; i++) print "nameserver " \$i }' | sudo tee /etc/resolv.conf >/dev/null
|
||||
sudo ifup --force ${interface_name} 2>/dev/null || true
|
||||
sudo systemctl restart ssh 2>/dev/null || true
|
||||
sudo ssh-keygen -A
|
||||
sudo install -d -m 0755 /run/sshd
|
||||
sudo mkdir -p /etc/ssh/sshd_config.d
|
||||
sudo tee /etc/ssh/sshd_config.d/99-homelab-worker-listen.conf >/dev/null <<'SSHD_CONFIG'
|
||||
Port 22
|
||||
ListenAddress 0.0.0.0
|
||||
PubkeyAuthentication yes
|
||||
PasswordAuthentication no
|
||||
KbdInteractiveAuthentication no
|
||||
PermitRootLogin no
|
||||
UsePAM yes
|
||||
SSHD_CONFIG
|
||||
sudo sshd -t
|
||||
sudo systemctl unmask ssh.service ssh.socket sshd.service sshd.socket >/dev/null 2>&1 || true
|
||||
sudo systemctl disable --now ssh.socket sshd.socket >/dev/null 2>&1 || true
|
||||
ssh_unit=ssh.service
|
||||
if ! systemctl list-unit-files ssh.service 2>/dev/null | grep -q '^ssh[.]service'; then
|
||||
ssh_unit=sshd.service
|
||||
fi
|
||||
sudo systemctl enable "\$ssh_unit" >/dev/null
|
||||
sudo systemctl restart "\$ssh_unit"
|
||||
ip -br addr show ${interface_name}
|
||||
ip route
|
||||
ip -4 addr show dev ${interface_name} | grep -q '${worker_address}/'
|
||||
${lan_probe_block}
|
||||
echo 'guest-check: ssh service'
|
||||
sudo systemctl is-active ssh
|
||||
sudo systemctl is-active "\$ssh_unit"
|
||||
if command -v ss >/dev/null 2>&1; then
|
||||
ss -ltn | grep -Eq '(^|[[:space:]])0\.0\.0\.0:22[[:space:]]|(^|[[:space:]])\[::\]:22[[:space:]]|(^|[[:space:]])\*:22[[:space:]]'
|
||||
ss -ltnp 2>/dev/null || ss -ltn
|
||||
ss -ltn | awk '\$4 == "0.0.0.0:22" { found = 1 } END { exit found ? 0 : 1 }'
|
||||
fi
|
||||
EOF
|
||||
)"
|
||||
|
|
|
|||
Loading…
Reference in New Issue