Add certbot retries

bootstrap/edge/main.tf

@@ -188,6 +188,29 @@ deploy_current_certificate() {
   sudo chmod 0600 "$install_dir/certs/current.key"
 }
 
+letsencrypt_certificate_present() {
+  sudo test -s "$install_dir/letsencrypt/live/$server_name/fullchain.pem" &&
+    sudo test -s "$install_dir/letsencrypt/live/$server_name/privkey.pem"
+}
+
+run_certbot_certonly() {
+  sudo docker run --rm \
+    -v "$install_dir/letsencrypt:/etc/letsencrypt" \
+    -v "$install_dir/certbot/www:/var/www/certbot" \
+    "$certbot_image" certonly \
+      --webroot \
+      -w /var/www/certbot \
+      --cert-name "$server_name" \
+      --expand \
+      $certbot_domain_args \
+      --preferred-challenges http \
+      --agree-tos \
+      --non-interactive \
+      --keep-until-expiring \
+      $email_args \
+      $staging_args
+}
+
 install_renewal_timer() {
   sudo tee /usr/local/sbin/homelab-edge-renew-certs.sh >/dev/null <<RENEW_EOT
 #!/bin/sh
@@ -271,26 +294,31 @@ if [ "$enable_letsencrypt" = "true" ]; then
   fi
 
   certbot_log="$(mktemp)"
-  if ! sudo docker run --rm \
-      -v "$install_dir/letsencrypt:/etc/letsencrypt" \
-      -v "$install_dir/certbot/www:/var/www/certbot" \
-      "$certbot_image" certonly \
-        --webroot \
-        -w /var/www/certbot \
-        --cert-name "$server_name" \
-        --expand \
-        $certbot_domain_args \
-        --preferred-challenges http \
-        --agree-tos \
-        --non-interactive \
-        --keep-until-expiring \
-        $email_args \
-        $staging_args > "$certbot_log" 2>&1; then
+  certbot_ok=false
+  for attempt in 1 2 3; do
+    if run_certbot_certonly > "$certbot_log" 2>&1; then
+      certbot_ok=true
+      break
+    fi
+
     cat "$certbot_log"
     if grep -Eq "Certificate not yet due for renewal|no action taken" "$certbot_log" &&
-       sudo test -s "$install_dir/letsencrypt/live/$server_name/fullchain.pem" &&
-       sudo test -s "$install_dir/letsencrypt/live/$server_name/privkey.pem"; then
+       letsencrypt_certificate_present; then
       echo "Using existing Let's Encrypt certificate for $server_name"
+      certbot_ok=true
+      break
+    fi
+
+    if [ "$attempt" -lt 3 ]; then
+      echo "Certbot failed on attempt $attempt; retrying after $((attempt * 15)) seconds..."
+      sleep $((attempt * 15))
+    fi
+  done
+
+  if [ "$certbot_ok" != "true" ]; then
+    if letsencrypt_certificate_present; then
+      echo "Certbot failed after retries; keeping existing Let's Encrypt certificate for $server_name."
+      certbot_ok=true
     else
       rm -f "$certbot_log"
       exit 1