diff --git a/lib/jeannie/commands.sh b/lib/jeannie/commands.sh index 7acbd39..750eea9 100644 --- a/lib/jeannie/commands.sh +++ b/lib/jeannie/commands.sh @@ -63,7 +63,7 @@ rebuild_cluster() { } nuke_for_rebuild() { - LAB_NUKE_SKIP_CONFIRM=true nuke + LAB_NUKE_SKIP_CONFIRM=true LAB_NUKE_ALLOW_REMOTE_CLEANUP_FAILURES=true nuke } cluster_worker_targets() { @@ -455,6 +455,18 @@ start_cluster() { echo "Kubernetes runtime start requested. Use 'kubectl get nodes -o wide' to watch readiness." } +nuke_remote_cleanup_failed() { + local target="$1" + + if truthy "${LAB_NUKE_ALLOW_REMOTE_CLEANUP_FAILURES:-false}"; then + echo "Warning: remote cleanup failed for ${target}; continuing because rebuild will destroy and recreate generated worker VMs." >&2 + return 0 + fi + + echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 + return 1 +} + kubernetes_api_reachable() { kubectl --kubeconfig "${KUBECONFIG_PATH}" get --raw=/readyz >/dev/null 2>&1 } @@ -1818,8 +1830,7 @@ sudo systemctl reset-failed kubelet containerd 2>/dev/null || true sudo systemctl start containerd 2>/dev/null || true EOF then - echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 - exit 1 + nuke_remote_cleanup_failed "${target}" || exit 1 fi done diff --git a/lib/jeannie/pipeline.sh b/lib/jeannie/pipeline.sh index 17dbe60..170d944 100644 --- a/lib/jeannie/pipeline.sh +++ b/lib/jeannie/pipeline.sh @@ -2946,6 +2946,18 @@ cleanup_iptables() { fi } +restore_docker_networking_after_iptables_cleanup() { + if ! command -v docker >/dev/null 2>&1; then + return 0 + fi + if ! systemctl is-active --quiet docker 2>/dev/null; then + return 0 + fi + + echo "Restarting Docker to restore published-port iptables rules after Kubernetes cleanup..." + sudo systemctl restart docker +} + cleanup_calico_runtime_files() { local path @@ -3009,6 +3021,7 @@ cleanup_node() { cleanup_iptables cleanup_calico_links + restore_docker_networking_after_iptables_cleanup restore_node_dns sudo mkdir -p /etc/containerd/certs.d diff --git a/tests/jeannie-unit b/tests/jeannie-unit index e91ae3e..8c976c5 100755 --- a/tests/jeannie-unit +++ b/tests/jeannie-unit @@ -170,6 +170,22 @@ test_cluster_worker_targets_include_pimox_and_manual_workers() { assert_eq "cluster targets include generated Pimox workers" "jv@192.168.100.66,jv@192.168.100.67,jv@192.168.100.76,jv@192.168.100.99" "${targets}" } +test_nuke_remote_cleanup_failure_blocks_state_delete_by_default() { + unset LAB_NUKE_ALLOW_REMOTE_CLEANUP_FAILURES + assert_command_failure_contains \ + "nuke remote cleanup failure blocks state delete by default" \ + "not deleting OpenTofu state" \ + nuke_remote_cleanup_failed "jv@192.168.100.67" +} + +test_nuke_remote_cleanup_failure_can_continue_for_rebuild() { + LAB_NUKE_ALLOW_REMOTE_CLEANUP_FAILURES=true + assert_command_success \ + "nuke remote cleanup failure can continue for rebuild" \ + nuke_remote_cleanup_failed "jv@192.168.100.67" + unset LAB_NUKE_ALLOW_REMOTE_CLEANUP_FAILURES +} + test_prepare_cluster_worker_var_file_merges_manual_and_pimox_workers() { unset LAB_INCLUDE_RASPBERRY_WORKER LAB_INCLUDE_MANUAL_WORKERS=true @@ -403,6 +419,8 @@ test_pimox_worker_count_detects_generated_topology test_configured_worker_count_cannot_hide_existing_workers test_configured_worker_count_can_expand_target_set test_cluster_worker_targets_include_pimox_and_manual_workers +test_nuke_remote_cleanup_failure_blocks_state_delete_by_default +test_nuke_remote_cleanup_failure_can_continue_for_rebuild test_prepare_cluster_worker_var_file_merges_manual_and_pimox_workers test_prepare_cluster_worker_var_file_excludes_manual_by_default test_prepare_cluster_worker_var_file_can_include_raspberry_worker