From 2740409469e5ebebf154716db50b29a71b79dca4 Mon Sep 17 00:00:00 2001
From: juvdiaz <juvenal.diaz@oracle.com>
Date: Thu, 28 May 2026 13:24:31 -0600
Subject: [PATCH] Fix worker CNI plugin path

---
 bootstrap/cluster/main.tf | 102 ++++++++++++++++++++++++++++++++++++--
 1 file changed, 98 insertions(+), 4 deletions(-)

diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf
index 0f679c8..65d9931 100644
--- a/bootstrap/cluster/main.tf
+++ b/bootstrap/cluster/main.tf
@@ -20,7 +20,7 @@ resource "null_resource" "kubeadm_control_plane" {
     kubeconfig_path         = var.kubeconfig_path
     kubeconfig_owner        = var.kubeconfig_owner
     registry_endpoint       = var.registry_endpoint
-    registry_config_version = "6"
+    registry_config_version = "7"
     node_dns_servers        = join(" ", var.node_dns_servers)
     persistent_volume_dirs  = join(",", var.persistent_volume_dirs)
   }
@@ -167,6 +167,51 @@ reset_containerd_registry_tables() {
   sudo mv "$tmp" /etc/containerd/config.toml
 }
 
+ensure_containerd_cni_bin_dir() {
+  local config_version
+  local tmp
+
+  config_version="$(containerd_config_version)"
+  tmp="$(mktemp)"
+  sudo awk -v config_version="$config_version" '
+    /^[[:space:]]*bin_dir[[:space:]]*=/ {
+      sub(/=.*/, "= \"/opt/cni/bin\"")
+      found = 1
+    }
+    /^[[:space:]]*bin_dirs[[:space:]]*=/ {
+      sub(/=.*/, "= [\"/opt/cni/bin\"]")
+      found = 1
+    }
+    { print }
+    END {
+      if (!found) {
+        print ""
+        if (config_version == "3") {
+          print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
+          print "  bin_dirs = [\"/opt/cni/bin\"]"
+        } else {
+          print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
+          print "  bin_dir = \"/opt/cni/bin\""
+        }
+        print "  conf_dir = \"/etc/cni/net.d\""
+      }
+    }
+  ' /etc/containerd/config.toml > "$tmp"
+  sudo mv "$tmp" /etc/containerd/config.toml
+}
+
+install_cni_plugins() {
+  local plugin
+
+  sudo mkdir -p /opt/cni/bin
+  if [ -d /usr/lib/cni ]; then
+    for plugin in /usr/lib/cni/*; do
+      [ -f "$plugin" ] || continue
+      sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
+    done
+  fi
+}
+
 configure_containerd_registry() {
   local registry_endpoint="$1"
   local config_version
@@ -175,6 +220,7 @@ configure_containerd_registry() {
   sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
 
   sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
+  ensure_containerd_cni_bin_dir
   config_version="$(containerd_config_version)"
   if [ "$config_version" = "3" ]; then
     ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
@@ -202,7 +248,8 @@ REGISTRY_EOT
 }
 
 configure_node_dns
-install_missing_packages open-iscsi nfs-common
+install_missing_packages containernetworking-plugins open-iscsi nfs-common
+install_cni_plugins
 sudo systemctl enable --now iscsid
 sudo systemctl enable kubelet || true
 
@@ -295,7 +342,7 @@ resource "null_resource" "kubeadm_worker" {
     user                                 = each.value.user
     ssh_key_path                         = each.value.ssh_key_path
     registry_endpoint                    = var.registry_endpoint
-    registry_config_version              = "6"
+    registry_config_version              = "7"
     node_dns_servers                     = join(" ", var.node_dns_servers)
     persistent_volume_dirs               = join(",", var.persistent_volume_dirs)
     tailscale_nodeport_version           = "3"
@@ -456,6 +503,51 @@ reset_containerd_registry_tables() {
   sudo mv "$tmp" /etc/containerd/config.toml
 }
 
+ensure_containerd_cni_bin_dir() {
+  local config_version
+  local tmp
+
+  config_version="$(containerd_config_version)"
+  tmp="$(mktemp)"
+  sudo awk -v config_version="$config_version" '
+    /^[[:space:]]*bin_dir[[:space:]]*=/ {
+      sub(/=.*/, "= \"/opt/cni/bin\"")
+      found = 1
+    }
+    /^[[:space:]]*bin_dirs[[:space:]]*=/ {
+      sub(/=.*/, "= [\"/opt/cni/bin\"]")
+      found = 1
+    }
+    { print }
+    END {
+      if (!found) {
+        print ""
+        if (config_version == "3") {
+          print "[plugins.\"io.containerd.cri.v1.runtime\".cni]"
+          print "  bin_dirs = [\"/opt/cni/bin\"]"
+        } else {
+          print "[plugins.\"io.containerd.grpc.v1.cri\".cni]"
+          print "  bin_dir = \"/opt/cni/bin\""
+        }
+        print "  conf_dir = \"/etc/cni/net.d\""
+      }
+    }
+  ' /etc/containerd/config.toml > "$tmp"
+  sudo mv "$tmp" /etc/containerd/config.toml
+}
+
+install_cni_plugins() {
+  local plugin
+
+  sudo mkdir -p /opt/cni/bin
+  if [ -d /usr/lib/cni ]; then
+    for plugin in /usr/lib/cni/*; do
+      [ -f "$plugin" ] || continue
+      sudo ln -sf "$plugin" "/opt/cni/bin/$(basename "$plugin")"
+    done
+  fi
+}
+
 configure_containerd_registry() {
   local registry_endpoint="$1"
   local config_version
@@ -464,6 +556,7 @@ configure_containerd_registry() {
   sudo containerd config default | sudo tee /etc/containerd/config.toml >/dev/null
 
   sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
+  ensure_containerd_cni_bin_dir
   config_version="$(containerd_config_version)"
   if [ "$config_version" = "3" ]; then
     ensure_containerd_registry_config_path "io.containerd.cri.v1.images" '[plugins."io.containerd.cri.v1.images".registry]'
@@ -491,7 +584,8 @@ REGISTRY_EOT
 }
 
 configure_node_dns
-install_missing_packages open-iscsi nfs-common
+install_missing_packages containernetworking-plugins open-iscsi nfs-common
+install_cni_plugins
 sudo systemctl enable --now iscsid
 sudo systemctl enable kubelet || true