Gates for pimoc workers with less than 2 cpus

Increase argocd limits
2026-07-23 13:16:11 -06:00 · 2026-07-23 11:32:23 -06:00
7 changed files with 378 additions and 46 deletions
--- a/README.md
+++ b/README.md
@ -718,7 +718,11 @@ template and worker VM clones automatically when the Orange Pi storage path is
 healthy. Defaults are intentionally tied to the observed host: Pimox SSH host
 `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two
 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`,
-and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe
+and KVM launch affinity `4-7` because this Pimox is pinned to Debian Bullseye
+on an Orange Pi 5 Plus. Jeannie launches the generated KVM command under that
+host CPU set so PSCI can bring both guest CPUs online; it removes legacy `args`
+entries so native `qm` CPU topology is the only `-smp` argument. Set
+`LAB_PIMOX_KVM_CPUSET=` to use normal `qm start` on a different host. The NVMe
 that was previously attached to the Orange Pi now belongs to the HP Debian
 laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox
 datastore is `opi5_ssd` and is used for rebuildable VM root disks. Details and
@ -759,6 +763,11 @@ qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by
 default and lets SSH be the reachability check. Set
 `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate.
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
+Before `./jeannie apps` changes any application resources, Jeannie requires
+every generated Pimox worker to be `Ready` and expose at least two Kubernetes
+CPUs. Override the threshold with `LAB_PIMOX_APP_DEPLOY_MIN_CPUS`; set
+`LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=false` only for a deliberately
+non-Pimox deployment.

 Long-running `./jeannie up` steps keep compact output by default, but
 the active step line refreshes with elapsed time and the latest log line. Set
@ -778,8 +787,10 @@ Run a full cluster rebuild from the Debian server with:

 That path preserves external Gitea, rebuilds the Pimox template
 with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and
-8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is
-disabled by default because the Bullseye-pinned Pimox `qm` does not support it.
+8 GiB memory, and joins those workers to the Kubernetes cluster. Pimox 7 ARM
+does not support `qm --affinity`; Jeannie instead starts the generated KVM
+command under CPU set `4-7` and verifies every worker reports two guest CPUs
+before Kubernetes workloads proceed.
 The Raspberry Pi is still included as a Kubernetes worker by default; `nuke`
 does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`.

--- a/README.md.tmpl
+++ b/README.md.tmpl
@ -718,7 +718,11 @@ template and worker VM clones automatically when the Orange Pi storage path is
 healthy. Defaults are intentionally tied to the observed host: Pimox SSH host
 `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two
 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`,
-and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe
+and KVM launch affinity `4-7` because this Pimox is pinned to Debian Bullseye
+on an Orange Pi 5 Plus. Jeannie launches the generated KVM command under that
+host CPU set so PSCI can bring both guest CPUs online; it removes legacy `args`
+entries so native `qm` CPU topology is the only `-smp` argument. Set
+`LAB_PIMOX_KVM_CPUSET=` to use normal `qm start` on a different host. The NVMe
 that was previously attached to the Orange Pi now belongs to the HP Debian
 laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox
 datastore is `opi5_ssd` and is used for rebuildable VM root disks. Details and
@ -759,6 +763,12 @@ qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by
 default and lets SSH be the reachability check. Set
 `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate.
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout.
+Before `./{{ main_script }} apps` changes any application resources, Jeannie
+requires every generated Pimox worker to be `Ready` and expose at least two
+Kubernetes CPUs. Override the threshold with
+`LAB_PIMOX_APP_DEPLOY_MIN_CPUS`; set
+`LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=false` only for a deliberately
+non-Pimox deployment.

 Long-running `./{{ main_script }} up` steps keep compact output by default, but
 the active step line refreshes with elapsed time and the latest log line. Set
@ -778,8 +788,10 @@ Run a full cluster rebuild from the Debian server with:

 That path preserves external Gitea, rebuilds the Pimox template
 with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and
-8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is
-disabled by default because the Bullseye-pinned Pimox `qm` does not support it.
+8 GiB memory, and joins those workers to the Kubernetes cluster. Pimox 7 ARM
+does not support `qm --affinity`; Jeannie instead starts the generated KVM
+command under CPU set `4-7` and verifies every worker reports two guest CPUs
+before Kubernetes workloads proceed.
 The Raspberry Pi is still included as a Kubernetes worker by default; `nuke`
 does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`.

--- a/bootstrap/platform/main.tf
+++ b/bootstrap/platform/main.tf
@ -1125,12 +1125,12 @@ resource "helm_release" "argocd" {
        topologySpreadConstraints = local.argocd_component_topology_spread_constraints.controller
        resources = {
          requests = {
-            cpu    = "100m"
+            cpu    = "10m"
            memory = "256Mi"
          }
          limits = {
-            cpu    = "300m"
-            memory = "512Mi"
+            cpu    = "1"
+            memory = "2Gi"
          }
        }
      }
@ -1217,11 +1217,12 @@ resource "null_resource" "argocd_ready" {
  depends_on = [helm_release.argocd]

  triggers = {
-    kubeconfig_path = var.kubeconfig_path
-    namespace       = var.argocd.namespace
-    version         = var.argocd.version
-    node_selector   = jsonencode(local.argocd_node_selector)
-    readiness_check = "7"
+    kubeconfig_path             = var.kubeconfig_path
+    namespace                   = var.argocd.namespace
+    version                     = var.argocd.version
+    node_selector               = jsonencode(local.argocd_node_selector)
+    controller_resource_profile = "10m|1|2Gi"
+    readiness_check             = "8"
  }

  provisioner "local-exec" {
@ -1273,6 +1274,15 @@ wait_for_argocd_component() {
 kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Established --timeout=180s crd/applications.argoproj.io
 wait_for_argocd_component deployment/argocd-applicationset-controller app.kubernetes.io/name=argocd-applicationset-controller
 wait_for_argocd_component statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller
+
+controller_resources="$(kubectl --kubeconfig "$kubeconfig" -n "$namespace" get statefulset argocd-application-controller \
+  -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')"
+if [ "$controller_resources" != "${self.triggers.controller_resource_profile}" ]; then
+  echo "Argo CD application controller resources are '$controller_resources', expected '${self.triggers.controller_resource_profile}'." >&2
+  dump_argocd_readiness_debug statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller
+  exit 1
+fi
+
 wait_for_argocd_component deployment/argocd-dex-server app.kubernetes.io/name=argocd-dex-server
 wait_for_argocd_component deployment/argocd-notifications-controller app.kubernetes.io/name=argocd-notifications-controller
 wait_for_argocd_component deployment/argocd-redis app.kubernetes.io/name=argocd-redis
--- a/bootstrap/provisioning/README.md
+++ b/bootstrap/provisioning/README.md
@ -128,7 +128,9 @@ template build timeout.

 Defaults match the observed Pimox template VM shape: OVMF firmware, virtio
 networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with
-2 cores, 8 GiB memory, and explicit `args: -smp 2,sockets=1,cores=2,threads=1`.
+2 cores, and 8 GiB memory. The native `qm` CPU topology is the only `-smp`
+argument; the provisioning stack removes legacy `args` values to prevent a
+second conflicting `-smp` option.
 Override `TF_VAR_pimox_template_scsi0`,
 `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or
 `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes.
@ -172,6 +174,15 @@ Worker clone failures are intentionally faster than template build failures:
 `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and
 `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600` by default. The broader
 `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation.
+
+On the Orange Pi 5 Plus with Pimox 7 ARM, Jeannie launches the generated KVM
+command under host CPU set `4-7`. This keeps the guest's secondary CPU on the
+compatible RK3588 cores during PSCI boot. The same setting is used for the
+template build and worker clones, and each worker must report its configured
+CPU count through SSH before it is added to the Kubernetes worker inventory.
+Set `LAB_PIMOX_KVM_CPUSET=` to use the normal `qm start` path on a different
+Pimox host, or override template and worker values separately with
+`LAB_PIMOX_TEMPLATE_KVM_CPUSET` and `LAB_PIMOX_WORKER_KVM_CPUSET`.
 Static guest-network validation skips the extra LAN ICMP probe by default
 because the ISP modem may not answer worker VM pings. Set
 `LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=true` only when the gateway/probe host
--- a/bootstrap/provisioning/main.tf
+++ b/bootstrap/provisioning/main.tf
@ -47,7 +47,6 @@ locals {
  node_dns_servers            = join(" ", var.node_dns_servers)
  kernel_cgroup_boot_options  = join(" ", var.kernel_cgroup_boot_options)
  pimox_template_net0         = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}"
-  pimox_template_args         = "-smp ${var.pimox_template_cores},sockets=1,cores=${var.pimox_template_cores},threads=1"
  template_package_list       = join(" ", local.template_packages)
  provisioning_http_base_url  = "http://${var.http_host}:${var.http_port}"
  provisioning_script_url     = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh"
@ -134,7 +133,7 @@ resource "null_resource" "pimox_template_vm_create" {
    pimox_user      = var.pimox_user
    ssh_key_path    = var.pimox_ssh_key_path
    qm_bin          = var.pimox_qm_bin
-    builder_version = "5"
+    builder_version = "6"
    config_hash     = local.config_hash
    vmid            = tostring(var.pimox_template_vmid)
    name            = var.pimox_template_name
@ -142,7 +141,7 @@ resource "null_resource" "pimox_template_vm_create" {
    memory          = tostring(var.pimox_template_memory)
    bridge          = var.pimox_template_bridge
    net0            = local.pimox_template_net0
-    args            = local.pimox_template_args
+    kvm_cpuset      = var.pimox_template_kvm_cpuset
    scsi0           = var.pimox_template_scsi0
    efidisk0        = var.pimox_template_efidisk0
  }
@ -177,6 +176,42 @@ if ! sudo -n true >/dev/null 2>&1; then
  exit 1
 fi

+start_vm() {
+  cpuset="${self.triggers.kvm_cpuset}"
+  if [ -z "$cpuset" ]; then
+    sudo "$qm_cmd" start "$vmid"
+    return
+  fi
+  if ! command -v taskset >/dev/null 2>&1; then
+    echo "taskset is required to start this Pimox VM with a KVM CPU set" >&2
+    exit 1
+  fi
+  if ! taskset -c "$cpuset" true >/dev/null 2>&1; then
+    echo "Pimox KVM CPU set $cpuset is not valid on this host" >&2
+    exit 1
+  fi
+
+  start_script="$(mktemp "/tmp/homelab-qemu-$vmid.XXXXXX")"
+  trap 'rm -f "$start_script"' 0
+
+  sudo "$qm_cmd" showcmd "$vmid" --pretty >"$start_script"
+  if ! awk -v vmid="$vmid" '
+    NR == 1 && ($1 == "/usr/bin/kvm" || $1 ~ /qemu-system-aarch64$/) { kvm = 1 }
+    $1 == "-id" && $2 == vmid { id = 1 }
+    END { exit (kvm && id) ? 0 : 1 }
+  ' "$start_script"; then
+    echo "qm showcmd for VM $vmid did not produce the expected KVM command" >&2
+    exit 1
+  fi
+  smp_count="$(awk '$1 == "-smp" { count++ } END { print count + 0 }' "$start_script")"
+  if [ "$smp_count" -ne 1 ]; then
+    echo "Expected one native -smp argument for VM $vmid, found $smp_count. Remove stale qm args before starting." >&2
+    exit 1
+  fi
+
+  sudo taskset -c "$cpuset" bash "$start_script"
+}
+
 if ! ip link show "${self.triggers.bridge}" >/dev/null 2>&1; then
  echo "Pimox bridge ${self.triggers.bridge} does not exist. Refusing to change Orange Pi networking." >&2
  exit 1
@ -187,7 +222,6 @@ if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then
    if [ "$replace_existing" != "true" ]; then
      sudo "$qm_cmd" set "$vmid" \
        --agent enabled=1 \
-        --args "${self.triggers.args}" \
        --bios ovmf \
        --boot "order=scsi0;net0" \
        --cores "${self.triggers.cores}" \
@ -200,6 +234,9 @@ if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then
        --vga virtio
      sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
      sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
+      if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then
+        sudo "$qm_cmd" set "$vmid" --delete args
+      fi
      exit 0
    fi
    sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid"
@ -235,11 +272,13 @@ sudo "$qm_cmd" create "$vmid" \
  --vga virtio \
  --agent enabled=1

-sudo "$qm_cmd" set "$vmid" --args "${self.triggers.args}"
 sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
 sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
 sudo "$qm_cmd" set "$vmid" --agent enabled=1
-sudo "$qm_cmd" start "$vmid"
+if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then
+  sudo "$qm_cmd" set "$vmid" --delete args
+fi
+start_vm
 EOT
    ]
  }
@ -251,22 +290,22 @@ resource "null_resource" "pimox_template_vm_seal" {
  depends_on = [null_resource.pimox_template_vm_create]

  triggers = {
-    pimox_host      = var.pimox_host
-    pimox_user      = var.pimox_user
-    pimox_key_path  = var.pimox_ssh_key_path
-    pimox_qm_bin    = var.pimox_qm_bin
-    guest_host      = var.pimox_template_build_host
-    guest_user      = var.pimox_template_build_user
-    guest_key_path  = var.pimox_template_build_ssh_key_path
-    known_hosts_file = "${local.repo_root}/.lab/pimox-template-known_hosts"
-    seal_version    = "5"
-    create_id       = null_resource.pimox_template_vm_create[0].id
-    config_hash     = local.config_hash
-    timeout         = var.pimox_template_build_timeout
-    timeout_seconds = tostring(var.pimox_template_build_timeout_seconds)
+    pimox_host          = var.pimox_host
+    pimox_user          = var.pimox_user
+    pimox_key_path      = var.pimox_ssh_key_path
+    pimox_qm_bin        = var.pimox_qm_bin
+    guest_host          = var.pimox_template_build_host
+    guest_user          = var.pimox_template_build_user
+    guest_key_path      = var.pimox_template_build_ssh_key_path
+    known_hosts_file    = "${local.repo_root}/.lab/pimox-template-known_hosts"
+    seal_version        = "5"
+    create_id           = null_resource.pimox_template_vm_create[0].id
+    config_hash         = local.config_hash
+    timeout             = var.pimox_template_build_timeout
+    timeout_seconds     = tostring(var.pimox_template_build_timeout_seconds)
    ssh_timeout_seconds = tostring(var.pimox_template_guest_ssh_timeout_seconds)
-    guest_ip_prefix = var.pimox_template_guest_ip_prefix
-    vmid            = tostring(var.pimox_template_vmid)
+    guest_ip_prefix     = var.pimox_template_guest_ip_prefix
+    vmid                = tostring(var.pimox_template_vmid)
  }

  provisioner "local-exec" {
@ -434,7 +473,7 @@ resource "null_resource" "pimox_template_vm_finalize" {
    pimox_user        = var.pimox_user
    ssh_key_path      = var.pimox_ssh_key_path
    qm_bin            = var.pimox_qm_bin
-    finalizer_version = "4"
+    finalizer_version = "5"
    config_hash       = local.config_hash
    seal_id           = null_resource.pimox_template_vm_seal[0].id
    vmid              = tostring(var.pimox_template_vmid)
@ -466,6 +505,9 @@ fi

 if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then
  sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0"
+  if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then
+    sudo "$qm_cmd" set "$vmid" --delete args
+  fi
  exit 0
 fi

@ -484,6 +526,9 @@ if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then
 fi

 sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0"
+if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then
+  sudo "$qm_cmd" set "$vmid" --delete args
+fi
 sudo "$qm_cmd" template "$vmid"
 EOT
    ]
--- a/bootstrap/provisioning/variables.tf
+++ b/bootstrap/provisioning/variables.tf
@ -209,6 +209,12 @@ variable "pimox_template_memory" {
  default = 8192
 }

+variable "pimox_template_kvm_cpuset" {
+  type        = string
+  default     = ""
+  description = "Optional Pimox host CPU set used when launching the generated KVM command. Jeannie defaults this to 4-7 for the Orange Pi 5 Plus."
+}
+
 variable "pimox_template_bridge" {
  type    = string
  default = "vmbr0"
--- a/255
+++ b/255
@ -1740,6 +1740,100 @@ pimox_worker_cpu_affinity() {
    exit 1
 }

+pimox_start_vm_with_cpuset() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local qm_bin="$4"
+    local vmid="$5"
+    local cpuset="$6"
+
+    if [[ -z "${cpuset}" ]]; then
+        pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'"
+        return
+    fi
+    if ! cpuset_cpu_count "${cpuset}" >/dev/null; then
+        echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2
+        exit 1
+    fi
+
+    # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated
+    # command under taskset so PSCI brings secondary guest CPUs online reliably.
+    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
+vmid='${vmid}'
+qm_bin='${qm_bin}'
+cpuset='${cpuset}'
+
+if ! command -v taskset >/dev/null 2>&1; then
+  echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2
+  exit 1
+fi
+if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then
+  echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2
+  exit 1
+fi
+
+start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\")
+cleanup() {
+  rm -f \"\$start_script\"
+}
+trap cleanup 0
+
+sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\"
+if ! awk -v vmid=\"\$vmid\" '
+  NR == 1 && (\$1 == "/usr/bin/kvm" || \$1 ~ /qemu-system-aarch64$/) { kvm = 1 }
+  \$1 == "-id" && \$2 == vmid { id = 1 }
+  END { exit (kvm && id) ? 0 : 1 }
+' \"\$start_script\"; then
+  echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2
+  exit 1
+fi
+smp_count=\$(awk '\$1 == "-smp" { count++ } END { print count + 0 }' \"\$start_script\")
+if [ \"\$smp_count\" -ne 1 ]; then
+  echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2
+  exit 1
+fi
+
+sudo taskset -c \"\$cpuset\" bash \"\$start_script\""
+}
+
+pimox_shutdown_vm_gracefully() {
+    local host="$1"
+    local user="$2"
+    local key_path="$3"
+    local qm_bin="$4"
+    local vmid="$5"
+
+    pimox_ssh "${host}" "${user}" "${key_path}" "set -eu
+sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
+elapsed=0
+while [ \"\$elapsed\" -lt 300 ]; do
+  if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
+    exit 0
+  fi
+  sleep 5
+  elapsed=\$((elapsed + 5))
+done
+echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2
+exit 1"
+}
+
+pimox_guest_cpu_count() {
+    local guest_ip="$1"
+    local guest_user="$2"
+    local guest_key_path="$3"
+    local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts"
+
+    ssh -i "${guest_key_path}" \
+        -o BatchMode=yes \
+        -o ConnectTimeout=10 \
+        -o IdentitiesOnly=yes \
+        -o StrictHostKeyChecking=accept-new \
+        -o UserKnownHostsFile="${known_hosts_file}" \
+        "${guest_user}@${guest_ip}" \
+        'nproc --all'
+}
+
 ensure_pimox_worker_node() {
    local index="$1"
    local spec_file="$2"
@ -1762,12 +1856,12 @@ ensure_pimox_worker_node() {
    local worker_storage="${19}"
    local worker_replace_existing="${20}"
    local worker_cpu_affinity="${21}"
+    local worker_kvm_cpuset="${22}"
    local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}"
    local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}"
    local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}"
    local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}"
    local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}"
-    local worker_args
    local padded
    local vmid
    local worker_key
@ -1775,6 +1869,7 @@ ensure_pimox_worker_node() {
    local node_name
    local net0_config
    local guest_ip
+    local guest_cpu_count
    local static_ip

    printf -v padded '%02d' "${index}"
@ -1785,8 +1880,6 @@ ensure_pimox_worker_node() {
    if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
        exit 1
    fi
-    worker_args="-smp ${worker_cores},sockets=1,cores=${worker_cores},threads=1"
-
    if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
        if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
            echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2
@ -1812,9 +1905,16 @@ done
 sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
        else
            pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
+was_running=false
+had_legacy_args=false
+if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then
+  was_running=true
+fi
+if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
+  had_legacy_args=true
+fi
 sudo '${qm_bin}' set '${vmid}' \
  --agent enabled=1 \
-  --args '${worker_args}' \
  --bios ovmf \
  --boot 'order=scsi0;net0' \
  --cores '${worker_cores}' \
@ -1825,6 +1925,9 @@ sudo '${qm_bin}' set '${vmid}' \
  --scsihw virtio-scsi-pci \
  --sockets 1 \
  --vga virtio
+if [ \"\$had_legacy_args\" = true ]; then
+  sudo '${qm_bin}' set '${vmid}' --delete args
+fi
 if [ -n '${worker_cpu_affinity}' ]; then
  affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
    case \"\$affinity_output\" in
@ -1838,7 +1941,22 @@ if [ -n '${worker_cpu_affinity}' ]; then
    esac
  }
 fi
-if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi"
+if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then
+  echo 'Restarting VM ${vmid} to apply removal of legacy qm args.'
+  sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true
+  elapsed=0
+  while [ \"\$elapsed\" -lt 300 ]; do
+    if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
+      break
+    fi
+    sleep 5
+    elapsed=\$((elapsed + 5))
+  done
+  if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then
+    echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2
+    exit 1
+  fi
+fi"
        fi
    fi

@ -1863,7 +1981,6 @@ fi
 sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}'
 sudo '${qm_bin}' set '${vmid}' \
  --agent enabled=1 \
-  --args '${worker_args}' \
  --bios ovmf \
  --boot 'order=scsi0;net0' \
  --cores '${worker_cores}' \
@ -1874,6 +1991,9 @@ sudo '${qm_bin}' set '${vmid}' \
  --scsihw virtio-scsi-pci \
  --sockets 1 \
  --vga virtio
+if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then
+  sudo '${qm_bin}' set '${vmid}' --delete args
+fi
 if [ -n '${worker_cpu_affinity}' ]; then
  affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
    case \"\$affinity_output\" in
@ -1887,8 +2007,14 @@ if [ -n '${worker_cpu_affinity}' ]; then
    esac
 }
 fi
-sudo '${qm_bin}' set '${vmid}' --onboot 1
-sudo '${qm_bin}' start '${vmid}'"
+sudo '${qm_bin}' set '${vmid}' --onboot 1"
+    fi
+
+    if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then
+        pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
+    elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then
+        echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2
+        exit 1
    fi

    case "${worker_network_mode}" in
@ -1930,6 +2056,23 @@ sudo '${qm_bin}' start '${vmid}'"
        exit 1
    fi

+    guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
+    if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
+        echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2
+        pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}"
+        pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}"
+        if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then
+            echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2
+            exit 1
+        fi
+        guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)"
+    fi
+    if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then
+        echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2
+        echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2
+        exit 1
+    fi
+
    printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}"
 }

@ -2458,6 +2601,9 @@ run_pimox_pipeline() {
    local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}"
    local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}"
    local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}"
+    local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}"
+    local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}"
+    local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}"
    local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}"
    local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"
    local worker_user="${LAB_PIMOX_WORKER_USER:-jv}"
@ -2471,6 +2617,10 @@ run_pimox_pipeline() {
    local readiness_output
    local readiness_status
    local worker_cpu_affinity
+    local cpuset_cpu_total
+    local cpuset_name
+    local cpuset_value
+    local required_cores

    if disabled_value "${mode}"; then
        return 0
@ -2496,6 +2646,24 @@ run_pimox_pipeline() {
        echo "Pimox template and worker CPU and memory values must be greater than zero." >&2
        exit 1
    fi
+    for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do
+        cpuset_value="${!cpuset_name}"
+        if [[ -z "${cpuset_value}" ]]; then
+            continue
+        fi
+        if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then
+            echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2
+            exit 1
+        fi
+        required_cores="${template_cores}"
+        if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then
+            required_cores="${worker_cores}"
+        fi
+        if ((cpuset_cpu_total < required_cores)); then
+            echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2
+            exit 1
+        fi
+    done
    if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then
        echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2
        exit 1
@ -2572,6 +2740,7 @@ fi" 2>&1)"
    export TF_VAR_pimox_template_cores="${template_cores}"
    export TF_VAR_pimox_template_memory="${template_memory}"
    export TF_VAR_pimox_template_replace_existing="${template_replace_existing}"
+    export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}"
    export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}"
    export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}"
    export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}"
@ -2625,7 +2794,8 @@ fi" 2>&1)"
            "${qm_bin}" \
            "${worker_storage}" \
            "${worker_replace_existing}" \
-            "${worker_cpu_affinity}"
+            "${worker_cpu_affinity}" \
+            "${worker_kvm_cpuset}"
    done

    write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}"
@ -4976,6 +5146,70 @@ EOF
    echo "Ollama homelab setup is ready."
 }

+require_argocd_application_controller() {
+    local namespace="${TF_VAR_argocd_namespace:-argocd}"
+    local expected_resources="10m|1|2Gi"
+    local actual_resources
+
+    if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then
+        echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2
+        kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pods -o wide >&2 || true
+        kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true
+        exit 1
+    fi
+
+    actual_resources="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get statefulset argocd-application-controller \
+        -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')"
+    if [[ "${actual_resources}" != "${expected_resources}" ]]; then
+        echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2
+        echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2
+        exit 1
+    fi
+}
+
+require_pimox_app_worker_capacity() {
+    local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}"
+    local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv"
+    local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}"
+    local worker_key
+    local worker_host
+    local worker_user
+    local worker_node
+    local worker_key_path
+    local capacity
+    local ready
+    local failures=0
+
+    if disabled_value "${enabled}"; then
+        echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}."
+        return 0
+    fi
+    if [[ ! -s "${spec_file}" ]]; then
+        echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate."
+        return 0
+    fi
+    if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then
+        echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2
+        exit 1
+    fi
+
+    while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do
+        [[ -n "${worker_node}" ]] || continue
+        capacity="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)"
+        ready="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
+        if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then
+            echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2
+            failures=$((failures + 1))
+        fi
+    done <"${spec_file}"
+
+    if ((failures > 0)); then
+        echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2
+        kubectl --kubeconfig "${KUBECONFIG}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true
+        exit 1
+    fi
+}
+
 apps() {
    local buildx_builder_ready=false
    local demos_image_built=false
@ -5008,6 +5242,9 @@ apps() {
    export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}"
    export KUBECONFIG="${TF_VAR_kubeconfig_path}"

+    require_argocd_application_controller
+    require_pimox_app_worker_capacity
+
    ensure_cosign_keypair
    publish_cosign_public_key_config_map
Author	SHA1	Message	Date
juvenal.diaz	d6e33250f4	Gates for pimoc workers with less than 2 cpus	2026-07-23 13:16:11 -06:00
juvenal.diaz	fb605ffa8b	Increase argocd limits	2026-07-23 11:32:23 -06:00