diff --git a/README.md b/README.md index 572dd5f..07bcb59 100644 --- a/README.md +++ b/README.md @@ -718,7 +718,11 @@ template and worker VM clones automatically when the Orange Pi storage path is healthy. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, -and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe +and KVM launch affinity `4-7` because this Pimox is pinned to Debian Bullseye +on an Orange Pi 5 Plus. Jeannie launches the generated KVM command under that +host CPU set so PSCI can bring both guest CPUs online; it removes legacy `args` +entries so native `qm` CPU topology is the only `-smp` argument. Set +`LAB_PIMOX_KVM_CPUSET=` to use normal `qm start` on a different host. The NVMe that was previously attached to the Orange Pi now belongs to the HP Debian laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox datastore is `opi5_ssd` and is used for rebuildable VM root disks. Details and @@ -759,6 +763,11 @@ qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by default and lets SSH be the reachability check. Set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate. `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout. +Before `./jeannie apps` changes any application resources, Jeannie requires +every generated Pimox worker to be `Ready` and expose at least two Kubernetes +CPUs. Override the threshold with `LAB_PIMOX_APP_DEPLOY_MIN_CPUS`; set +`LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=false` only for a deliberately +non-Pimox deployment. Long-running `./jeannie up` steps keep compact output by default, but the active step line refreshes with elapsed time and the latest log line. Set @@ -778,8 +787,10 @@ Run a full cluster rebuild from the Debian server with: That path preserves external Gitea, rebuilds the Pimox template with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and -8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is -disabled by default because the Bullseye-pinned Pimox `qm` does not support it. +8 GiB memory, and joins those workers to the Kubernetes cluster. Pimox 7 ARM +does not support `qm --affinity`; Jeannie instead starts the generated KVM +command under CPU set `4-7` and verifies every worker reports two guest CPUs +before Kubernetes workloads proceed. The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. diff --git a/README.md.tmpl b/README.md.tmpl index ce74917..c1089a5 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -718,7 +718,11 @@ template and worker VM clones automatically when the Orange Pi storage path is healthy. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, -and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe +and KVM launch affinity `4-7` because this Pimox is pinned to Debian Bullseye +on an Orange Pi 5 Plus. Jeannie launches the generated KVM command under that +host CPU set so PSCI can bring both guest CPUs online; it removes legacy `args` +entries so native `qm` CPU topology is the only `-smp` argument. Set +`LAB_PIMOX_KVM_CPUSET=` to use normal `qm start` on a different host. The NVMe that was previously attached to the Orange Pi now belongs to the HP Debian laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox datastore is `opi5_ssd` and is used for rebuildable VM root disks. Details and @@ -759,6 +763,12 @@ qemu-guest-agent reports an IP, Debian skips the extra ICMP host probe by default and lets SSH be the reachability check. Set `LAB_PIMOX_WORKER_REQUIRE_HOST_PROBE=true` only when you want a faster ICMP gate. `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` remains the broader template-build timeout. +Before `./{{ main_script }} apps` changes any application resources, Jeannie +requires every generated Pimox worker to be `Ready` and expose at least two +Kubernetes CPUs. Override the threshold with +`LAB_PIMOX_APP_DEPLOY_MIN_CPUS`; set +`LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=false` only for a deliberately +non-Pimox deployment. Long-running `./{{ main_script }} up` steps keep compact output by default, but the active step line refreshes with elapsed time and the latest log line. Set @@ -778,8 +788,10 @@ Run a full cluster rebuild from the Debian server with: That path preserves external Gitea, rebuilds the Pimox template with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and -8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is -disabled by default because the Bullseye-pinned Pimox `qm` does not support it. +8 GiB memory, and joins those workers to the Kubernetes cluster. Pimox 7 ARM +does not support `qm --affinity`; Jeannie instead starts the generated KVM +command under CPU set `4-7` and verifies every worker reports two guest CPUs +before Kubernetes workloads proceed. The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf index 1713b3d..244d654 100644 --- a/bootstrap/platform/main.tf +++ b/bootstrap/platform/main.tf @@ -1217,11 +1217,12 @@ resource "null_resource" "argocd_ready" { depends_on = [helm_release.argocd] triggers = { - kubeconfig_path = var.kubeconfig_path - namespace = var.argocd.namespace - version = var.argocd.version - node_selector = jsonencode(local.argocd_node_selector) - readiness_check = "7" + kubeconfig_path = var.kubeconfig_path + namespace = var.argocd.namespace + version = var.argocd.version + node_selector = jsonencode(local.argocd_node_selector) + controller_resource_profile = "10m|1|2Gi" + readiness_check = "8" } provisioner "local-exec" { @@ -1273,6 +1274,15 @@ wait_for_argocd_component() { kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Established --timeout=180s crd/applications.argoproj.io wait_for_argocd_component deployment/argocd-applicationset-controller app.kubernetes.io/name=argocd-applicationset-controller wait_for_argocd_component statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller + +controller_resources="$(kubectl --kubeconfig "$kubeconfig" -n "$namespace" get statefulset argocd-application-controller \ + -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')" +if [ "$controller_resources" != "${self.triggers.controller_resource_profile}" ]; then + echo "Argo CD application controller resources are '$controller_resources', expected '${self.triggers.controller_resource_profile}'." >&2 + dump_argocd_readiness_debug statefulset/argocd-application-controller app.kubernetes.io/name=argocd-application-controller + exit 1 +fi + wait_for_argocd_component deployment/argocd-dex-server app.kubernetes.io/name=argocd-dex-server wait_for_argocd_component deployment/argocd-notifications-controller app.kubernetes.io/name=argocd-notifications-controller wait_for_argocd_component deployment/argocd-redis app.kubernetes.io/name=argocd-redis diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 1e5b6da..27888eb 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -128,7 +128,9 @@ template build timeout. Defaults match the observed Pimox template VM shape: OVMF firmware, virtio networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with -2 cores, 8 GiB memory, and explicit `args: -smp 2,sockets=1,cores=2,threads=1`. +2 cores, and 8 GiB memory. The native `qm` CPU topology is the only `-smp` +argument; the provisioning stack removes legacy `args` values to prevent a +second conflicting `-smp` option. Override `TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. @@ -172,6 +174,15 @@ Worker clone failures are intentionally faster than template build failures: `LAB_PIMOX_WORKER_WAIT_TIMEOUT_SECONDS=600`, and `LAB_PIMOX_GUEST_SSH_TIMEOUT_SECONDS=600` by default. The broader `LAB_PIMOX_GUEST_TIMEOUT_SECONDS` still controls template preparation. + +On the Orange Pi 5 Plus with Pimox 7 ARM, Jeannie launches the generated KVM +command under host CPU set `4-7`. This keeps the guest's secondary CPU on the +compatible RK3588 cores during PSCI boot. The same setting is used for the +template build and worker clones, and each worker must report its configured +CPU count through SSH before it is added to the Kubernetes worker inventory. +Set `LAB_PIMOX_KVM_CPUSET=` to use the normal `qm start` path on a different +Pimox host, or override template and worker values separately with +`LAB_PIMOX_TEMPLATE_KVM_CPUSET` and `LAB_PIMOX_WORKER_KVM_CPUSET`. Static guest-network validation skips the extra LAN ICMP probe by default because the ISP modem may not answer worker VM pings. Set `LAB_PIMOX_WORKER_REQUIRE_LAN_PROBE=true` only when the gateway/probe host diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index 1f4d34e..2a0da2d 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -47,7 +47,6 @@ locals { node_dns_servers = join(" ", var.node_dns_servers) kernel_cgroup_boot_options = join(" ", var.kernel_cgroup_boot_options) pimox_template_net0 = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}" - pimox_template_args = "-smp ${var.pimox_template_cores},sockets=1,cores=${var.pimox_template_cores},threads=1" template_package_list = join(" ", local.template_packages) provisioning_http_base_url = "http://${var.http_host}:${var.http_port}" provisioning_script_url = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh" @@ -134,7 +133,7 @@ resource "null_resource" "pimox_template_vm_create" { pimox_user = var.pimox_user ssh_key_path = var.pimox_ssh_key_path qm_bin = var.pimox_qm_bin - builder_version = "5" + builder_version = "6" config_hash = local.config_hash vmid = tostring(var.pimox_template_vmid) name = var.pimox_template_name @@ -142,7 +141,7 @@ resource "null_resource" "pimox_template_vm_create" { memory = tostring(var.pimox_template_memory) bridge = var.pimox_template_bridge net0 = local.pimox_template_net0 - args = local.pimox_template_args + kvm_cpuset = var.pimox_template_kvm_cpuset scsi0 = var.pimox_template_scsi0 efidisk0 = var.pimox_template_efidisk0 } @@ -177,6 +176,42 @@ if ! sudo -n true >/dev/null 2>&1; then exit 1 fi +start_vm() { + cpuset="${self.triggers.kvm_cpuset}" + if [ -z "$cpuset" ]; then + sudo "$qm_cmd" start "$vmid" + return + fi + if ! command -v taskset >/dev/null 2>&1; then + echo "taskset is required to start this Pimox VM with a KVM CPU set" >&2 + exit 1 + fi + if ! taskset -c "$cpuset" true >/dev/null 2>&1; then + echo "Pimox KVM CPU set $cpuset is not valid on this host" >&2 + exit 1 + fi + + start_script="$(mktemp "/tmp/homelab-qemu-$vmid.XXXXXX")" + trap 'rm -f "$start_script"' 0 + + sudo "$qm_cmd" showcmd "$vmid" --pretty >"$start_script" + if ! awk -v vmid="$vmid" ' + NR == 1 && ($1 == "/usr/bin/kvm" || $1 ~ /qemu-system-aarch64$/) { kvm = 1 } + $1 == "-id" && $2 == vmid { id = 1 } + END { exit (kvm && id) ? 0 : 1 } + ' "$start_script"; then + echo "qm showcmd for VM $vmid did not produce the expected KVM command" >&2 + exit 1 + fi + smp_count="$(awk '$1 == "-smp" { count++ } END { print count + 0 }' "$start_script")" + if [ "$smp_count" -ne 1 ]; then + echo "Expected one native -smp argument for VM $vmid, found $smp_count. Remove stale qm args before starting." >&2 + exit 1 + fi + + sudo taskset -c "$cpuset" bash "$start_script" +} + if ! ip link show "${self.triggers.bridge}" >/dev/null 2>&1; then echo "Pimox bridge ${self.triggers.bridge} does not exist. Refusing to change Orange Pi networking." >&2 exit 1 @@ -187,7 +222,6 @@ if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then if [ "$replace_existing" != "true" ]; then sudo "$qm_cmd" set "$vmid" \ --agent enabled=1 \ - --args "${self.triggers.args}" \ --bios ovmf \ --boot "order=scsi0;net0" \ --cores "${self.triggers.cores}" \ @@ -200,6 +234,9 @@ if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then --vga virtio sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" + if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then + sudo "$qm_cmd" set "$vmid" --delete args + fi exit 0 fi sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid" @@ -235,11 +272,13 @@ sudo "$qm_cmd" create "$vmid" \ --vga virtio \ --agent enabled=1 -sudo "$qm_cmd" set "$vmid" --args "${self.triggers.args}" sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --agent enabled=1 -sudo "$qm_cmd" start "$vmid" +if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then + sudo "$qm_cmd" set "$vmid" --delete args +fi +start_vm EOT ] } @@ -251,22 +290,22 @@ resource "null_resource" "pimox_template_vm_seal" { depends_on = [null_resource.pimox_template_vm_create] triggers = { - pimox_host = var.pimox_host - pimox_user = var.pimox_user - pimox_key_path = var.pimox_ssh_key_path - pimox_qm_bin = var.pimox_qm_bin - guest_host = var.pimox_template_build_host - guest_user = var.pimox_template_build_user - guest_key_path = var.pimox_template_build_ssh_key_path - known_hosts_file = "${local.repo_root}/.lab/pimox-template-known_hosts" - seal_version = "5" - create_id = null_resource.pimox_template_vm_create[0].id - config_hash = local.config_hash - timeout = var.pimox_template_build_timeout - timeout_seconds = tostring(var.pimox_template_build_timeout_seconds) + pimox_host = var.pimox_host + pimox_user = var.pimox_user + pimox_key_path = var.pimox_ssh_key_path + pimox_qm_bin = var.pimox_qm_bin + guest_host = var.pimox_template_build_host + guest_user = var.pimox_template_build_user + guest_key_path = var.pimox_template_build_ssh_key_path + known_hosts_file = "${local.repo_root}/.lab/pimox-template-known_hosts" + seal_version = "5" + create_id = null_resource.pimox_template_vm_create[0].id + config_hash = local.config_hash + timeout = var.pimox_template_build_timeout + timeout_seconds = tostring(var.pimox_template_build_timeout_seconds) ssh_timeout_seconds = tostring(var.pimox_template_guest_ssh_timeout_seconds) - guest_ip_prefix = var.pimox_template_guest_ip_prefix - vmid = tostring(var.pimox_template_vmid) + guest_ip_prefix = var.pimox_template_guest_ip_prefix + vmid = tostring(var.pimox_template_vmid) } provisioner "local-exec" { @@ -434,7 +473,7 @@ resource "null_resource" "pimox_template_vm_finalize" { pimox_user = var.pimox_user ssh_key_path = var.pimox_ssh_key_path qm_bin = var.pimox_qm_bin - finalizer_version = "4" + finalizer_version = "5" config_hash = local.config_hash seal_id = null_resource.pimox_template_vm_seal[0].id vmid = tostring(var.pimox_template_vmid) @@ -466,6 +505,9 @@ fi if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0" + if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then + sudo "$qm_cmd" set "$vmid" --delete args + fi exit 0 fi @@ -484,6 +526,9 @@ if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then fi sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0" +if sudo "$qm_cmd" config "$vmid" | grep -q '^args:'; then + sudo "$qm_cmd" set "$vmid" --delete args +fi sudo "$qm_cmd" template "$vmid" EOT ] diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index e908778..028373f 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -209,6 +209,12 @@ variable "pimox_template_memory" { default = 8192 } +variable "pimox_template_kvm_cpuset" { + type = string + default = "" + description = "Optional Pimox host CPU set used when launching the generated KVM command. Jeannie defaults this to 4-7 for the Orange Pi 5 Plus." +} + variable "pimox_template_bridge" { type = string default = "vmbr0" diff --git a/jeannie b/jeannie index 5b7de02..04ca5aa 100755 --- a/jeannie +++ b/jeannie @@ -1740,6 +1740,100 @@ pimox_worker_cpu_affinity() { exit 1 } +pimox_start_vm_with_cpuset() { + local host="$1" + local user="$2" + local key_path="$3" + local qm_bin="$4" + local vmid="$5" + local cpuset="$6" + + if [[ -z "${cpuset}" ]]; then + pimox_ssh "${host}" "${user}" "${key_path}" "sudo '${qm_bin}' start '${vmid}'" + return + fi + if ! cpuset_cpu_count "${cpuset}" >/dev/null; then + echo "Invalid Pimox KVM CPU set '${cpuset}'. Use CPU IDs or ranges, such as 4-7." >&2 + exit 1 + fi + + # Pimox 7 daemonizes qm start before KVM is spawned. Launch qm's generated + # command under taskset so PSCI brings secondary guest CPUs online reliably. + pimox_ssh "${host}" "${user}" "${key_path}" "set -eu +vmid='${vmid}' +qm_bin='${qm_bin}' +cpuset='${cpuset}' + +if ! command -v taskset >/dev/null 2>&1; then + echo 'taskset is required to start this Pimox VM with a KVM CPU set' >&2 + exit 1 +fi +if ! taskset -c \"\$cpuset\" true >/dev/null 2>&1; then + echo \"Pimox KVM CPU set \$cpuset is not valid on this host\" >&2 + exit 1 +fi + +start_script=\$(mktemp \"/tmp/homelab-qemu-\${vmid}.XXXXXX\") +cleanup() { + rm -f \"\$start_script\" +} +trap cleanup 0 + +sudo \"\$qm_bin\" showcmd \"\$vmid\" --pretty >\"\$start_script\" +if ! awk -v vmid=\"\$vmid\" ' + NR == 1 && (\$1 == "/usr/bin/kvm" || \$1 ~ /qemu-system-aarch64$/) { kvm = 1 } + \$1 == "-id" && \$2 == vmid { id = 1 } + END { exit (kvm && id) ? 0 : 1 } +' \"\$start_script\"; then + echo \"qm showcmd for VM \$vmid did not produce the expected KVM command\" >&2 + exit 1 +fi +smp_count=\$(awk '\$1 == "-smp" { count++ } END { print count + 0 }' \"\$start_script\") +if [ \"\$smp_count\" -ne 1 ]; then + echo \"Expected one native -smp argument for VM \$vmid, found \$smp_count. Remove stale qm args before starting.\" >&2 + exit 1 +fi + +sudo taskset -c \"\$cpuset\" bash \"\$start_script\"" +} + +pimox_shutdown_vm_gracefully() { + local host="$1" + local user="$2" + local key_path="$3" + local qm_bin="$4" + local vmid="$5" + + pimox_ssh "${host}" "${user}" "${key_path}" "set -eu +sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true +elapsed=0 +while [ \"\$elapsed\" -lt 300 ]; do + if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + exit 0 + fi + sleep 5 + elapsed=\$((elapsed + 5)) +done +echo 'VM ${vmid} did not stop gracefully within 300 seconds.' >&2 +exit 1" +} + +pimox_guest_cpu_count() { + local guest_ip="$1" + local guest_user="$2" + local guest_key_path="$3" + local known_hosts_file="${REPO_ROOT}/.lab/pimox-worker-known_hosts" + + ssh -i "${guest_key_path}" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=accept-new \ + -o UserKnownHostsFile="${known_hosts_file}" \ + "${guest_user}@${guest_ip}" \ + 'nproc --all' +} + ensure_pimox_worker_node() { local index="$1" local spec_file="$2" @@ -1762,12 +1856,12 @@ ensure_pimox_worker_node() { local worker_storage="${19}" local worker_replace_existing="${20}" local worker_cpu_affinity="${21}" + local worker_kvm_cpuset="${22}" local worker_network_mode="${LAB_PIMOX_WORKER_NETWORK_MODE:-static}" local worker_static_ips="${LAB_PIMOX_WORKER_STATIC_IPS:-192.168.100.66 192.168.100.67 192.168.100.76}" local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" - local worker_args local padded local vmid local worker_key @@ -1775,6 +1869,7 @@ ensure_pimox_worker_node() { local node_name local net0_config local guest_ip + local guest_cpu_count local static_ip printf -v padded '%02d' "${index}" @@ -1785,8 +1880,6 @@ ensure_pimox_worker_node() { if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then exit 1 fi - worker_args="-smp ${worker_cores},sockets=1,cores=${worker_cores},threads=1" - if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then echo "VM ${vmid} exists as a template; refusing to reuse it as worker ${worker_name}." >&2 @@ -1812,9 +1905,16 @@ done sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" else pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +was_running=false +had_legacy_args=false +if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'; then + was_running=true +fi +if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then + had_legacy_args=true +fi sudo '${qm_bin}' set '${vmid}' \ --agent enabled=1 \ - --args '${worker_args}' \ --bios ovmf \ --boot 'order=scsi0;net0' \ --cores '${worker_cores}' \ @@ -1825,6 +1925,9 @@ sudo '${qm_bin}' set '${vmid}' \ --scsihw virtio-scsi-pci \ --sockets 1 \ --vga virtio +if [ \"\$had_legacy_args\" = true ]; then + sudo '${qm_bin}' set '${vmid}' --delete args +fi if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in @@ -1838,7 +1941,22 @@ if [ -n '${worker_cpu_affinity}' ]; then esac } fi -if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi" +if [ \"\$had_legacy_args\" = true ] && [ \"\$was_running\" = true ]; then + echo 'Restarting VM ${vmid} to apply removal of legacy qm args.' + sudo '${qm_bin}' shutdown '${vmid}' --timeout 120 || true + elapsed=0 + while [ \"\$elapsed\" -lt 300 ]; do + if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + break + fi + sleep 5 + elapsed=\$((elapsed + 5)) + done + if ! sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then + echo 'VM ${vmid} did not stop after removing legacy qm args.' >&2 + exit 1 + fi +fi" fi fi @@ -1863,7 +1981,6 @@ fi sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' set '${vmid}' \ --agent enabled=1 \ - --args '${worker_args}' \ --bios ovmf \ --boot 'order=scsi0;net0' \ --cores '${worker_cores}' \ @@ -1874,6 +1991,9 @@ sudo '${qm_bin}' set '${vmid}' \ --scsihw virtio-scsi-pci \ --sockets 1 \ --vga virtio +if sudo '${qm_bin}' config '${vmid}' | grep -q '^args:'; then + sudo '${qm_bin}' set '${vmid}' --delete args +fi if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in @@ -1887,8 +2007,14 @@ if [ -n '${worker_cpu_affinity}' ]; then esac } fi -sudo '${qm_bin}' set '${vmid}' --onboot 1 -sudo '${qm_bin}' start '${vmid}'" +sudo '${qm_bin}' set '${vmid}' --onboot 1" + fi + + if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'"; then + pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" + elif ! pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' | grep -q 'status: running'"; then + echo "Pimox worker VM ${vmid} (${worker_name}) is neither stopped nor running after configuration." >&2 + exit 1 fi case "${worker_network_mode}" in @@ -1930,6 +2056,23 @@ sudo '${qm_bin}' start '${vmid}'" exit 1 fi + guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" + if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then + echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); restarting through the pinned KVM launch path." >&2 + pimox_shutdown_vm_gracefully "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" + pimox_start_vm_with_cpuset "${pimox_host}" "${pimox_user}" "${pimox_key}" "${qm_bin}" "${vmid}" "${worker_kvm_cpuset}" + if ! guest_ip="$(wait_for_pimox_guest_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "${vmid}" "${worker_user}" "${worker_key_path}" "${ip_prefix}" "${timeout_seconds}" "${qm_bin}")"; then + echo "Timed out waiting for worker VM ${vmid} (${worker_name}) after pinned KVM restart." >&2 + exit 1 + fi + guest_cpu_count="$(pimox_guest_cpu_count "${guest_ip}" "${worker_user}" "${worker_key_path}" 2>/dev/null || true)" + fi + if ! [[ "${guest_cpu_count}" =~ ^[0-9]+$ ]] || ((guest_cpu_count < worker_cores)); then + echo "Pimox worker VM ${vmid} (${worker_name}) reports ${guest_cpu_count:-unknown} CPU(s); expected at least ${worker_cores}." >&2 + echo "On Pimox 7 ARM this means KVM was not launched on a compatible host CPU set. Check LAB_PIMOX_WORKER_KVM_CPUSET (default: 4-7)." >&2 + exit 1 + fi + printf '%s\t%s\t%s\t%s\t%s\n' "${worker_key}" "${guest_ip}" "${worker_user}" "${node_name}" "${worker_key_path}" >>"${spec_file}" } @@ -2458,6 +2601,9 @@ run_pimox_pipeline() { local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}" local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" + local pimox_kvm_cpuset="${LAB_PIMOX_KVM_CPUSET:-4-7}" + local template_kvm_cpuset="${LAB_PIMOX_TEMPLATE_KVM_CPUSET:-${pimox_kvm_cpuset}}" + local worker_kvm_cpuset="${LAB_PIMOX_WORKER_KVM_CPUSET:-${pimox_kvm_cpuset}}" local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" @@ -2471,6 +2617,10 @@ run_pimox_pipeline() { local readiness_output local readiness_status local worker_cpu_affinity + local cpuset_cpu_total + local cpuset_name + local cpuset_value + local required_cores if disabled_value "${mode}"; then return 0 @@ -2496,6 +2646,24 @@ run_pimox_pipeline() { echo "Pimox template and worker CPU and memory values must be greater than zero." >&2 exit 1 fi + for cpuset_name in template_kvm_cpuset worker_kvm_cpuset; do + cpuset_value="${!cpuset_name}" + if [[ -z "${cpuset_value}" ]]; then + continue + fi + if ! cpuset_cpu_total="$(cpuset_cpu_count "${cpuset_value}")"; then + echo "${cpuset_name}='${cpuset_value}' is invalid. Use CPU IDs or ranges, such as 4-7." >&2 + exit 1 + fi + required_cores="${template_cores}" + if [[ "${cpuset_name}" == "worker_kvm_cpuset" ]]; then + required_cores="${worker_cores}" + fi + if ((cpuset_cpu_total < required_cores)); then + echo "${cpuset_name}='${cpuset_value}' has ${cpuset_cpu_total} CPU(s), but ${required_cores} are required." >&2 + exit 1 + fi + done if ! [[ "${timeout_seconds}" =~ ^[0-9]+$ && "${timeout_seconds}" -gt 0 ]]; then echo "LAB_PIMOX_GUEST_TIMEOUT_SECONDS must be a positive integer." >&2 exit 1 @@ -2572,6 +2740,7 @@ fi" 2>&1)" export TF_VAR_pimox_template_cores="${template_cores}" export TF_VAR_pimox_template_memory="${template_memory}" export TF_VAR_pimox_template_replace_existing="${template_replace_existing}" + export TF_VAR_pimox_template_kvm_cpuset="${template_kvm_cpuset}" export TF_VAR_pimox_template_builder_enabled="${TF_VAR_pimox_template_builder_enabled:-true}" export TF_VAR_pimox_template_build_ssh_key_path="${TF_VAR_pimox_template_build_ssh_key_path:-${worker_key_path}}" export TF_VAR_pimox_template_build_user="${TF_VAR_pimox_template_build_user:-${worker_user}}" @@ -2625,7 +2794,8 @@ fi" 2>&1)" "${qm_bin}" \ "${worker_storage}" \ "${worker_replace_existing}" \ - "${worker_cpu_affinity}" + "${worker_cpu_affinity}" \ + "${worker_kvm_cpuset}" done write_cluster_worker_var_file "${var_file}" "${REPO_ROOT}/.lab/manual-workers.tsv" "${spec_file}" @@ -4976,6 +5146,70 @@ EOF echo "Ollama homelab setup is ready." } +require_argocd_application_controller() { + local namespace="${TF_VAR_argocd_namespace:-argocd}" + local expected_resources="10m|1|2Gi" + local actual_resources + + if ! kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" rollout status statefulset/argocd-application-controller --timeout=300s; then + echo "Argo CD application controller did not become ready; refusing to start application deployment." >&2 + kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get pods -o wide >&2 || true + kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" describe statefulset argocd-application-controller >&2 || true + exit 1 + fi + + actual_resources="$(kubectl --kubeconfig "${KUBECONFIG}" -n "${namespace}" get statefulset argocd-application-controller \ + -o jsonpath='{.spec.template.spec.containers[0].resources.requests.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.cpu}{"|"}{.spec.template.spec.containers[0].resources.limits.memory}')" + if [[ "${actual_resources}" != "${expected_resources}" ]]; then + echo "Argo CD application controller resources are '${actual_resources:-missing}', expected '${expected_resources}'. Refusing to start application deployment." >&2 + echo "Run ./jeannie platform after correcting bootstrap/platform/main.tf." >&2 + exit 1 + fi +} + +require_pimox_app_worker_capacity() { + local enabled="${LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY:-true}" + local spec_file="${REPO_ROOT}/.lab/pimox-workers.tsv" + local minimum_cpus="${LAB_PIMOX_APP_DEPLOY_MIN_CPUS:-2}" + local worker_key + local worker_host + local worker_user + local worker_node + local worker_key_path + local capacity + local ready + local failures=0 + + if disabled_value "${enabled}"; then + echo "Skipping Pimox worker CPU gate because LAB_PIMOX_APP_DEPLOY_REQUIRE_CAPACITY=${enabled}." + return 0 + fi + if [[ ! -s "${spec_file}" ]]; then + echo "No generated Pimox worker inventory exists; skipping Pimox worker CPU gate." + return 0 + fi + if ! [[ "${minimum_cpus}" =~ ^[1-9][0-9]*$ ]]; then + echo "LAB_PIMOX_APP_DEPLOY_MIN_CPUS must be a positive integer." >&2 + exit 1 + fi + + while IFS=$'\t' read -r worker_key worker_host worker_user worker_node worker_key_path; do + [[ -n "${worker_node}" ]] || continue + capacity="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || true)" + ready="$(kubectl --kubeconfig "${KUBECONFIG}" get node "${worker_node}" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" + if [[ "${ready}" != "True" ]] || ! [[ "${capacity}" =~ ^[0-9]+$ ]] || ((capacity < minimum_cpus)); then + echo "Pimox worker ${worker_node} must be Ready with at least ${minimum_cpus} Kubernetes CPU(s); Ready=${ready:-missing}, capacity=${capacity:-missing}." >&2 + failures=$((failures + 1)) + fi + done <"${spec_file}" + + if ((failures > 0)); then + echo "Refusing to deploy applications until Pimox workers expose the required Kubernetes CPU capacity." >&2 + kubectl --kubeconfig "${KUBECONFIG}" get nodes -o custom-columns='NAME:.metadata.name,CPUS:.status.capacity.cpu,READY:.status.conditions[?(@.type=="Ready")].status' >&2 || true + exit 1 + fi +} + apps() { local buildx_builder_ready=false local demos_image_built=false @@ -5008,6 +5242,9 @@ apps() { export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" export KUBECONFIG="${TF_VAR_kubeconfig_path}" + require_argocd_application_controller + require_pimox_app_worker_capacity + ensure_cosign_keypair publish_cosign_public_key_config_map