diff --git a/README.md b/README.md index 879bd35..572dd5f 100644 --- a/README.md +++ b/README.md @@ -717,7 +717,7 @@ For Pimox on Orange Pi 5 Plus, `./jeannie up` can create the Debian 13 arm64 template and worker VM clones automatically when the Orange Pi storage path is healthy. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two -4 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, +8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe that was previously attached to the Orange Pi now belongs to the HP Debian laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox @@ -777,8 +777,8 @@ Run a full cluster rebuild from the Debian server with: ``` That path preserves external Gitea, rebuilds the Pimox template -with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and -4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is +with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and +8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is disabled by default because the Bullseye-pinned Pimox `qm` does not support it. The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. diff --git a/README.md.tmpl b/README.md.tmpl index bca134b..ce74917 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -717,7 +717,7 @@ For Pimox on Orange Pi 5 Plus, `./{{ main_script }} up` can create the Debian 13 template and worker VM clones automatically when the Orange Pi storage path is healthy. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two -4 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, +8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe that was previously attached to the Orange Pi now belongs to the HP Debian laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox @@ -777,8 +777,8 @@ Run a full cluster rebuild from the Debian server with: ``` That path preserves external Gitea, rebuilds the Pimox template -with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and -4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is +with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and +8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is disabled by default because the Bullseye-pinned Pimox `qm` does not support it. The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index e71455d..9ef4e32 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -124,14 +124,15 @@ template-build VM. If you still want to force a known address, set Defaults match the observed Pimox template VM shape: OVMF firmware, virtio networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with -2 cores, and 4 GiB memory. Override `TF_VAR_pimox_template_scsi0`, +2 cores, 8 GiB memory, and explicit `args: -smp 2,sockets=1,cores=2,threads=1`. +Override `TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. `./jeannie up` also creates or reuses worker clones after the template exists. It defaults to one worker, VMID `9010`, name `pimox-worker-01`, deterministic locally administered MAC address, 1 socket -with 2 cores, 4 GiB RAM, `opi5_ssd` clone storage, and qemu-guest-agent IP +with 2 cores, 8 GiB RAM, `opi5_ssd` clone storage, and qemu-guest-agent IP discovery when that Pimox storage is available. New workers are full clones created with `qm clone --storage`, so the template can remain on `local` while worker disks land on the Orange Pi SSD-backed non-local Pimox datastore. Keep diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index 799a2a2..5703575 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -46,6 +46,7 @@ locals { node_dns_servers = join(" ", var.node_dns_servers) kernel_cgroup_boot_options = join(" ", var.kernel_cgroup_boot_options) pimox_template_net0 = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}" + pimox_template_args = "-smp ${var.pimox_template_cores},sockets=1,cores=${var.pimox_template_cores},threads=1" template_package_list = join(" ", local.template_packages) provisioning_http_base_url = "http://${var.http_host}:${var.http_port}" provisioning_script_url = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh" @@ -140,6 +141,7 @@ resource "null_resource" "pimox_template_vm_create" { memory = tostring(var.pimox_template_memory) bridge = var.pimox_template_bridge net0 = local.pimox_template_net0 + args = local.pimox_template_args scsi0 = var.pimox_template_scsi0 efidisk0 = var.pimox_template_efidisk0 } @@ -182,7 +184,21 @@ fi if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then if [ "$replace_existing" != "true" ]; then - sudo "$qm_cmd" set "$vmid" --agent enabled=1 + sudo "$qm_cmd" set "$vmid" \ + --agent enabled=1 \ + --args "${self.triggers.args}" \ + --bios ovmf \ + --boot "order=scsi0;net0" \ + --cores "${self.triggers.cores}" \ + --memory "${self.triggers.memory}" \ + --net0 "${self.triggers.net0}" \ + --numa 0 \ + --ostype l26 \ + --scsihw virtio-scsi-pci \ + --sockets 1 \ + --vga virtio + sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" + sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" exit 0 fi sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid" @@ -218,6 +234,7 @@ sudo "$qm_cmd" create "$vmid" \ --vga virtio \ --agent enabled=1 +sudo "$qm_cmd" set "$vmid" --args "${self.triggers.args}" sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --agent enabled=1 @@ -392,7 +409,7 @@ if [ -z "$qm_cmd" ]; then fi if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then - sudo "$qm_cmd" set "$vmid" --agent enabled=1 + sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0" exit 0 fi @@ -410,7 +427,7 @@ if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then exit 1 fi -sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0" +sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0" sudo "$qm_cmd" template "$vmid" EOT ] diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index b09fc0b..e879b9f 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -206,7 +206,7 @@ variable "pimox_template_cores" { variable "pimox_template_memory" { type = number - default = 4096 + default = 8192 } variable "pimox_template_bridge" { diff --git a/jeannie b/jeannie index 0bc0bc7..a8bd844 100755 --- a/jeannie +++ b/jeannie @@ -1743,6 +1743,7 @@ ensure_pimox_worker_node() { local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" + local worker_args local padded local vmid local worker_key @@ -1760,6 +1761,7 @@ ensure_pimox_worker_node() { if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then exit 1 fi + worker_args="-smp ${worker_cores},sockets=1,cores=${worker_cores},threads=1" if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then @@ -1786,7 +1788,19 @@ done sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" else pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu -sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' +sudo '${qm_bin}' set '${vmid}' \ + --agent enabled=1 \ + --args '${worker_args}' \ + --bios ovmf \ + --boot 'order=scsi0;net0' \ + --cores '${worker_cores}' \ + --memory '${worker_memory}' \ + --net0 '${net0_config}' \ + --numa 0 \ + --ostype l26 \ + --scsihw virtio-scsi-pci \ + --sockets 1 \ + --vga virtio if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in @@ -1823,8 +1837,19 @@ if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && exit 1 fi sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' -sudo '${qm_bin}' set '${vmid}' --agent enabled=1 -sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' +sudo '${qm_bin}' set '${vmid}' \ + --agent enabled=1 \ + --args '${worker_args}' \ + --bios ovmf \ + --boot 'order=scsi0;net0' \ + --cores '${worker_cores}' \ + --memory '${worker_memory}' \ + --net0 '${net0_config}' \ + --numa 0 \ + --ostype l26 \ + --scsihw virtio-scsi-pci \ + --sockets 1 \ + --vga virtio if [ -n '${worker_cpu_affinity}' ]; then affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { case \"\$affinity_output\" in @@ -1836,10 +1861,8 @@ if [ -n '${worker_cpu_affinity}' ]; then exit 1 ;; esac - } +} fi -sudo '${qm_bin}' set '${vmid}' --net0 '${net0_config}' -sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0' sudo '${qm_bin}' set '${vmid}' --onboot 1 sudo '${qm_bin}' start '${vmid}'" fi @@ -2399,7 +2422,7 @@ run_pimox_pipeline() { local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}" local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" - local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-4096}}" + local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}" local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" local provisioning_interface local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" @@ -2409,7 +2432,7 @@ run_pimox_pipeline() { local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" - local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}" + local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}" local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"