Fixing pimox worker seen one cpu instead of two

This commit is contained in:
juvenal.diaz 2026-07-22 19:09:55 -06:00
parent de990cdfb3
commit 58af55e7e0
6 changed files with 61 additions and 20 deletions

View File

@ -717,7 +717,7 @@ For Pimox on Orange Pi 5 Plus, `./jeannie up` can create the Debian 13 arm64
template and worker VM clones automatically when the Orange Pi storage path is template and worker VM clones automatically when the Orange Pi storage path is
healthy. Defaults are intentionally tied to the observed host: Pimox SSH host healthy. Defaults are intentionally tied to the observed host: Pimox SSH host
`192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two
4 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`,
and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe
that was previously attached to the Orange Pi now belongs to the HP Debian that was previously attached to the Orange Pi now belongs to the HP Debian
laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox
@ -777,8 +777,8 @@ Run a full cluster rebuild from the Debian server with:
``` ```
That path preserves external Gitea, rebuilds the Pimox template That path preserves external Gitea, rebuilds the Pimox template
with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and
4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is 8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is
disabled by default because the Bullseye-pinned Pimox `qm` does not support it. disabled by default because the Bullseye-pinned Pimox `qm` does not support it.
The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` The Raspberry Pi is still included as a Kubernetes worker by default; `nuke`
does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`.

View File

@ -717,7 +717,7 @@ For Pimox on Orange Pi 5 Plus, `./{{ main_script }} up` can create the Debian 13
template and worker VM clones automatically when the Orange Pi storage path is template and worker VM clones automatically when the Orange Pi storage path is
healthy. Defaults are intentionally tied to the observed host: Pimox SSH host healthy. Defaults are intentionally tied to the observed host: Pimox SSH host
`192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two `192.168.100.80`, bridge `vmbr0`, template VMID `9000` on `local` storage, two
4 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`, 8 GiB worker VMs starting at VMID `9010`, worker clone storage `opi5_ssd`,
and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe and no CPU affinity because this Pimox is pinned to Debian Bullseye. The NVMe
that was previously attached to the Orange Pi now belongs to the HP Debian that was previously attached to the Orange Pi now belongs to the HP Debian
laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox laptop as LVM volume group `data-vg`; the current Orange Pi SSD-backed Pimox
@ -777,8 +777,8 @@ Run a full cluster rebuild from the Debian server with:
``` ```
That path preserves external Gitea, rebuilds the Pimox template That path preserves external Gitea, rebuilds the Pimox template
with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and with 2 cores and 8 GiB memory, replaces two Pimox worker VMs with 2 cores and
4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is 8 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is
disabled by default because the Bullseye-pinned Pimox `qm` does not support it. disabled by default because the Bullseye-pinned Pimox `qm` does not support it.
The Raspberry Pi is still included as a Kubernetes worker by default; `nuke` The Raspberry Pi is still included as a Kubernetes worker by default; `nuke`
does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`. does not clean it unless you explicitly add it to `WORKER_SSH_TARGETS`.

View File

@ -124,14 +124,15 @@ template-build VM. If you still want to force a known address, set
Defaults match the observed Pimox template VM shape: OVMF firmware, virtio Defaults match the observed Pimox template VM shape: OVMF firmware, virtio
networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with
2 cores, and 4 GiB memory. Override `TF_VAR_pimox_template_scsi0`, 2 cores, 8 GiB memory, and explicit `args: -smp 2,sockets=1,cores=2,threads=1`.
Override `TF_VAR_pimox_template_scsi0`,
`TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or
`TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. `TF_VAR_pimox_template_memory` if the Orange Pi template layout changes.
`./jeannie up` also creates or reuses worker clones after the template exists. It `./jeannie up` also creates or reuses worker clones after the template exists. It
defaults to one worker, VMID `9010`, name `pimox-worker-01`, deterministic defaults to one worker, VMID `9010`, name `pimox-worker-01`, deterministic
locally administered MAC address, 1 socket locally administered MAC address, 1 socket
with 2 cores, 4 GiB RAM, `opi5_ssd` clone storage, and qemu-guest-agent IP with 2 cores, 8 GiB RAM, `opi5_ssd` clone storage, and qemu-guest-agent IP
discovery when that Pimox storage is available. New workers are full clones discovery when that Pimox storage is available. New workers are full clones
created with `qm clone --storage`, so the template can remain on `local` while created with `qm clone --storage`, so the template can remain on `local` while
worker disks land on the Orange Pi SSD-backed non-local Pimox datastore. Keep worker disks land on the Orange Pi SSD-backed non-local Pimox datastore. Keep

View File

@ -46,6 +46,7 @@ locals {
node_dns_servers = join(" ", var.node_dns_servers) node_dns_servers = join(" ", var.node_dns_servers)
kernel_cgroup_boot_options = join(" ", var.kernel_cgroup_boot_options) kernel_cgroup_boot_options = join(" ", var.kernel_cgroup_boot_options)
pimox_template_net0 = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}" pimox_template_net0 = var.pimox_template_mac != "" ? "virtio=${var.pimox_template_mac},bridge=${var.pimox_template_bridge}" : "virtio,bridge=${var.pimox_template_bridge}"
pimox_template_args = "-smp ${var.pimox_template_cores},sockets=1,cores=${var.pimox_template_cores},threads=1"
template_package_list = join(" ", local.template_packages) template_package_list = join(" ", local.template_packages)
provisioning_http_base_url = "http://${var.http_host}:${var.http_port}" provisioning_http_base_url = "http://${var.http_host}:${var.http_port}"
provisioning_script_url = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh" provisioning_script_url = "${local.provisioning_http_base_url}/scripts/golden-node-prepare.sh"
@ -140,6 +141,7 @@ resource "null_resource" "pimox_template_vm_create" {
memory = tostring(var.pimox_template_memory) memory = tostring(var.pimox_template_memory)
bridge = var.pimox_template_bridge bridge = var.pimox_template_bridge
net0 = local.pimox_template_net0 net0 = local.pimox_template_net0
args = local.pimox_template_args
scsi0 = var.pimox_template_scsi0 scsi0 = var.pimox_template_scsi0
efidisk0 = var.pimox_template_efidisk0 efidisk0 = var.pimox_template_efidisk0
} }
@ -182,7 +184,21 @@ fi
if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then if sudo "$qm_cmd" status "$vmid" >/dev/null 2>&1; then
if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then
if [ "$replace_existing" != "true" ]; then if [ "$replace_existing" != "true" ]; then
sudo "$qm_cmd" set "$vmid" --agent enabled=1 sudo "$qm_cmd" set "$vmid" \
--agent enabled=1 \
--args "${self.triggers.args}" \
--bios ovmf \
--boot "order=scsi0;net0" \
--cores "${self.triggers.cores}" \
--memory "${self.triggers.memory}" \
--net0 "${self.triggers.net0}" \
--numa 0 \
--ostype l26 \
--scsihw virtio-scsi-pci \
--sockets 1 \
--vga virtio
sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
exit 0 exit 0
fi fi
sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid" sudo "$qm_cmd" destroy "$vmid" --purge 1 >/dev/null 2>&1 || sudo "$qm_cmd" destroy "$vmid"
@ -218,6 +234,7 @@ sudo "$qm_cmd" create "$vmid" \
--vga virtio \ --vga virtio \
--agent enabled=1 --agent enabled=1
sudo "$qm_cmd" set "$vmid" --args "${self.triggers.args}"
sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}"
sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}"
sudo "$qm_cmd" set "$vmid" --agent enabled=1 sudo "$qm_cmd" set "$vmid" --agent enabled=1
@ -392,7 +409,7 @@ if [ -z "$qm_cmd" ]; then
fi fi
if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then if sudo "$qm_cmd" config "$vmid" | grep -q '^template: 1$'; then
sudo "$qm_cmd" set "$vmid" --agent enabled=1 sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0"
exit 0 exit 0
fi fi
@ -410,7 +427,7 @@ if ! sudo "$qm_cmd" status "$vmid" | grep -q 'status: stopped'; then
exit 1 exit 1
fi fi
sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0" sudo "$qm_cmd" set "$vmid" --agent enabled=1 --boot "order=scsi0;net0"
sudo "$qm_cmd" template "$vmid" sudo "$qm_cmd" template "$vmid"
EOT EOT
] ]

View File

@ -206,7 +206,7 @@ variable "pimox_template_cores" {
variable "pimox_template_memory" { variable "pimox_template_memory" {
type = number type = number
default = 4096 default = 8192
} }
variable "pimox_template_bridge" { variable "pimox_template_bridge" {

37
jeannie
View File

@ -1743,6 +1743,7 @@ ensure_pimox_worker_node() {
local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}" local worker_gateway="${LAB_PIMOX_WORKER_GATEWAY:-192.168.100.1}"
local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}" local worker_dns_servers="${LAB_PIMOX_WORKER_DNS_SERVERS:-192.168.100.89 1.1.1.1}"
local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}" local worker_interface="${LAB_PIMOX_WORKER_INTERFACE:-enp0s18}"
local worker_args
local padded local padded
local vmid local vmid
local worker_key local worker_key
@ -1760,6 +1761,7 @@ ensure_pimox_worker_node() {
if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then if ! net0_config="$(pimox_worker_net0_config "${vmid}" "${bridge}")"; then
exit 1 exit 1
fi fi
worker_args="-smp ${worker_cores},sockets=1,cores=${worker_cores},threads=1"
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' status '${vmid}' >/dev/null 2>&1"; then
if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then if pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' config '${vmid}' | grep -q '^template: 1$'"; then
@ -1786,7 +1788,19 @@ done
sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'"
else else
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu
sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' sudo '${qm_bin}' set '${vmid}' \
--agent enabled=1 \
--args '${worker_args}' \
--bios ovmf \
--boot 'order=scsi0;net0' \
--cores '${worker_cores}' \
--memory '${worker_memory}' \
--net0 '${net0_config}' \
--numa 0 \
--ostype l26 \
--scsihw virtio-scsi-pci \
--sockets 1 \
--vga virtio
if [ -n '${worker_cpu_affinity}' ]; then if [ -n '${worker_cpu_affinity}' ]; then
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
case \"\$affinity_output\" in case \"\$affinity_output\" in
@ -1823,8 +1837,19 @@ if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 &&
exit 1 exit 1
fi fi
sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}'
sudo '${qm_bin}' set '${vmid}' --agent enabled=1 sudo '${qm_bin}' set '${vmid}' \
sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' --agent enabled=1 \
--args '${worker_args}' \
--bios ovmf \
--boot 'order=scsi0;net0' \
--cores '${worker_cores}' \
--memory '${worker_memory}' \
--net0 '${net0_config}' \
--numa 0 \
--ostype l26 \
--scsihw virtio-scsi-pci \
--sockets 1 \
--vga virtio
if [ -n '${worker_cpu_affinity}' ]; then if [ -n '${worker_cpu_affinity}' ]; then
affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || {
case \"\$affinity_output\" in case \"\$affinity_output\" in
@ -1838,8 +1863,6 @@ if [ -n '${worker_cpu_affinity}' ]; then
esac esac
} }
fi fi
sudo '${qm_bin}' set '${vmid}' --net0 '${net0_config}'
sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0'
sudo '${qm_bin}' set '${vmid}' --onboot 1 sudo '${qm_bin}' set '${vmid}' --onboot 1
sudo '${qm_bin}' start '${vmid}'" sudo '${qm_bin}' start '${vmid}'"
fi fi
@ -2399,7 +2422,7 @@ run_pimox_pipeline() {
local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}" local template_vmid="${LAB_PIMOX_TEMPLATE_VMID:-${TF_VAR_pimox_template_vmid:-9000}}"
local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}"
local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}"
local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-4096}}" local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-8192}}"
local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}"
local provisioning_interface local provisioning_interface
local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}" local worker_count="${LAB_PIMOX_WORKER_COUNT:-1}"
@ -2409,7 +2432,7 @@ run_pimox_pipeline() {
local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}"
local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}"
local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}"
local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}" local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-8192}"
local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}"
local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}"
local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-opi5_ssd}}"