diff --git a/README.md b/README.md index d9e50fc..f9b2236 100644 --- a/README.md +++ b/README.md @@ -164,9 +164,9 @@ hostname. For Pimox on Orange Pi 5 Plus, `./lab.sh up` can create the Debian 13 arm64 template and worker VM clones automatically. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID -`9000` on `local` storage, two 4 GiB worker VMs starting at VMID `9010`, CPU -affinities `4-5` and `6-7`, and worker clone storage `nvme_thin_pool`. Details -and override variables are in +`9000` on `local` storage, two 4 GiB worker VMs starting at VMID `9010`, worker +clone storage `nvme_thin_pool`, and no CPU affinity because this Pimox is pinned +to Debian Bullseye. Details and override variables are in `bootstrap/provisioning/README.md`. Worker indexes are stable. Index `1` maps to VMID `9010`, node name @@ -181,10 +181,10 @@ Run a full cluster rebuild from the Debian server with: ``` That path preserves external Raspberry Pi Gitea, rebuilds the Pimox template -with 2 cores, 4 GiB memory, and CPU affinity `4-5`, replaces two Pimox worker -VMs with 2 cores, 4 GiB memory, and affinities `4-5` and `6-7`, and joins those -workers to the Kubernetes cluster. The Raspberry Pi worker is excluded by -default while it hosts external Gitea. +with 2 cores and 4 GiB memory, replaces two Pimox worker VMs with 2 cores and +4 GiB memory, and joins those workers to the Kubernetes cluster. CPU affinity is +disabled by default because the Bullseye-pinned Pimox `qm` does not support it. +The Raspberry Pi worker is excluded by default while it hosts external Gitea. To opt the Raspberry Pi back into the Kubernetes cluster, set `LAB_INCLUDE_RASPBERRY_WORKER=true` or add entries to diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 358a4c2..022d71f 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -102,17 +102,18 @@ LAB_PIMOX_PIPELINE=true ./lab.sh up Defaults match the observed Pimox template VM shape: OVMF firmware, virtio networking, virtio-scsi disk, `vmbr0`, `local` template storage, 1 socket with -2 cores, 4 GiB memory, and high-speed CPU affinity `4-5`. Override +2 cores, and 4 GiB memory. The Bullseye-pinned Pimox `qm` does not support +`--affinity`, so CPU affinity is disabled by default. Override `TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, -`TF_VAR_pimox_template_cores`, `TF_VAR_pimox_template_memory`, or -`TF_VAR_pimox_template_cpu_affinity` if the Orange Pi template layout changes. +`TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange +Pi template layout changes. `TF_VAR_pimox_template_cpu_affinity` remains an +opt-in for a newer `qm`; if unsupported, automation logs the skip and continues. `./lab.sh up` also creates or reuses worker clones after the template exists. It defaults to two workers, VMIDs `9010` and `9011`, names like `pimox-worker-01`, deterministic locally administered MAC addresses, 1 socket -with 2 cores, 4 GiB RAM, Orange Pi 5 high-speed CPU affinity pairs `4-5` and -`6-7`, `nvme_thin_pool` clone storage, and qemu-guest-agent IP discovery. New -workers are full clones created with +with 2 cores, 4 GiB RAM, `nvme_thin_pool` clone storage, and qemu-guest-agent IP +discovery. New workers are full clones created with `qm clone --storage`, so the template can remain on `local` while worker disks land on the NVMe thin pool. Set `LAB_PIMOX_WORKER_REPLACE_EXISTING=true` to destroy and recreate existing worker VMs from the current template. The pipeline @@ -128,7 +129,6 @@ LAB_PIMOX_WORKER_COUNT=2 ./lab.sh up LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up LAB_PIMOX_WORKER_REPLACE_EXISTING=true ./lab.sh up -LAB_PIMOX_WORKER_CPU_AFFINITIES="4-5 6-7" ./lab.sh up LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up ``` diff --git a/bootstrap/provisioning/main.tf b/bootstrap/provisioning/main.tf index 1b02122..cd7553a 100644 --- a/bootstrap/provisioning/main.tf +++ b/bootstrap/provisioning/main.tf @@ -124,7 +124,7 @@ resource "null_resource" "pimox_template_vm_create" { pimox_user = var.pimox_user ssh_key_path = var.pimox_ssh_key_path qm_bin = var.pimox_qm_bin - builder_version = "5" + builder_version = "7" vmid = tostring(var.pimox_template_vmid) name = var.pimox_template_name cores = tostring(var.pimox_template_cores) @@ -209,7 +209,19 @@ sudo "$qm_cmd" set "$vmid" --efidisk0 "${self.triggers.efidisk0}" sudo "$qm_cmd" set "$vmid" --scsi0 "${self.triggers.scsi0}" sudo "$qm_cmd" set "$vmid" --boot "order=scsi0;net0" sudo "$qm_cmd" set "$vmid" --agent enabled=1 -sudo "$qm_cmd" set "$vmid" --affinity "${self.triggers.cpu_affinity}" +if [ -n "${self.triggers.cpu_affinity}" ]; then + affinity_output="$(sudo "$qm_cmd" set "$vmid" --affinity "${self.triggers.cpu_affinity}" 2>&1)" || { + case "$affinity_output" in + *"Unknown option: affinity"*) + echo "Pimox qm does not support --affinity; skipping CPU affinity ${self.triggers.cpu_affinity} for VM $vmid." + ;; + *) + printf '%s\n' "$affinity_output" >&2 + exit 1 + ;; + esac + } +fi sudo "$qm_cmd" start "$vmid" EOT ] diff --git a/bootstrap/provisioning/variables.tf b/bootstrap/provisioning/variables.tf index e9efb35..58b0c6e 100644 --- a/bootstrap/provisioning/variables.tf +++ b/bootstrap/provisioning/variables.tf @@ -201,7 +201,7 @@ variable "pimox_template_memory" { variable "pimox_template_cpu_affinity" { type = string - default = "4-5" + default = "" } variable "pimox_template_bridge" { diff --git a/lab.sh b/lab.sh index 1e1e6e4..afbe15e 100755 --- a/lab.sh +++ b/lab.sh @@ -299,7 +299,21 @@ while [ \"\$elapsed\" -lt 300 ]; do done sudo '${qm_bin}' destroy '${vmid}' --purge 1 >/dev/null 2>&1 || sudo '${qm_bin}' destroy '${vmid}'" else - pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' --affinity '${worker_cpu_affinity}' + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu +sudo '${qm_bin}' set '${vmid}' --agent enabled=1 --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' +if [ -n '${worker_cpu_affinity}' ]; then + affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { + case \"\$affinity_output\" in + *'Unknown option: affinity'*) + echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' + ;; + *) + printf '%s\n' \"\$affinity_output\" >&2 + exit 1 + ;; + esac + } +fi if sudo '${qm_bin}' status '${vmid}' | grep -q 'status: stopped'; then sudo '${qm_bin}' start '${vmid}'; fi" fi fi @@ -324,7 +338,20 @@ if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && fi sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' set '${vmid}' --agent enabled=1 -sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' --affinity '${worker_cpu_affinity}' +sudo '${qm_bin}' set '${vmid}' --sockets 1 --cores '${worker_cores}' --memory '${worker_memory}' +if [ -n '${worker_cpu_affinity}' ]; then + affinity_output=\"\$(sudo '${qm_bin}' set '${vmid}' --affinity '${worker_cpu_affinity}' 2>&1)\" || { + case \"\$affinity_output\" in + *'Unknown option: affinity'*) + echo 'Pimox qm does not support --affinity; skipping CPU affinity ${worker_cpu_affinity} for VM ${vmid}.' + ;; + *) + printf '%s\n' \"\$affinity_output\" >&2 + exit 1 + ;; + esac + } +fi sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}' sudo '${qm_bin}' set '${vmid}' --boot 'order=scsi0;net0' sudo '${qm_bin}' set '${vmid}' --onboot 1 @@ -405,7 +432,7 @@ run_pimox_pipeline() { local template_name="${LAB_PIMOX_TEMPLATE_NAME:-${TF_VAR_pimox_template_name:-debian13-arm64-k8s-template}}" local template_cores="${LAB_PIMOX_TEMPLATE_CORES:-${TF_VAR_pimox_template_cores:-2}}" local template_memory="${LAB_PIMOX_TEMPLATE_MEMORY:-${TF_VAR_pimox_template_memory:-4096}}" - local template_cpu_affinity="${LAB_PIMOX_TEMPLATE_CPU_AFFINITY:-${TF_VAR_pimox_template_cpu_affinity:-4-5}}" + local template_cpu_affinity="${LAB_PIMOX_TEMPLATE_CPU_AFFINITY:-${TF_VAR_pimox_template_cpu_affinity:-}}" local template_replace_existing="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-${TF_VAR_pimox_template_replace_existing:-false}}" local provisioning_interface local worker_count="${LAB_PIMOX_WORKER_COUNT:-2}" @@ -416,7 +443,7 @@ run_pimox_pipeline() { local worker_skip_indexes="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-4096}" - local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-4-5 6-7}" + local worker_cpu_affinities="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" local worker_replace_existing="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-false}" local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-nvme_thin_pool}}" local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" @@ -455,13 +482,15 @@ run_pimox_pipeline() { echo "Pimox template and worker CPU and memory values must be greater than zero." >&2 exit 1 fi - if ! template_cpu_count="$(cpuset_cpu_count "${template_cpu_affinity}")"; then - echo "Invalid Pimox template CPU affinity '${template_cpu_affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 - exit 1 - fi - if ((template_cpu_count != template_cores)); then - echo "Pimox template uses ${template_cores} cores but affinity '${template_cpu_affinity}' contains ${template_cpu_count} CPUs." >&2 - exit 1 + if [[ -n "${template_cpu_affinity}" ]]; then + if ! template_cpu_count="$(cpuset_cpu_count "${template_cpu_affinity}")"; then + echo "Invalid Pimox template CPU affinity '${template_cpu_affinity}'. Use CPU IDs or ranges, such as 4-5." >&2 + exit 1 + fi + if ((template_cpu_count != template_cores)); then + echo "Pimox template uses ${template_cores} cores but affinity '${template_cpu_affinity}' contains ${template_cpu_count} CPUs." >&2 + exit 1 + fi fi if ! truthy "${worker_replace_existing}" && ! disabled_value "${worker_replace_existing}"; then echo "LAB_PIMOX_WORKER_REPLACE_EXISTING must be true or false." >&2 @@ -548,7 +577,10 @@ fi" 2>&1)" continue fi - worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" + worker_cpu_affinity="" + if [[ -n "${worker_cpu_affinities}" ]]; then + worker_cpu_affinity="$(pimox_worker_cpu_affinity "${index}" "${worker_cpu_affinities}" "${worker_cores}")" + fi ensure_pimox_worker_node \ "${index}" \ "${spec_file}" \ @@ -2298,13 +2330,13 @@ rebuild_cluster() { export LAB_PIMOX_TEMPLATE_REPLACE_EXISTING="${LAB_PIMOX_TEMPLATE_REPLACE_EXISTING:-true}" export LAB_PIMOX_TEMPLATE_CORES="${LAB_PIMOX_TEMPLATE_CORES:-2}" export LAB_PIMOX_TEMPLATE_MEMORY="${LAB_PIMOX_TEMPLATE_MEMORY:-4096}" - export LAB_PIMOX_TEMPLATE_CPU_AFFINITY="${LAB_PIMOX_TEMPLATE_CPU_AFFINITY:-4-5}" + export LAB_PIMOX_TEMPLATE_CPU_AFFINITY="${LAB_PIMOX_TEMPLATE_CPU_AFFINITY:-}" export LAB_PIMOX_WORKER_COUNT="${LAB_PIMOX_WORKER_COUNT:-2}" export LAB_PIMOX_SKIP_WORKER_INDEXES="${LAB_PIMOX_SKIP_WORKER_INDEXES:-}" export LAB_PIMOX_WORKER_REPLACE_EXISTING="${LAB_PIMOX_WORKER_REPLACE_EXISTING:-true}" export LAB_PIMOX_WORKER_CORES="${LAB_PIMOX_WORKER_CORES:-2}" export LAB_PIMOX_WORKER_MEMORY="${LAB_PIMOX_WORKER_MEMORY:-4096}" - export LAB_PIMOX_WORKER_CPU_AFFINITIES="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-4-5 6-7}" + export LAB_PIMOX_WORKER_CPU_AFFINITIES="${LAB_PIMOX_WORKER_CPU_AFFINITIES:-}" export WORKER_SSH_TARGETS="${WORKER_SSH_TARGETS:-}" echo "Rebuilding the Kubernetes cluster without touching external Raspberry Pi Gitea..."