From 9a8b4ba26d5c2c5eff47a8fe47385af415005713 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 26 May 2026 21:43:12 -0600 Subject: [PATCH] Place Pimox worker clones on NVMe storage --- README.md | 10 ++++++---- bootstrap/provisioning/README.md | 17 ++++++++++------- lab.sh | 24 ++++++++++++++++++++++-- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index ef29625..66e57c6 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,9 @@ new image is built, and applies the edge stack. Set `LAB_PIMOX_PIPELINE=false` to skip Pimox automation. Set `LAB_PIMOX_WORKER_COUNT=0` to create or refresh only the template. The pipeline -checks that the Pimox bridge already exists and refuses to edit Orange Pi host -networking. +keeps the template on its configured `local` storage, creates new worker VM +clones on `nvme_thin_pool` by default, checks that the Pimox bridge already +exists, and refuses to edit Orange Pi host networking. The website and demos images default to `linux/arm64` because both deployments are pinned to the Raspberry Pi worker. Override with `WEBSITE_IMAGE_PLATFORMS` @@ -130,8 +131,9 @@ hostname. For Pimox on Orange Pi 5 Plus, `./lab.sh up` can create the Debian 13 arm64 template and worker VM clones automatically. Defaults are intentionally tied to the observed host: Pimox SSH host `192.168.100.80`, bridge `vmbr0`, template VMID -`9000`, and worker VMIDs starting at `9010`. Details and override variables are -in `bootstrap/provisioning/README.md`. +`9000` on `local` storage, worker VMIDs starting at `9010`, and worker clone +storage `nvme_thin_pool`. Details and override variables are in +`bootstrap/provisioning/README.md`. Add entries to `bootstrap/cluster/variables.tf` or a `.tfvars` file: diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 027e564..d9e3540 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -100,21 +100,24 @@ template-build VM. If you still want to force a known address, set LAB_PIMOX_PIPELINE=true ./lab.sh up ``` -Defaults match the observed Pimox VM shape: OVMF firmware, virtio networking, -virtio-scsi disk, `vmbr0`, `local` storage, 2 vCPU, and 2 GiB memory. Override -`TF_VAR_pimox_template_scsi0`, `TF_VAR_pimox_template_efidisk0`, -`TF_VAR_pimox_template_cores`, or `TF_VAR_pimox_template_memory` if the Orange -Pi storage layout changes. +Defaults match the observed Pimox template VM shape: OVMF firmware, virtio +networking, virtio-scsi disk, `vmbr0`, `local` template storage, 2 vCPU, and +2 GiB memory. Override `TF_VAR_pimox_template_scsi0`, +`TF_VAR_pimox_template_efidisk0`, `TF_VAR_pimox_template_cores`, or +`TF_VAR_pimox_template_memory` if the Orange Pi template layout changes. `./lab.sh up` also creates or reuses worker clones after the template exists. It defaults to one worker, VMID `9010`, names like `pimox-worker-01`, deterministic -locally administered MAC addresses, and qemu-guest-agent IP discovery. Useful -overrides: +locally administered MAC addresses, `nvme_thin_pool` clone storage, and +qemu-guest-agent IP discovery. New workers are full clones created with +`qm clone --storage`, so the template can remain on `local` while worker disks +land on the NVMe thin pool. Useful overrides: ```bash LAB_PIMOX_PIPELINE=false ./lab.sh up LAB_PIMOX_WORKER_COUNT=0 ./lab.sh up LAB_PIMOX_WORKER_COUNT=2 ./lab.sh up LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up +LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up ``` diff --git a/lab.sh b/lab.sh index bb3da9c..4cbae46 100755 --- a/lab.sh +++ b/lab.sh @@ -187,6 +187,7 @@ ensure_pimox_worker_node() { local ip_prefix="${16}" local timeout_seconds="${17}" local qm_bin="${18}" + local worker_storage="${19}" local padded local vmid local worker_key @@ -215,7 +216,19 @@ if ! ip link show '${bridge}' >/dev/null 2>&1; then echo 'Pimox bridge ${bridge} does not exist. Refusing to change Orange Pi networking.' >&2 exit 1 fi -sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 +pvesm_cmd=\"\$(command -v pvesm 2>/dev/null || true)\" +if [ -z \"\$pvesm_cmd\" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z \"\$pvesm_cmd\" ]; then + echo 'pvesm was not found; cannot validate Pimox worker storage ${worker_storage}' >&2 + exit 1 +fi +if ! sudo \"\$pvesm_cmd\" status | awk -v storage='${worker_storage}' 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then + echo 'Pimox worker storage ${worker_storage} was not found. Refusing to create worker ${worker_name}.' >&2 + exit 1 +fi +sudo '${qm_bin}' clone '${template_vmid}' '${vmid}' --name '${worker_name}' --full 1 --storage '${worker_storage}' sudo '${qm_bin}' set '${vmid}' --agent enabled=1 sudo '${qm_bin}' set '${vmid}' --cores '${worker_cores}' --memory '${worker_memory}' sudo '${qm_bin}' set '${vmid}' --net0 'virtio=${mac},bridge=${bridge}' @@ -294,6 +307,7 @@ run_pimox_pipeline() { local worker_key_prefix="${LAB_PIMOX_WORKER_KEY_PREFIX:-pimox}" local worker_cores="${LAB_PIMOX_WORKER_CORES:-2}" local worker_memory="${LAB_PIMOX_WORKER_MEMORY:-2048}" + local worker_storage="${LAB_PIMOX_WORKER_STORAGE:-${TF_VAR_pimox_worker_storage:-nvme_thin_pool}}" local worker_user="${LAB_PIMOX_WORKER_USER:-jv}" local worker_key_path="${LAB_PIMOX_WORKER_SSH_KEY_PATH:-/home/jv/.ssh/id_ed25519}" local ip_prefix="${LAB_PIMOX_GUEST_IP_PREFIX:-192.168.100.}" @@ -316,6 +330,10 @@ run_pimox_pipeline() { echo "LAB_PIMOX_WORKER_COUNT must be a non-negative integer." >&2 exit 1 fi + if ! [[ "${worker_storage}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "LAB_PIMOX_WORKER_STORAGE must be a valid Pimox storage identifier." >&2 + exit 1 + fi set +e readiness_output="$(pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "set -eu @@ -376,6 +394,7 @@ fi" 2>&1)" exit 1 fi pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "sudo '${qm_bin}' set '${template_vmid}' --agent enabled=1" + echo "Worker VM clones will be created on Pimox storage ${worker_storage}; template VM ${template_vmid} stays on its configured template storage." mkdir -p "${REPO_ROOT}/.lab" : >"${spec_file}" @@ -398,7 +417,8 @@ fi" 2>&1)" "${worker_key_path}" \ "${ip_prefix}" \ "${timeout_seconds}" \ - "${qm_bin}" + "${qm_bin}" \ + "${worker_storage}" done write_cluster_worker_var_file "${spec_file}" "${var_file}"