Add OpenWrt Pimox VM automation

This commit is contained in:
juvdiaz 2026-05-26 22:05:35 -06:00
parent 60911646bd
commit 180c1b1cca
3 changed files with 373 additions and 0 deletions

View File

@ -97,6 +97,13 @@ clones on `nvme_thin_pool` by default, checks that the Pimox bridge already
exists, refuses `local` as worker clone storage, and refuses to edit Orange Pi exists, refuses `local` as worker clone storage, and refuses to edit Orange Pi
host networking. host networking.
OpenWrt firewall VM automation is opt-in because it attaches to both WAN and
LAN bridges. Set `LAB_OPENWRT_VM=true` after `vmbr1` already exists on the
Orange Pi. The pipeline downloads the OpenWrt ARM SystemReady EFI image, writes
basic WAN/LAN/firewall config into the image, imports it as VM `9050`, attaches
`vmbr0` as WAN and `vmbr1` as LAN, and stores the VM disk on `nvme_thin_pool`.
It does not use the Debian Kubernetes golden-node template for OpenWrt.
The website and demos images default to `linux/arm64` because both deployments The website and demos images default to `linux/arm64` because both deployments
are pinned to the Raspberry Pi worker. Override with `WEBSITE_IMAGE_PLATFORMS` are pinned to the Raspberry Pi worker. Override with `WEBSITE_IMAGE_PLATFORMS`
or `DEMOS_IMAGE_PLATFORMS` only if node placement changes. or `DEMOS_IMAGE_PLATFORMS` only if node placement changes.

View File

@ -122,3 +122,46 @@ LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up
LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up
LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up
``` ```
## OpenWrt firewall VM
OpenWrt is not built from the Debian golden-node template. The Kubernetes
template remains Debian-only; OpenWrt uses the upstream ARM SystemReady
`armsr/armv8` combined EFI image instead.
The OpenWrt path is disabled by default. Enable it only after `vmbr1` exists on
the Pimox host and the second NIC/LAN side is safe to use:
```bash
LAB_OPENWRT_VM=true ./lab.sh up
```
Defaults:
- VMID `9050`
- VM name `openwrt-firewall`
- disk storage `nvme_thin_pool`
- WAN bridge `vmbr0`
- LAN bridge `vmbr1`
- LAN address `192.168.50.1/24`
- LAN DHCP disabled by default
- OpenWrt version `24.10.6`
Useful overrides:
```bash
LAB_OPENWRT_VMID=9050
LAB_OPENWRT_STORAGE=nvme_thin_pool
LAB_OPENWRT_WAN_BRIDGE=vmbr0
LAB_OPENWRT_LAN_BRIDGE=vmbr1
LAB_OPENWRT_LAN_IP=192.168.50.1
LAB_OPENWRT_LAN_NETMASK=255.255.255.0
LAB_OPENWRT_LAN_DHCP_ENABLED=true
LAB_OPENWRT_START=true
LAB_OPENWRT_VERSION=24.10.6
LAB_OPENWRT_IMAGE_URL=https://downloads.openwrt.org/releases/24.10.6/targets/armsr/armv8/openwrt-24.10.6-armsr-armv8-generic-ext4-combined-efi.img.gz
```
The pipeline validates `vmbr0`, `vmbr1`, and `nvme_thin_pool` on the Pimox host.
It refuses `local` as OpenWrt storage and refuses to create or modify host
network bridges.

323
lab.sh
View File

@ -429,6 +429,328 @@ fi" 2>&1)"
export LAB_CLUSTER_VAR_FILE="${var_file}" export LAB_CLUSTER_VAR_FILE="${var_file}"
} }
run_openwrt_pipeline() {
local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}"
local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}"
local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}"
local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}"
local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}"
local vmid="${LAB_OPENWRT_VMID:-9050}"
local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}"
local storage="${LAB_OPENWRT_STORAGE:-nvme_thin_pool}"
local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}"
local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}"
local cores="${LAB_OPENWRT_CORES:-2}"
local memory="${LAB_OPENWRT_MEMORY:-512}"
local version="${LAB_OPENWRT_VERSION:-24.10.6}"
local image_url="${LAB_OPENWRT_IMAGE_URL:-}"
local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}"
local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}"
local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}"
local start_vm="${LAB_OPENWRT_START:-true}"
local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}"
local root_key_b64=""
local lan_dhcp_ignore="1"
local start_vm_flag="false"
if disabled_value "${mode}"; then
return 0
fi
if ! truthy "${mode}"; then
echo "LAB_OPENWRT_VM must be true or false." >&2
exit 1
fi
if [[ -z "${image_url}" ]]; then
image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz"
fi
if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then
echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2
exit 1
fi
for value_name in storage wan_bridge lan_bridge vm_name; do
local value="${!value_name}"
if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then
echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2
exit 1
fi
done
if [[ "${storage}" == "local" ]]; then
echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2
exit 1
fi
if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then
echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2
exit 1
fi
if truthy "${lan_dhcp_enabled}"; then
lan_dhcp_ignore="0"
fi
if truthy "${start_vm}"; then
start_vm_flag="true"
fi
if [[ -r "${root_key_path}" ]]; then
root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')"
fi
echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..."
pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <<EOF
set -euo pipefail
vmid="${vmid}"
vm_name="${vm_name}"
storage="${storage}"
wan_bridge="${wan_bridge}"
lan_bridge="${lan_bridge}"
cores="${cores}"
memory="${memory}"
image_url="${image_url}"
lan_ip="${lan_ip}"
lan_netmask="${lan_netmask}"
lan_dhcp_ignore="${lan_dhcp_ignore}"
start_vm="${start_vm_flag}"
root_key_b64="${root_key_b64}"
qm_cmd="${qm_bin}"
if [ ! -x "\$qm_cmd" ]; then
qm_cmd="\$(command -v qm 2>/dev/null || true)"
fi
if [ -z "\$qm_cmd" ]; then
echo "qm is not installed on this Pimox host" >&2
exit 1
fi
pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)"
if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then
pvesm_cmd=/usr/sbin/pvesm
fi
if [ -z "\$pvesm_cmd" ]; then
echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2
exit 1
fi
if ! sudo -n true >/dev/null 2>&1; then
echo "passwordless sudo is required for OpenWrt VM automation" >&2
exit 1
fi
if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then
echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2
exit 1
fi
if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then
echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2
exit 1
fi
if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then
echo "Pimox storage \$storage was not found." >&2
exit 1
fi
if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then
if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then
echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2
exit 1
fi
sudo "\$qm_cmd" set "\$vmid" \\
--net0 "virtio,bridge=\$wan_bridge" \\
--net1 "virtio,bridge=\$lan_bridge" \\
--cores "\$cores" \\
--memory "\$memory" \\
--onboot 1
if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then
sudo "\$qm_cmd" start "\$vmid"
fi
exit 0
fi
for required_cmd in curl gzip losetup mount umount awk sed; do
if ! command -v "\$required_cmd" >/dev/null 2>&1; then
echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2
exit 1
fi
done
tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)"
mnt_dir="\$tmp_dir/root"
loopdev=""
cleanup() {
if mountpoint -q "\$mnt_dir" 2>/dev/null; then
sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true
fi
if [ -n "\$loopdev" ]; then
sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true
fi
rm -rf "\$tmp_dir"
}
trap cleanup EXIT
mkdir -p "\$mnt_dir"
curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz"
gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img"
loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")"
root_part="\${loopdev}p2"
if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then
root_part="\${loopdev}p2"
fi
if [ ! -b "\$root_part" ]; then
echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2
exit 1
fi
sudo mount "\$root_part" "\$mnt_dir"
sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh"
cat >"\$tmp_dir/network" <<NETWORK
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'
config globals 'globals'
option ula_prefix 'fd00:68:50::/48'
config interface 'wan'
option device 'eth0'
option proto 'dhcp'
config interface 'lan'
option device 'eth1'
option proto 'static'
option ipaddr '\$lan_ip'
option netmask '\$lan_netmask'
option ip6assign '60'
NETWORK
cat >"\$tmp_dir/dhcp" <<DHCP
config dnsmasq
option domainneeded '1'
option boguspriv '1'
option filterwin2k '0'
option localise_queries '1'
option rebind_protection '1'
option rebind_localhost '1'
option local '/lan/'
option domain 'lan'
option expandhosts '1'
option cachesize '1000'
option authoritative '1'
option readethers '1'
option leasefile '/tmp/dhcp.leases'
option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
config dhcp 'lan'
option interface 'lan'
option start '100'
option limit '150'
option leasetime '12h'
option ignore '\$lan_dhcp_ignore'
config dhcp 'wan'
option interface 'wan'
option ignore '1'
DHCP
cat >"\$tmp_dir/firewall" <<'FIREWALL'
config defaults
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option synflood_protect '1'
config zone
option name 'lan'
list network 'lan'
option input 'ACCEPT'
option output 'ACCEPT'
option forward 'ACCEPT'
config zone
option name 'wan'
list network 'wan'
option input 'REJECT'
option output 'ACCEPT'
option forward 'REJECT'
option masq '1'
option mtu_fix '1'
config forwarding
option src 'lan'
option dest 'wan'
config rule
option name 'Allow-DHCP-Renew'
option src 'wan'
option proto 'udp'
option dest_port '68'
option target 'ACCEPT'
option family 'ipv4'
config rule
option name 'Allow-Ping'
option src 'wan'
option proto 'icmp'
option icmp_type 'echo-request'
option family 'ipv4'
option target 'ACCEPT'
FIREWALL
cat >"\$tmp_dir/system" <<SYSTEM
config system
option hostname '\$vm_name'
option timezone 'UTC'
option ttylogin '0'
option log_size '64'
option urandom_seed '0'
SYSTEM
sudo cp "\$tmp_dir/network" "\$mnt_dir/etc/config/network"
sudo cp "\$tmp_dir/dhcp" "\$mnt_dir/etc/config/dhcp"
sudo cp "\$tmp_dir/firewall" "\$mnt_dir/etc/config/firewall"
sudo cp "\$tmp_dir/system" "\$mnt_dir/etc/config/system"
if [ -n "\$root_key_b64" ]; then
printf '%s' "\$root_key_b64" | base64 -d >"\$tmp_dir/authorized_keys"
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys"
sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys"
fi
sync
sudo umount "\$mnt_dir"
sudo losetup -d "\$loopdev"
loopdev=""
sudo "\$qm_cmd" create "\$vmid" \\
--name "\$vm_name" \\
--bios ovmf \\
--cores "\$cores" \\
--memory "\$memory" \\
--net0 "virtio,bridge=\$wan_bridge" \\
--net1 "virtio,bridge=\$lan_bridge" \\
--numa 0 \\
--ostype l26 \\
--scsihw virtio-scsi-pci \\
--sockets 1 \\
--vga virtio \\
--onboot 1
sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0"
sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null
disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')"
if [ -z "\$disk_volume" ]; then
echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2
exit 1
fi
sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume"
sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0"
if [ "\$start_vm" = "true" ]; then
sudo "\$qm_cmd" start "\$vmid"
fi
EOF
}
cleanup_calico_links() { cleanup_calico_links() {
ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true
sudo ip link delete vxlan.calico 2>/dev/null || true sudo ip link delete vxlan.calico 2>/dev/null || true
@ -1157,6 +1479,7 @@ up() {
echo "Deploying the homelab infrastructure..." echo "Deploying the homelab infrastructure..."
run_pimox_pipeline run_pimox_pipeline
run_openwrt_pipeline
run_tofu_stack "bootstrap/cluster" run_tofu_stack "bootstrap/cluster"
run_tofu_stack "bootstrap/platform" run_tofu_stack "bootstrap/platform"
install_gitea_backup_timer install_gitea_backup_timer