From 180c1b1cca43b3e7a1a7a9330c48647fbfbf2257 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Tue, 26 May 2026 22:05:35 -0600 Subject: [PATCH] Add OpenWrt Pimox VM automation --- README.md | 7 + bootstrap/provisioning/README.md | 43 ++++ lab.sh | 323 +++++++++++++++++++++++++++++++ 3 files changed, 373 insertions(+) diff --git a/README.md b/README.md index b684438..a5ee0e5 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,13 @@ clones on `nvme_thin_pool` by default, checks that the Pimox bridge already exists, refuses `local` as worker clone storage, and refuses to edit Orange Pi host networking. +OpenWrt firewall VM automation is opt-in because it attaches to both WAN and +LAN bridges. Set `LAB_OPENWRT_VM=true` after `vmbr1` already exists on the +Orange Pi. The pipeline downloads the OpenWrt ARM SystemReady EFI image, writes +basic WAN/LAN/firewall config into the image, imports it as VM `9050`, attaches +`vmbr0` as WAN and `vmbr1` as LAN, and stores the VM disk on `nvme_thin_pool`. +It does not use the Debian Kubernetes golden-node template for OpenWrt. + The website and demos images default to `linux/arm64` because both deployments are pinned to the Raspberry Pi worker. Override with `WEBSITE_IMAGE_PLATFORMS` or `DEMOS_IMAGE_PLATFORMS` only if node placement changes. diff --git a/bootstrap/provisioning/README.md b/bootstrap/provisioning/README.md index 2ba15da..359cecb 100644 --- a/bootstrap/provisioning/README.md +++ b/bootstrap/provisioning/README.md @@ -122,3 +122,46 @@ LAB_PIMOX_WORKER_BASE_VMID=9020 ./lab.sh up LAB_PIMOX_WORKER_STORAGE=nvme_thin_pool ./lab.sh up LAB_PIMOX_HOST=192.168.100.80 LAB_PIMOX_BRIDGE=vmbr0 ./lab.sh up ``` + +## OpenWrt firewall VM + +OpenWrt is not built from the Debian golden-node template. The Kubernetes +template remains Debian-only; OpenWrt uses the upstream ARM SystemReady +`armsr/armv8` combined EFI image instead. + +The OpenWrt path is disabled by default. Enable it only after `vmbr1` exists on +the Pimox host and the second NIC/LAN side is safe to use: + +```bash +LAB_OPENWRT_VM=true ./lab.sh up +``` + +Defaults: + +- VMID `9050` +- VM name `openwrt-firewall` +- disk storage `nvme_thin_pool` +- WAN bridge `vmbr0` +- LAN bridge `vmbr1` +- LAN address `192.168.50.1/24` +- LAN DHCP disabled by default +- OpenWrt version `24.10.6` + +Useful overrides: + +```bash +LAB_OPENWRT_VMID=9050 +LAB_OPENWRT_STORAGE=nvme_thin_pool +LAB_OPENWRT_WAN_BRIDGE=vmbr0 +LAB_OPENWRT_LAN_BRIDGE=vmbr1 +LAB_OPENWRT_LAN_IP=192.168.50.1 +LAB_OPENWRT_LAN_NETMASK=255.255.255.0 +LAB_OPENWRT_LAN_DHCP_ENABLED=true +LAB_OPENWRT_START=true +LAB_OPENWRT_VERSION=24.10.6 +LAB_OPENWRT_IMAGE_URL=https://downloads.openwrt.org/releases/24.10.6/targets/armsr/armv8/openwrt-24.10.6-armsr-armv8-generic-ext4-combined-efi.img.gz +``` + +The pipeline validates `vmbr0`, `vmbr1`, and `nvme_thin_pool` on the Pimox host. +It refuses `local` as OpenWrt storage and refuses to create or modify host +network bridges. diff --git a/lab.sh b/lab.sh index 92979ad..1525fc9 100755 --- a/lab.sh +++ b/lab.sh @@ -429,6 +429,328 @@ fi" 2>&1)" export LAB_CLUSTER_VAR_FILE="${var_file}" } +run_openwrt_pipeline() { + local mode="${LAB_OPENWRT_VM:-${LAB_OPENWRT_PIPELINE:-false}}" + local pimox_host="${LAB_PIMOX_HOST:-${TF_VAR_pimox_host:-192.168.100.80}}" + local pimox_user="${LAB_PIMOX_USER:-${TF_VAR_pimox_user:-jv}}" + local pimox_key="${LAB_PIMOX_SSH_KEY_PATH:-${TF_VAR_pimox_ssh_key_path:-/home/jv/.ssh/id_ed25519}}" + local qm_bin="${LAB_PIMOX_QM_BIN:-${TF_VAR_pimox_qm_bin:-/usr/sbin/qm}}" + local vmid="${LAB_OPENWRT_VMID:-9050}" + local vm_name="${LAB_OPENWRT_NAME:-openwrt-firewall}" + local storage="${LAB_OPENWRT_STORAGE:-nvme_thin_pool}" + local wan_bridge="${LAB_OPENWRT_WAN_BRIDGE:-vmbr0}" + local lan_bridge="${LAB_OPENWRT_LAN_BRIDGE:-vmbr1}" + local cores="${LAB_OPENWRT_CORES:-2}" + local memory="${LAB_OPENWRT_MEMORY:-512}" + local version="${LAB_OPENWRT_VERSION:-24.10.6}" + local image_url="${LAB_OPENWRT_IMAGE_URL:-}" + local lan_ip="${LAB_OPENWRT_LAN_IP:-192.168.50.1}" + local lan_netmask="${LAB_OPENWRT_LAN_NETMASK:-255.255.255.0}" + local lan_dhcp_enabled="${LAB_OPENWRT_LAN_DHCP_ENABLED:-false}" + local start_vm="${LAB_OPENWRT_START:-true}" + local root_key_path="${LAB_OPENWRT_ROOT_SSH_PUBLIC_KEY_PATH:-${pimox_key}.pub}" + local root_key_b64="" + local lan_dhcp_ignore="1" + local start_vm_flag="false" + + if disabled_value "${mode}"; then + return 0 + fi + if ! truthy "${mode}"; then + echo "LAB_OPENWRT_VM must be true or false." >&2 + exit 1 + fi + + if [[ -z "${image_url}" ]]; then + image_url="https://downloads.openwrt.org/releases/${version}/targets/armsr/armv8/openwrt-${version}-armsr-armv8-generic-ext4-combined-efi.img.gz" + fi + + if ! [[ "${vmid}" =~ ^[0-9]+$ ]]; then + echo "LAB_OPENWRT_VMID must be a numeric Pimox VMID." >&2 + exit 1 + fi + for value_name in storage wan_bridge lan_bridge vm_name; do + local value="${!value_name}" + if ! [[ "${value}" =~ ^[A-Za-z0-9_.:-]+$ ]]; then + echo "LAB_OPENWRT_${value_name^^} contains unsupported characters." >&2 + exit 1 + fi + done + if [[ "${storage}" == "local" ]]; then + echo "LAB_OPENWRT_STORAGE cannot be local; reserve local storage for the Pimox Debian template." >&2 + exit 1 + fi + if ! [[ "${lan_ip}" =~ ^[0-9.]+$ && "${lan_netmask}" =~ ^[0-9.]+$ ]]; then + echo "LAB_OPENWRT_LAN_IP and LAB_OPENWRT_LAN_NETMASK must be IPv4-style values." >&2 + exit 1 + fi + if truthy "${lan_dhcp_enabled}"; then + lan_dhcp_ignore="0" + fi + if truthy "${start_vm}"; then + start_vm_flag="true" + fi + if [[ -r "${root_key_path}" ]]; then + root_key_b64="$(base64 <"${root_key_path}" | tr -d '\n')" + fi + + echo "Preparing OpenWrt firewall VM ${vmid} on ${pimox_host}; validating ${wan_bridge}, ${lan_bridge}, and ${storage} without changing Orange Pi networking..." + pimox_ssh "${pimox_host}" "${pimox_user}" "${pimox_key}" "bash -s" <&2 + exit 1 +fi + +pvesm_cmd="\$(command -v pvesm 2>/dev/null || true)" +if [ -z "\$pvesm_cmd" ] && [ -x /usr/sbin/pvesm ]; then + pvesm_cmd=/usr/sbin/pvesm +fi +if [ -z "\$pvesm_cmd" ]; then + echo "pvesm was not found; cannot validate Pimox storage \$storage" >&2 + exit 1 +fi + +if ! sudo -n true >/dev/null 2>&1; then + echo "passwordless sudo is required for OpenWrt VM automation" >&2 + exit 1 +fi +if ! ip link show "\$wan_bridge" >/dev/null 2>&1; then + echo "WAN bridge \$wan_bridge does not exist. Refusing to change Orange Pi networking." >&2 + exit 1 +fi +if ! ip link show "\$lan_bridge" >/dev/null 2>&1; then + echo "LAN bridge \$lan_bridge does not exist. Create it manually before enabling OpenWrt automation." >&2 + exit 1 +fi +if ! sudo "\$pvesm_cmd" status | awk -v storage="\$storage" 'NR > 1 && \$1 == storage { found = 1 } END { exit found ? 0 : 1 }'; then + echo "Pimox storage \$storage was not found." >&2 + exit 1 +fi + +if sudo "\$qm_cmd" status "\$vmid" >/dev/null 2>&1; then + if sudo "\$qm_cmd" config "\$vmid" | grep -q '^template: 1$'; then + echo "VM \$vmid exists as a template; refusing to reuse it for OpenWrt." >&2 + exit 1 + fi + sudo "\$qm_cmd" set "\$vmid" \\ + --net0 "virtio,bridge=\$wan_bridge" \\ + --net1 "virtio,bridge=\$lan_bridge" \\ + --cores "\$cores" \\ + --memory "\$memory" \\ + --onboot 1 + if [ "\$start_vm" = "true" ] && sudo "\$qm_cmd" status "\$vmid" | grep -q 'status: stopped'; then + sudo "\$qm_cmd" start "\$vmid" + fi + exit 0 +fi + +for required_cmd in curl gzip losetup mount umount awk sed; do + if ! command -v "\$required_cmd" >/dev/null 2>&1; then + echo "\$required_cmd is required on the Pimox host for OpenWrt image preparation" >&2 + exit 1 + fi +done + +tmp_dir="\$(mktemp -d /tmp/homelab-openwrt.XXXXXX)" +mnt_dir="\$tmp_dir/root" +loopdev="" +cleanup() { + if mountpoint -q "\$mnt_dir" 2>/dev/null; then + sudo umount "\$mnt_dir" || sudo umount -l "\$mnt_dir" || true + fi + if [ -n "\$loopdev" ]; then + sudo losetup -d "\$loopdev" >/dev/null 2>&1 || true + fi + rm -rf "\$tmp_dir" +} +trap cleanup EXIT + +mkdir -p "\$mnt_dir" +curl -fsSL "\$image_url" -o "\$tmp_dir/openwrt.img.gz" +gzip -dc "\$tmp_dir/openwrt.img.gz" >"\$tmp_dir/openwrt.img" + +loopdev="\$(sudo losetup --find --partscan --show "\$tmp_dir/openwrt.img")" +root_part="\${loopdev}p2" +if [ ! -b "\$root_part" ] && echo "\$loopdev" | grep -q 'loop[0-9]\$'; then + root_part="\${loopdev}p2" +fi +if [ ! -b "\$root_part" ]; then + echo "Could not find OpenWrt root partition \$root_part after attaching image." >&2 + exit 1 +fi + +sudo mount "\$root_part" "\$mnt_dir" +sudo mkdir -p "\$mnt_dir/etc/config" "\$mnt_dir/etc/dropbear" "\$mnt_dir/root/.ssh" + +cat >"\$tmp_dir/network" <"\$tmp_dir/dhcp" <"\$tmp_dir/firewall" <<'FIREWALL' +config defaults + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option synflood_protect '1' + +config zone + option name 'lan' + list network 'lan' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'ACCEPT' + +config zone + option name 'wan' + list network 'wan' + option input 'REJECT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option mtu_fix '1' + +config forwarding + option src 'lan' + option dest 'wan' + +config rule + option name 'Allow-DHCP-Renew' + option src 'wan' + option proto 'udp' + option dest_port '68' + option target 'ACCEPT' + option family 'ipv4' + +config rule + option name 'Allow-Ping' + option src 'wan' + option proto 'icmp' + option icmp_type 'echo-request' + option family 'ipv4' + option target 'ACCEPT' +FIREWALL + +cat >"\$tmp_dir/system" <"\$tmp_dir/authorized_keys" + sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/etc/dropbear/authorized_keys" + sudo cp "\$tmp_dir/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" + sudo chmod 0600 "\$mnt_dir/etc/dropbear/authorized_keys" "\$mnt_dir/root/.ssh/authorized_keys" +fi +sync +sudo umount "\$mnt_dir" +sudo losetup -d "\$loopdev" +loopdev="" + +sudo "\$qm_cmd" create "\$vmid" \\ + --name "\$vm_name" \\ + --bios ovmf \\ + --cores "\$cores" \\ + --memory "\$memory" \\ + --net0 "virtio,bridge=\$wan_bridge" \\ + --net1 "virtio,bridge=\$lan_bridge" \\ + --numa 0 \\ + --ostype l26 \\ + --scsihw virtio-scsi-pci \\ + --sockets 1 \\ + --vga virtio \\ + --onboot 1 + +sudo "\$qm_cmd" set "\$vmid" --efidisk0 "\$storage:1,efitype=4m,pre-enrolled-keys=0" +sudo "\$qm_cmd" importdisk "\$vmid" "\$tmp_dir/openwrt.img" "\$storage" --format raw >/dev/null +disk_volume="\$(sudo "\$qm_cmd" config "\$vmid" | awk -F': ' '/^unused[0-9]+:/ { print \$2; exit }')" +if [ -z "\$disk_volume" ]; then + echo "Could not find imported OpenWrt disk volume for VM \$vmid" >&2 + exit 1 +fi +sudo "\$qm_cmd" set "\$vmid" --scsi0 "\$disk_volume" +sudo "\$qm_cmd" set "\$vmid" --boot "order=scsi0" + +if [ "\$start_vm" = "true" ]; then + sudo "\$qm_cmd" start "\$vmid" +fi +EOF +} + cleanup_calico_links() { ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true sudo ip link delete vxlan.calico 2>/dev/null || true @@ -1157,6 +1479,7 @@ up() { echo "Deploying the homelab infrastructure..." run_pimox_pipeline + run_openwrt_pipeline run_tofu_stack "bootstrap/cluster" run_tofu_stack "bootstrap/platform" install_gitea_backup_timer