From dce4fa60222b9cfcd575d3316820ed7b544e6e2d Mon Sep 17 00:00:00 2001
From: jv <jv@debian>
Date: Wed, 16 Sep 2026 11:37:35 -0500
Subject: [PATCH] Edge matrix vhost, node-exporter scrape, arr/rpi refinements

- bootstrap/edge: add matrix-vhost template + matrix.conf mount
  (isolated cert path), config_hash includes matrix.conf so tofu
  regenerates edge config when it changes; add node-exporter sidecar
- bootstrap/platform: prometheus additionalScrapeConfigs for rpi-node
  and oci-edge-node targets, plus node-exporter edge port 9100
- apps/homelab-alerts: host filesystem >85% alert
- infra/arr-stack + rpi-services: compose refinements
- .gitignore: ignore tofu tfstate backups (*.tfstate.*.backup)
---
 .gitignore                                    |  4 +
 apps/homelab-alerts/prometheus-rules.yaml     |  8 ++
 bootstrap/edge/main.tf                        | 16 ++++
 .../edge/templates/docker-compose.yml.tftpl   | 16 ++++
 .../edge/templates/matrix-vhost.conf.tftpl    | 76 +++++++++++++++++++
 bootstrap/edge/variables.tf                   | 43 +++++++++++
 bootstrap/platform/main.tf                    | 40 ++++++++++
 infra/arr-stack/.env.example                  |  2 +
 infra/arr-stack/docker-compose.yml            | 20 +++++
 infra/rpi-services/docker-compose.yml         | 18 +++++
 10 files changed, 243 insertions(+)
 create mode 100644 bootstrap/edge/templates/matrix-vhost.conf.tftpl

diff --git a/.gitignore b/.gitignore
index c359ca9..5fc0b03 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,7 @@ sops-age.key
 # Ignore local Python bytecode from validation helpers
 __pycache__/
 *.py[cod]
+
+# Tofu state backups
+*.tfstate.*.backup
+*.tfstate.bak*
diff --git a/apps/homelab-alerts/prometheus-rules.yaml b/apps/homelab-alerts/prometheus-rules.yaml
index 807ab14..b114433 100644
--- a/apps/homelab-alerts/prometheus-rules.yaml
+++ b/apps/homelab-alerts/prometheus-rules.yaml
@@ -52,6 +52,14 @@ spec:
           annotations:
             summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full"
             description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage."
+        - alert: HomelabHostFilesystemNearlyFull
+          expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker"}) > 85
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Host filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is over 85% full"
+            description: "The {{ $labels.mountpoint }} mount on {{ $labels.instance }} is near exhaustion. Review disk usage before it becomes an outage."
     - name: homelab.edge
       rules:
         - alert: HomelabTraefik5xxSpike
diff --git a/bootstrap/edge/main.tf b/bootstrap/edge/main.tf
index ca49e3d..7e91dab 100644
--- a/bootstrap/edge/main.tf
+++ b/bootstrap/edge/main.tf
@@ -39,12 +39,22 @@ locals {
     backend_host = var.backend_host
     backend_port = tostring(var.backend_port)
   })
+  default_conf_matrix = var.matrix_enabled ? templatefile("${path.module}/templates/matrix-vhost.conf.tftpl", {
+    matrix_server_name  = var.matrix_server_name
+    matrix_backend_host = var.matrix_backend_host
+    matrix_backend_port = tostring(var.matrix_backend_port)
+    matrix_cert_dir     = var.matrix_cert_dir
+    element_server_name  = var.element_server_name
+    element_backend_host = var.element_backend_host
+    element_backend_port = tostring(var.element_backend_port)
+  }) : ""
   config_hash = sha256(join("\n---\n", [
     local.compose_file,
     local.default_conf,
     local.default_vcl,
     local.haproxy_cfg,
     local.squid_conf,
+    local.default_conf_matrix,
   ]))
 }
 
@@ -109,6 +119,11 @@ resource "null_resource" "edge_services" {
     destination = "/tmp/homelab-edge/config_files/squid.conf"
   }
 
+  provisioner "file" {
+    content     = local.default_conf_matrix
+    destination = "/tmp/homelab-edge/config_files/matrix.conf"
+  }
+
   provisioner "remote-exec" {
     inline = [
       <<EOT
@@ -184,6 +199,7 @@ sudo cp /tmp/homelab-edge/config_files/default.conf "$install_dir/config_files/d
 sudo cp /tmp/homelab-edge/config_files/default.vcl "$install_dir/config_files/default.vcl"
 sudo cp /tmp/homelab-edge/config_files/haproxy.cfg "$install_dir/config_files/haproxy.cfg"
 sudo cp /tmp/homelab-edge/config_files/squid.conf "$install_dir/config_files/squid.conf"
+sudo cp /tmp/homelab-edge/config_files/matrix.conf "$install_dir/config_files/matrix.conf"
 
 if [ ! -s "$install_dir/certs/current.crt" ] || [ ! -s "$install_dir/certs/current.key" ]; then
   sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
diff --git a/bootstrap/edge/templates/docker-compose.yml.tftpl b/bootstrap/edge/templates/docker-compose.yml.tftpl
index 5a18073..a1a236a 100644
--- a/bootstrap/edge/templates/docker-compose.yml.tftpl
+++ b/bootstrap/edge/templates/docker-compose.yml.tftpl
@@ -9,6 +9,7 @@ services:
       - "443:443"
     volumes:
       - ./config_files/default.conf:/etc/nginx/conf.d/default.conf:ro
+      - ./config_files/matrix.conf:/etc/nginx/conf.d/matrix.conf:ro
       - ./certs:/etc/nginx/certs:ro
       - ./certbot/www:/var/www/certbot:ro
       - nginx_cache:/var/cache/nginx
@@ -37,6 +38,21 @@ services:
       - squid_cache:/var/spool/squid
       - squid_logs:/var/log/squid
 
+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    pid: host
+    ports:
+      - "0.0.0.0:9100:9100/tcp"
+    command:
+      - --path.procfs=/host/proc
+      - --path.sysfs=/host/sys
+      - --path.rootfs=/rootfs
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+
 volumes:
   nginx_cache:
   nginx_dynamic_cache:
diff --git a/bootstrap/edge/templates/matrix-vhost.conf.tftpl b/bootstrap/edge/templates/matrix-vhost.conf.tftpl
new file mode 100644
index 0000000..b042991
--- /dev/null
+++ b/bootstrap/edge/templates/matrix-vhost.conf.tftpl
@@ -0,0 +1,76 @@
+#----------------------------------------------------------------------
+# MATRIX + ELEMENT (isolated certs, does NOT touch shared current.crt)
+# Rendered by bootstrap/edge terraform. Keep this file idempotent:
+# it regenerates identically on every apply.
+#----------------------------------------------------------------------
+
+# --- ACME + HTTP redirect for matrix / element ---
+server {
+    listen 80;
+    server_name ${matrix_server_name} ${element_server_name};
+
+    location ^~ /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+        default_type "text/plain";
+        try_files $uri =404;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+# --- matrix server_name — Synapse client-server API ---
+server {
+    listen 443 ssl;
+    http2 on;
+    server_name ${matrix_server_name};
+
+    ssl_certificate     ${matrix_cert_dir}/matrix.crt;
+    ssl_certificate_key ${matrix_cert_dir}/matrix.key;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers HIGH:!aNULL:!MD5;
+    ssl_session_cache shared:matrix_ssl:10m;
+    ssl_session_timeout 1d;
+
+    location / {
+        proxy_pass http://${matrix_backend_host}:${matrix_backend_port};
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_read_timeout 90s;
+    }
+
+    location /.well-known/matrix {
+        proxy_pass http://${matrix_backend_host}:${matrix_backend_port}/.well-known/matrix;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
+
+# --- element server_name — Element web UI ---
+server {
+    listen 443 ssl;
+    http2 on;
+    server_name ${element_server_name};
+
+    ssl_certificate     ${matrix_cert_dir}/matrix.crt;
+    ssl_certificate_key ${matrix_cert_dir}/matrix.key;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers HIGH:!aNULL:!MD5;
+    ssl_session_cache shared:matrix_ssl:10m;
+    ssl_session_timeout 1d;
+
+    location / {
+        proxy_pass http://${element_backend_host}:${element_backend_port};
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
diff --git a/bootstrap/edge/variables.tf b/bootstrap/edge/variables.tf
index b5c210c..e2f0d30 100644
--- a/bootstrap/edge/variables.tf
+++ b/bootstrap/edge/variables.tf
@@ -103,6 +103,49 @@ variable "heimdall_backend_port" {
   default = 8082
 }
 
+# Matrix (isolated vhost + its own cert; does NOT touch shared current.crt)
+variable "matrix_enabled" {
+  type    = bool
+  default = true
+}
+
+variable "matrix_server_name" {
+  type    = string
+  default = "matrix.lab2025.duckdns.org"
+}
+
+# Synapse client-server port on the LAN host. Override to use a Tailscale IP.
+variable "matrix_backend_host" {
+  type    = string
+  default = "192.168.100.73"
+}
+
+variable "matrix_backend_port" {
+  type    = number
+  default = 8008
+}
+
+variable "matrix_cert_dir" {
+  type    = string
+  default = "/etc/nginx/certs"
+}
+
+# Element (shares the matrix isolated cert)
+variable "element_server_name" {
+  type    = string
+  default = "element.lab2025.duckdns.org"
+}
+
+variable "element_backend_host" {
+  type    = string
+  default = "192.168.100.73"
+}
+
+variable "element_backend_port" {
+  type    = number
+  default = 8081
+}
+
 variable "haproxy_stats_user" {
   type    = string
   default = "admin"
diff --git a/bootstrap/platform/main.tf b/bootstrap/platform/main.tf
index 244d654..610ec27 100644
--- a/bootstrap/platform/main.tf
+++ b/bootstrap/platform/main.tf
@@ -1865,6 +1865,46 @@ resource "helm_release" "prometheus_stack" {
         prometheusSpec = {
           nodeSelector = local.prometheus_stack_node_selector
           retention    = var.observability.prometheus.retention
+          additionalScrapeConfigs = yamlencode([
+            {
+              job_name = "rpi-node"
+              static_configs = [
+                {
+                  targets = ["192.168.100.89:9100"]
+                  labels = {
+                    host_group = "edge-dns-worker"
+                  }
+                }
+              ]
+              metrics_path = "/metrics"
+              scheme       = "http"
+              relabel_configs = [
+                {
+                  source_labels = ["__address__"]
+                  target_label  = "instance"
+                }
+              ]
+            },
+            {
+              job_name = "oci-edge-node"
+              static_configs = [
+                {
+                  targets = ["132.145.170.74:9100"]
+                  labels = {
+                    host_group = "public-edge"
+                  }
+                }
+              ]
+              metrics_path = "/metrics"
+              scheme       = "http"
+              relabel_configs = [
+                {
+                  source_labels = ["__address__"]
+                  target_label  = "instance"
+                }
+              ]
+            }
+          ])
           resources = {
             requests = {
               cpu    = "100m"
diff --git a/infra/arr-stack/.env.example b/infra/arr-stack/.env.example
index ae3362f..aa35503 100644
--- a/infra/arr-stack/.env.example
+++ b/infra/arr-stack/.env.example
@@ -14,6 +14,8 @@ FLARESOLVERR_PORT=8191
 KAPOWARR_PORT=5656
 SUWAYOMI_PORT=4567
 MAINTAINERR_PORT=6246
+JELLYFIN_PORT=8096
+JELLYSEERR_PORT=5055
 
 # Optional bootstrap values. Leave blank until the apps have created API keys.
 PROWLARR_API_KEY=
diff --git a/infra/arr-stack/docker-compose.yml b/infra/arr-stack/docker-compose.yml
index 59b82ab..f9fa00c 100644
--- a/infra/arr-stack/docker-compose.yml
+++ b/infra/arr-stack/docker-compose.yml
@@ -141,6 +141,26 @@ services:
     networks:
       - arr
 
+  jellyfin:
+    <<: *linuxserver-common
+    image: ${JELLYFIN_IMAGE:-lscr.io/linuxserver/jellyfin:latest}
+    container_name: ${JELLYFIN_CONTAINER_NAME:-arr-jellyfin}
+    dns:
+      - ${ARR_DNS_PRIMARY:-1.1.1.1}
+      - ${ARR_DNS_SECONDARY:-8.8.8.8}
+    group_add:
+      - "44"  # video group (for access to DRM render node on the host)
+    devices:
+      - "/dev/dri/renderD128:/dev/dri/renderD128"
+      - "/dev/dri/card0:/dev/dri/card0"
+    ports:
+      - "${JELLYFIN_PORT:-8096}:8096"
+    # /config keeps its state; /data/media carries the media root so libraries
+    # can point at movies/tv/comics/manga/youtube. See README/media layout.
+    volumes:
+      - ${ARR_ROOT:-/data/arr}/config/jellyfin:/config
+      - ${ARR_ROOT:-/data/arr}/media:/data/media
+
 networks:
   arr:
     name: homelab-arr
diff --git a/infra/rpi-services/docker-compose.yml b/infra/rpi-services/docker-compose.yml
index 3668f53..625381a 100644
--- a/infra/rpi-services/docker-compose.yml
+++ b/infra/rpi-services/docker-compose.yml
@@ -55,6 +55,24 @@ services:
     networks:
       - dns
 
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: ${NODE_EXPORTER_CONTAINER_NAME:-homelab-node-exporter}
+    restart: unless-stopped
+    pid: host
+    ports:
+      - "${NODE_EXPORTER_BIND_IP:-192.168.100.89}:9100:9100/tcp"
+    command:
+      - --path.procfs=/host/proc
+      - --path.sysfs=/host/sys
+      - --path.rootfs=/rootfs
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    networks:
+      - dns
+
 networks:
   dns:
     name: ${RPI_SERVICES_NETWORK:-homelab-rpi-services}