Edge matrix vhost, node-exporter scrape, arr/rpi refinements

- bootstrap/edge: add matrix-vhost template + matrix.conf mount (isolated cert path), config_hash includes matrix.conf so tofu regenerates edge config when it changes; add node-exporter sidecar - bootstrap/platform: prometheus additionalScrapeConfigs for rpi-node and oci-edge-node targets, plus node-exporter edge port 9100 - apps/homelab-alerts: host filesystem >85% alert - infra/arr-stack + rpi-services: compose refinements - .gitignore: ignore tofu tfstate backups (*.tfstate.*.backup)
2026-09-16 11:37:35 -05:00 · 2026-09-16 11:37:35 -05:00 · dce4fa6022
parent fe661f9b24
commit dce4fa6022
10 changed files with 243 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -37,3 +37,7 @@ sops-age.key
 # Ignore local Python bytecode from validation helpers
 __pycache__/
 *.py[cod]
+
+# Tofu state backups
+*.tfstate.*.backup
+*.tfstate.bak*
--- a/apps/homelab-alerts/prometheus-rules.yaml
+++ b/apps/homelab-alerts/prometheus-rules.yaml
@ -52,6 +52,14 @@ spec:
          annotations:
            summary: "PVC {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is over 85% full"
            description: "PVC capacity is near exhaustion. Review app storage before it becomes an outage."
+        - alert: HomelabHostFilesystemNearlyFull
+          expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint=~"/|/data|/var/lib/docker"}) > 85
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Host filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} is over 85% full"
+            description: "The {{ $labels.mountpoint }} mount on {{ $labels.instance }} is near exhaustion. Review disk usage before it becomes an outage."
    - name: homelab.edge
      rules:
        - alert: HomelabTraefik5xxSpike
--- a/bootstrap/edge/main.tf
+++ b/bootstrap/edge/main.tf
@ -39,12 +39,22 @@ locals {
    backend_host = var.backend_host
    backend_port = tostring(var.backend_port)
  })
+  default_conf_matrix = var.matrix_enabled ? templatefile("${path.module}/templates/matrix-vhost.conf.tftpl", {
+    matrix_server_name  = var.matrix_server_name
+    matrix_backend_host = var.matrix_backend_host
+    matrix_backend_port = tostring(var.matrix_backend_port)
+    matrix_cert_dir     = var.matrix_cert_dir
+    element_server_name  = var.element_server_name
+    element_backend_host = var.element_backend_host
+    element_backend_port = tostring(var.element_backend_port)
+  }) : ""
  config_hash = sha256(join("\n---\n", [
    local.compose_file,
    local.default_conf,
    local.default_vcl,
    local.haproxy_cfg,
    local.squid_conf,
+    local.default_conf_matrix,
  ]))
 }

@ -109,6 +119,11 @@ resource "null_resource" "edge_services" {
    destination = "/tmp/homelab-edge/config_files/squid.conf"
  }

+  provisioner "file" {
+    content     = local.default_conf_matrix
+    destination = "/tmp/homelab-edge/config_files/matrix.conf"
+  }
+
  provisioner "remote-exec" {
    inline = [
      <<EOT
@ -184,6 +199,7 @@ sudo cp /tmp/homelab-edge/config_files/default.conf "$install_dir/config_files/d
 sudo cp /tmp/homelab-edge/config_files/default.vcl "$install_dir/config_files/default.vcl"
 sudo cp /tmp/homelab-edge/config_files/haproxy.cfg "$install_dir/config_files/haproxy.cfg"
 sudo cp /tmp/homelab-edge/config_files/squid.conf "$install_dir/config_files/squid.conf"
+sudo cp /tmp/homelab-edge/config_files/matrix.conf "$install_dir/config_files/matrix.conf"

 if [ ! -s "$install_dir/certs/current.crt" ] || [ ! -s "$install_dir/certs/current.key" ]; then
  sudo openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
--- a/bootstrap/edge/templates/docker-compose.yml.tftpl
+++ b/bootstrap/edge/templates/docker-compose.yml.tftpl
@ -9,6 +9,7 @@ services:
      - "443:443"
    volumes:
      - ./config_files/default.conf:/etc/nginx/conf.d/default.conf:ro
+      - ./config_files/matrix.conf:/etc/nginx/conf.d/matrix.conf:ro
      - ./certs:/etc/nginx/certs:ro
      - ./certbot/www:/var/www/certbot:ro
      - nginx_cache:/var/cache/nginx
@ -37,6 +38,21 @@ services:
      - squid_cache:/var/spool/squid
      - squid_logs:/var/log/squid

+  node-exporter:
+    image: prom/node-exporter:latest
+    restart: unless-stopped
+    pid: host
+    ports:
+      - "0.0.0.0:9100:9100/tcp"
+    command:
+      - --path.procfs=/host/proc
+      - --path.sysfs=/host/sys
+      - --path.rootfs=/rootfs
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+
 volumes:
  nginx_cache:
  nginx_dynamic_cache:
--- a/bootstrap/edge/templates/matrix-vhost.conf.tftpl
+++ b/bootstrap/edge/templates/matrix-vhost.conf.tftpl
@ -0,0 +1,76 @@
+#----------------------------------------------------------------------
+# MATRIX + ELEMENT (isolated certs, does NOT touch shared current.crt)
+# Rendered by bootstrap/edge terraform. Keep this file idempotent:
+# it regenerates identically on every apply.
+#----------------------------------------------------------------------
+
+# --- ACME + HTTP redirect for matrix / element ---
+server {
+    listen 80;
+    server_name ${matrix_server_name} ${element_server_name};
+
+    location ^~ /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+        default_type "text/plain";
+        try_files $uri =404;
+    }
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+# --- matrix server_name — Synapse client-server API ---
+server {
+    listen 443 ssl;
+    http2 on;
+    server_name ${matrix_server_name};
+
+    ssl_certificate     ${matrix_cert_dir}/matrix.crt;
+    ssl_certificate_key ${matrix_cert_dir}/matrix.key;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers HIGH:!aNULL:!MD5;
+    ssl_session_cache shared:matrix_ssl:10m;
+    ssl_session_timeout 1d;
+
+    location / {
+        proxy_pass http://${matrix_backend_host}:${matrix_backend_port};
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+        proxy_read_timeout 90s;
+    }
+
+    location /.well-known/matrix {
+        proxy_pass http://${matrix_backend_host}:${matrix_backend_port}/.well-known/matrix;
+        proxy_set_header Host $host;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
+
+# --- element server_name — Element web UI ---
+server {
+    listen 443 ssl;
+    http2 on;
+    server_name ${element_server_name};
+
+    ssl_certificate     ${matrix_cert_dir}/matrix.crt;
+    ssl_certificate_key ${matrix_cert_dir}/matrix.key;
+    ssl_protocols TLSv1.2 TLSv1.3;
+    ssl_ciphers HIGH:!aNULL:!MD5;
+    ssl_session_cache shared:matrix_ssl:10m;
+    ssl_session_timeout 1d;
+
+    location / {
+        proxy_pass http://${element_backend_host}:${element_backend_port};
+        proxy_http_version 1.1;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto https;
+    }
+}
--- a/bootstrap/edge/variables.tf
+++ b/bootstrap/edge/variables.tf
@ -103,6 +103,49 @@ variable "heimdall_backend_port" {
  default = 8082
 }

+# Matrix (isolated vhost + its own cert; does NOT touch shared current.crt)
+variable "matrix_enabled" {
+  type    = bool
+  default = true
+}
+
+variable "matrix_server_name" {
+  type    = string
+  default = "matrix.lab2025.duckdns.org"
+}
+
+# Synapse client-server port on the LAN host. Override to use a Tailscale IP.
+variable "matrix_backend_host" {
+  type    = string
+  default = "192.168.100.73"
+}
+
+variable "matrix_backend_port" {
+  type    = number
+  default = 8008
+}
+
+variable "matrix_cert_dir" {
+  type    = string
+  default = "/etc/nginx/certs"
+}
+
+# Element (shares the matrix isolated cert)
+variable "element_server_name" {
+  type    = string
+  default = "element.lab2025.duckdns.org"
+}
+
+variable "element_backend_host" {
+  type    = string
+  default = "192.168.100.73"
+}
+
+variable "element_backend_port" {
+  type    = number
+  default = 8081
+}
+
 variable "haproxy_stats_user" {
  type    = string
  default = "admin"
--- a/bootstrap/platform/main.tf
+++ b/bootstrap/platform/main.tf
@ -1865,6 +1865,46 @@ resource "helm_release" "prometheus_stack" {
        prometheusSpec = {
          nodeSelector = local.prometheus_stack_node_selector
          retention    = var.observability.prometheus.retention
+          additionalScrapeConfigs = yamlencode([
+            {
+              job_name = "rpi-node"
+              static_configs = [
+                {
+                  targets = ["192.168.100.89:9100"]
+                  labels = {
+                    host_group = "edge-dns-worker"
+                  }
+                }
+              ]
+              metrics_path = "/metrics"
+              scheme       = "http"
+              relabel_configs = [
+                {
+                  source_labels = ["__address__"]
+                  target_label  = "instance"
+                }
+              ]
+            },
+            {
+              job_name = "oci-edge-node"
+              static_configs = [
+                {
+                  targets = ["132.145.170.74:9100"]
+                  labels = {
+                    host_group = "public-edge"
+                  }
+                }
+              ]
+              metrics_path = "/metrics"
+              scheme       = "http"
+              relabel_configs = [
+                {
+                  source_labels = ["__address__"]
+                  target_label  = "instance"
+                }
+              ]
+            }
+          ])
          resources = {
            requests = {
              cpu    = "100m"
--- a/infra/arr-stack/.env.example
+++ b/infra/arr-stack/.env.example
@ -14,6 +14,8 @@ FLARESOLVERR_PORT=8191
 KAPOWARR_PORT=5656
 SUWAYOMI_PORT=4567
 MAINTAINERR_PORT=6246
+JELLYFIN_PORT=8096
+JELLYSEERR_PORT=5055

 # Optional bootstrap values. Leave blank until the apps have created API keys.
 PROWLARR_API_KEY=
--- a/infra/arr-stack/docker-compose.yml
+++ b/infra/arr-stack/docker-compose.yml
@ -141,6 +141,26 @@ services:
    networks:
      - arr

+  jellyfin:
+    <<: *linuxserver-common
+    image: ${JELLYFIN_IMAGE:-lscr.io/linuxserver/jellyfin:latest}
+    container_name: ${JELLYFIN_CONTAINER_NAME:-arr-jellyfin}
+    dns:
+      - ${ARR_DNS_PRIMARY:-1.1.1.1}
+      - ${ARR_DNS_SECONDARY:-8.8.8.8}
+    group_add:
+      - "44"  # video group (for access to DRM render node on the host)
+    devices:
+      - "/dev/dri/renderD128:/dev/dri/renderD128"
+      - "/dev/dri/card0:/dev/dri/card0"
+    ports:
+      - "${JELLYFIN_PORT:-8096}:8096"
+    # /config keeps its state; /data/media carries the media root so libraries
+    # can point at movies/tv/comics/manga/youtube. See README/media layout.
+    volumes:
+      - ${ARR_ROOT:-/data/arr}/config/jellyfin:/config
+      - ${ARR_ROOT:-/data/arr}/media:/data/media
+
 networks:
  arr:
    name: homelab-arr
--- a/infra/rpi-services/docker-compose.yml
+++ b/infra/rpi-services/docker-compose.yml
@ -55,6 +55,24 @@ services:
    networks:
      - dns

+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: ${NODE_EXPORTER_CONTAINER_NAME:-homelab-node-exporter}
+    restart: unless-stopped
+    pid: host
+    ports:
+      - "${NODE_EXPORTER_BIND_IP:-192.168.100.89}:9100:9100/tcp"
+    command:
+      - --path.procfs=/host/proc
+      - --path.sysfs=/host/sys
+      - --path.rootfs=/rootfs
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    networks:
+      - dns
+
 networks:
  dns:
    name: ${RPI_SERVICES_NETWORK:-homelab-rpi-services}