diff --git a/README.md b/README.md index d4fb17e..879bd35 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,8 @@ accidentally modify the cluster. listens on loopback inside the container, OPcache is enabled, and pre-rendered HTML handles the read-heavy pages - default apps are `container-registry`, `website-production`, - `demos-static`, `heimdall`, `n8n`, and `supply-chain-policy` + `demos-static`, the `heimdall` route bundle, `n8n`, and + `supply-chain-policy` 5. `bootstrap/edge` - connects to the OCI jump box @@ -460,9 +461,9 @@ kubectl -n argocd get applications kubectl -n container-registry get pods kubectl -n website-production get pods -o wide kubectl -n demos-static get pods -o wide -kubectl -n heimdall get pods -o wide ssh jv@192.168.100.73 'cd /data/homelab-gitea && sudo docker compose ps' +ssh jv@192.168.100.73 'cd /data/homelab-heimdall && sudo docker compose ps' docker info --format '{{.DockerRootDir}}' df -h / /data /data/openebs/local /data/docker @@ -907,7 +908,6 @@ Traefik's address: ```text 192.168.100.240 lab2025.duckdns.org 192.168.100.240 demos.lab2025.duckdns.org -192.168.100.240 heimdall.lab2025.duckdns.org 192.168.100.240 n8n.lab2025.duckdns.org 192.168.100.240 prowlarr.lab2025.duckdns.org 192.168.100.240 sonarr.lab2025.duckdns.org @@ -920,6 +920,10 @@ Traefik's address: 192.168.100.240 argocd.lab2025.duckdns.org ``` +Heimdall is external to Kubernetes. For direct LAN testing, use +`http://192.168.100.73:8082/`; public access still uses +`https://heimdall.lab2025.duckdns.org/` through the OCI edge. + The edge stack uses Traefik as its backend by default and validates `http://192.168.100.240:80/` before updating the edge containers. If Tailscale subnet-route approval is not automatic in the tailnet policy, the edge deploy @@ -994,11 +998,11 @@ export TF_VAR_metallb='{ }' ``` -Traefik uses MetalLB for a LAN `LoadBalancer` address. App services such as the -website, demos, and Heimdall should be `ClusterIP` services behind Kubernetes +Traefik uses MetalLB for a LAN `LoadBalancer` address. Kubernetes app services +such as the website, demos, and n8n should be `ClusterIP` services behind Ingress objects. The local registry remains a `NodePort` because the cluster -nodes use it as a pull endpoint. Gitea is not a Kubernetes service; it runs on -the Debian Docker host. +nodes use it as a pull endpoint. Gitea and Heimdall are not Kubernetes services; +they run on the Debian Docker host. ## Secrets @@ -1074,16 +1078,11 @@ Add Kubernetes manifests under `apps/` and register them in `bootstrap/apps`'s `applications` map. Argo CD will own sync, pruning, and self-healing for the app. -The `heimdall` app is intentionally waited on at the end of `./jeannie apps`. -It runs the LinuxServer.io Heimdall dashboard, persists `/config` on -OpenEBS, and seeds tiles for the website, demo apps, Gitea, Grafana, -Prometheus, Alertmanager, Argo CD, the local registry, and Heimdall itself. -Its manifest only selects Linux nodes; the bound OpenEBS local PV supplies the -node affinity. Do not also pin it to a workload class unless the retained PV is -rebuilt on a node with that label. -Because Heimdall does not support reverse-proxy subfolder hosting cleanly, it -is exposed through the dedicated hostname `heimdall.lab2025.duckdns.org` rather -than a `/heimdall/` path. +The `heimdall` Argo CD app name is retained as a small route bundle for +Grafana, Prometheus, Alertmanager, and Argo CD ingresses so the next app sync can +prune the old in-cluster Heimdall workload cleanly. Do not add new Kubernetes +workloads there. Use `./jeannie deploy-heimdall` for the dashboard +itself. The `n8n` app runs in the `n8n` namespace with retained OpenEBS storage and is exposed at `https://n8n.lab2025.duckdns.org`. It receives @@ -1128,6 +1127,25 @@ Public source browsing stays available through can view public repositories, so the blog can link to code read-only while writes still require an authenticated Gitea account. +## Heimdall + +Heimdall is external bootstrap infrastructure. It runs on the Debian host as an +always-on Docker Compose service from `infra/heimdall/docker-compose.yml`, not +as a Kubernetes workload. This keeps the lab navigation page available when the +cluster is destroyed, rebuilding, or unhealthy. + +Deploy or refresh it from the Debian server with: + +```bash +./jeannie deploy-heimdall +``` + +`./jeannie up` also runs this step during the independent host +services stage unless `LAB_HEIMDALL_DEPLOY=false` is set. The default data path +is `/data/homelab-heimdall/data/config`, and the local HTTP port is `8082`. +Dashboard links are seeded from `infra/heimdall/links.json` by a sidecar +container that upserts repo-managed tiles without deleting manually added ones. + The Debian bare repo remains the GitOps mirror: ```text diff --git a/README.md.tmpl b/README.md.tmpl index 79cbeed..bca134b 100644 --- a/README.md.tmpl +++ b/README.md.tmpl @@ -90,7 +90,8 @@ accidentally modify the cluster. listens on loopback inside the container, OPcache is enabled, and pre-rendered HTML handles the read-heavy pages - default apps are `container-registry`, `website-production`, - `demos-static`, `heimdall`, `n8n`, and `supply-chain-policy` + `demos-static`, the `heimdall` route bundle, `n8n`, and + `supply-chain-policy` 5. `bootstrap/edge` - connects to the OCI jump box @@ -460,9 +461,9 @@ kubectl -n argocd get applications kubectl -n container-registry get pods kubectl -n website-production get pods -o wide kubectl -n demos-static get pods -o wide -kubectl -n heimdall get pods -o wide ssh jv@192.168.100.73 'cd /data/homelab-gitea && sudo docker compose ps' +ssh jv@192.168.100.73 'cd /data/homelab-heimdall && sudo docker compose ps' docker info --format '{{.DockerRootDir}}' df -h / /data /data/openebs/local /data/docker @@ -907,7 +908,6 @@ Traefik's address: ```text 192.168.100.240 lab2025.duckdns.org 192.168.100.240 demos.lab2025.duckdns.org -192.168.100.240 heimdall.lab2025.duckdns.org 192.168.100.240 n8n.lab2025.duckdns.org 192.168.100.240 prowlarr.lab2025.duckdns.org 192.168.100.240 sonarr.lab2025.duckdns.org @@ -920,6 +920,10 @@ Traefik's address: 192.168.100.240 argocd.lab2025.duckdns.org ``` +Heimdall is external to Kubernetes. For direct LAN testing, use +`http://192.168.100.73:8082/`; public access still uses +`https://heimdall.lab2025.duckdns.org/` through the OCI edge. + The edge stack uses Traefik as its backend by default and validates `http://192.168.100.240:80/` before updating the edge containers. If Tailscale subnet-route approval is not automatic in the tailnet policy, the edge deploy @@ -994,11 +998,11 @@ export TF_VAR_metallb='{ }' ``` -Traefik uses MetalLB for a LAN `LoadBalancer` address. App services such as the -website, demos, and Heimdall should be `ClusterIP` services behind Kubernetes +Traefik uses MetalLB for a LAN `LoadBalancer` address. Kubernetes app services +such as the website, demos, and n8n should be `ClusterIP` services behind Ingress objects. The local registry remains a `NodePort` because the cluster -nodes use it as a pull endpoint. Gitea is not a Kubernetes service; it runs on -the Debian Docker host. +nodes use it as a pull endpoint. Gitea and Heimdall are not Kubernetes services; +they run on the Debian Docker host. ## Secrets @@ -1074,16 +1078,11 @@ Add Kubernetes manifests under `apps/` and register them in `bootstrap/apps`'s `applications` map. Argo CD will own sync, pruning, and self-healing for the app. -The `heimdall` app is intentionally waited on at the end of `./{{ main_script }} apps`. -It runs the LinuxServer.io Heimdall dashboard, persists `/config` on -OpenEBS, and seeds tiles for the website, demo apps, Gitea, Grafana, -Prometheus, Alertmanager, Argo CD, the local registry, and Heimdall itself. -Its manifest only selects Linux nodes; the bound OpenEBS local PV supplies the -node affinity. Do not also pin it to a workload class unless the retained PV is -rebuilt on a node with that label. -Because Heimdall does not support reverse-proxy subfolder hosting cleanly, it -is exposed through the dedicated hostname `heimdall.lab2025.duckdns.org` rather -than a `/heimdall/` path. +The `heimdall` Argo CD app name is retained as a small route bundle for +Grafana, Prometheus, Alertmanager, and Argo CD ingresses so the next app sync can +prune the old in-cluster Heimdall workload cleanly. Do not add new Kubernetes +workloads there. Use `./{{ main_script }} deploy-heimdall` for the dashboard +itself. The `n8n` app runs in the `n8n` namespace with retained OpenEBS storage and is exposed at `https://n8n.lab2025.duckdns.org`. It receives @@ -1128,6 +1127,25 @@ Public source browsing stays available through can view public repositories, so the blog can link to code read-only while writes still require an authenticated Gitea account. +## Heimdall + +Heimdall is external bootstrap infrastructure. It runs on the Debian host as an +always-on Docker Compose service from `infra/heimdall/docker-compose.yml`, not +as a Kubernetes workload. This keeps the lab navigation page available when the +cluster is destroyed, rebuilding, or unhealthy. + +Deploy or refresh it from the Debian server with: + +```bash +./{{ main_script }} deploy-heimdall +``` + +`./{{ main_script }} up` also runs this step during the independent host +services stage unless `LAB_HEIMDALL_DEPLOY=false` is set. The default data path +is `/data/homelab-heimdall/data/config`, and the local HTTP port is `8082`. +Dashboard links are seeded from `infra/heimdall/links.json` by a sidecar +container that upserts repo-managed tiles without deleting manually added ones. + The Debian bare repo remains the GitOps mirror: ```text diff --git a/apps/demos-static/public/homelab-catalog/index.html b/apps/demos-static/public/homelab-catalog/index.html index 8a1e6b2..106f01a 100644 --- a/apps/demos-static/public/homelab-catalog/index.html +++ b/apps/demos-static/public/homelab-catalog/index.html @@ -178,11 +178,11 @@ Heimdall - Kubernetes - Argo CD / apps/heimdall - http://192.168.100.240/ + debian (control-plane, 192.168.100.73) + Docker Compose / jeannie deploy-heimdall + http://192.168.100.73:8082/ https://heimdall.lab2025.duckdns.org/ - apps/heimdall/ + infra/heimdall/README.md n8n diff --git a/apps/heimdall/README.md b/apps/heimdall/README.md new file mode 100644 index 0000000..f29ef40 --- /dev/null +++ b/apps/heimdall/README.md @@ -0,0 +1,8 @@ +# Heimdall Route Bundle + +This Argo CD app name is retained for compatibility and pruning after Heimdall +moved out of Kubernetes. It now carries only platform ingress routes for +Grafana, Prometheus, Alertmanager, and Argo CD. + +The Heimdall dashboard itself is managed by `./jeannie deploy-heimdall` from +`infra/heimdall/` and runs on the Debian host with Docker Compose. diff --git a/apps/heimdall/ingress.yaml b/apps/heimdall/ingress.yaml index f493c2f..16fff66 100644 --- a/apps/heimdall/ingress.yaml +++ b/apps/heimdall/ingress.yaml @@ -1,24 +1,5 @@ apiVersion: networking.k8s.io/v1 kind: Ingress -metadata: - name: heimdall - namespace: heimdall -spec: - ingressClassName: traefik - rules: - - host: heimdall.lab2025.duckdns.org - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: heimdall - port: - number: 80 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress metadata: name: grafana namespace: monitoring diff --git a/apps/heimdall/kustomization.yaml b/apps/heimdall/kustomization.yaml index 4ee1166..972f3ed 100644 --- a/apps/heimdall/kustomization.yaml +++ b/apps/heimdall/kustomization.yaml @@ -1,6 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - namespace.yaml - - web-app.yaml - ingress.yaml diff --git a/apps/heimdall/namespace.yaml b/apps/heimdall/namespace.yaml deleted file mode 100644 index c042b6b..0000000 --- a/apps/heimdall/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: heimdall diff --git a/apps/heimdall/web-app.yaml b/apps/heimdall/web-app.yaml deleted file mode 100644 index 9b2fc54..0000000 --- a/apps/heimdall/web-app.yaml +++ /dev/null @@ -1,413 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: heimdall-config - namespace: heimdall -spec: - accessModes: - - ReadWriteOnce - storageClassName: openebs-hostpath-retain - resources: - requests: - storage: 1Gi ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: heimdall-link-seed - namespace: heimdall -data: - links.json: | - [ - { - "title": "Website", - "url": "https://lab2025.duckdns.org/", - "description": "Public portfolio website", - "colour": "#0f766e", - "class": "Website" - }, - { - "title": "Demo Apps", - "url": "https://demos.lab2025.duckdns.org/", - "description": "Static browser-side demo catalog", - "colour": "#2563eb", - "class": "Demos" - }, - { - "title": "Service Catalog", - "url": "https://demos.lab2025.duckdns.org/homelab-catalog/", - "description": "Generated host, service, route, and runbook map from homelab.yml", - "colour": "#0891b2", - "class": "Catalog" - }, - { - "title": "Gitea", - "url": "https://lab2025.duckdns.org/git/", - "description": "External Git service on the Debian host", - "colour": "#609926", - "class": "Gitea" - }, - { - "title": "Grafana", - "url": "https://grafana.lab2025.duckdns.org/", - "description": "Monitoring dashboards", - "colour": "#f97316", - "class": "Grafana" - }, - { - "title": "Prometheus", - "url": "https://prometheus.lab2025.duckdns.org/", - "description": "Prometheus query UI", - "colour": "#dc2626", - "class": "Prometheus" - }, - { - "title": "Alertmanager", - "url": "https://alertmanager.lab2025.duckdns.org/", - "description": "Alert routing and silences", - "colour": "#b91c1c", - "class": "Alertmanager" - }, - { - "title": "Argo CD", - "url": "https://argocd.lab2025.duckdns.org/", - "description": "GitOps application sync status", - "colour": "#0ea5e9", - "class": "ArgoCD" - }, - { - "title": "Container Registry", - "url": "http://100.77.80.72:30500/v2/_catalog", - "description": "Local image registry catalog endpoint", - "colour": "#334155", - "class": "Docker" - }, - { - "title": "Heimdall", - "url": "https://heimdall.lab2025.duckdns.org/", - "description": "Homelab service dashboard", - "colour": "#7c3aed", - "class": "Heimdall" - }, - { - "title": "n8n", - "url": "https://n8n.lab2025.duckdns.org/", - "description": "Workflow automation for translation jobs and homelab tasks", - "colour": "#ea4b71", - "class": "N8N" - }, - { - "title": "Prowlarr", - "url": "https://prowlarr.lab2025.duckdns.org/", - "description": "Indexer management for the media automation stack", - "colour": "#2563eb", - "class": "Prowlarr" - }, - { - "title": "Sonarr", - "url": "https://sonarr.lab2025.duckdns.org/", - "description": "TV library automation", - "colour": "#22c55e", - "class": "Sonarr" - }, - { - "title": "Radarr", - "url": "https://radarr.lab2025.duckdns.org/", - "description": "Movie library automation", - "colour": "#f59e0b", - "class": "Radarr" - }, - { - "title": "qBittorrent", - "url": "https://qbittorrent.lab2025.duckdns.org/", - "description": "Torrent download client", - "colour": "#3b82f6", - "class": "Qbittorrent" - }, - { - "title": "Kapowarr", - "url": "https://kapowarr.lab2025.duckdns.org/", - "description": "Comics library automation", - "colour": "#ef4444", - "class": "Kapowarr" - }, - { - "title": "Suwayomi", - "url": "https://suwayomi.lab2025.duckdns.org/", - "description": "Manga server and scraper engine", - "colour": "#8b5cf6", - "class": "Suwayomi" - }, - { - "title": "Maintainerr", - "url": "https://maintainerr.lab2025.duckdns.org/", - "description": "Media list maintenance automation", - "colour": "#14b8a6", - "class": "Maintainerr" - } - ] - seed.py: | - import json - import os - import sqlite3 - import time - from datetime import datetime, timezone - - DB_CANDIDATES = ( - "/config/www/app.sqlite", - "/config/app.sqlite", - ) - LINKS_PATH = "/seed/links.json" - - def find_database(): - while True: - for path in DB_CANDIDATES: - if os.path.exists(path): - return path - time.sleep(5) - - def table_exists(conn, name): - row = conn.execute( - "select name from sqlite_master where type = 'table' and name = ?", - (name,), - ).fetchone() - return row is not None - - def columns(conn, table): - return {row[1] for row in conn.execute(f"pragma table_info({table})")} - - def quote_identifier(name): - return '"' + name.replace('"', '""') + '"' - - def ensure_public_dashboard(conn, user_id): - if not table_exists(conn, "users"): - return - user_columns = columns(conn, "users") - if "public_front" not in user_columns: - return - conn.execute( - 'update "users" set "public_front" = 1 where "id" = ?', - (user_id,), - ) - - def ensure_dashboard_tag(conn, item_id, tag_id, now): - if not table_exists(conn, "item_tag"): - return - item_tag_columns = columns(conn, "item_tag") - if not {"item_id", "tag_id"}.issubset(item_tag_columns): - return - existing = conn.execute( - 'select 1 from "item_tag" where "item_id" = ? and "tag_id" = ?', - (item_id, tag_id), - ).fetchone() - if existing: - return - values = { - "item_id": item_id, - "tag_id": tag_id, - "created_at": now, - "updated_at": now, - } - insert_columns = [name for name in values if name in item_tag_columns] - placeholders = ", ".join("?" for _ in insert_columns) - quoted_insert_columns = ", ".join(quote_identifier(name) for name in insert_columns) - conn.execute( - f'insert into "item_tag" ({quoted_insert_columns}) values ({placeholders})', - [values[name] for name in insert_columns], - ) - - def home_dashboard_tag_id(conn): - item_columns = columns(conn, "items") - if not {"id", "title"}.issubset(item_columns): - return 0 - row = conn.execute( - 'select "id" from "items" where "title" = ? order by "id" limit 1', - ("app.dashboard",), - ).fetchone() - if row: - return row[0] - return 0 - - def wait_for_items_table(db_path): - while True: - try: - conn = sqlite3.connect(db_path) - if table_exists(conn, "items"): - return conn - conn.close() - except sqlite3.Error: - pass - time.sleep(5) - - def upsert_links(conn, links): - item_columns = columns(conn, "items") - seed_user_id = 1 - home_dashboard_tag = home_dashboard_tag_id(conn) - now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") - - ensure_public_dashboard(conn, seed_user_id) - - for order, link in enumerate(links): - existing = conn.execute( - 'select "id" from "items" where "title" = ?', - (link["title"],), - ).fetchone() - values = { - "title": link["title"], - "url": link["url"], - "description": None, - "colour": link.get("colour"), - "pinned": 1, - "order": order, - "type": 0, - "user_id": seed_user_id, - "class": None, - "appid": None, - "appdescription": link.get("description"), - "created_at": now, - "updated_at": now, - "deleted_at": None, - } - - if existing: - update_columns = [ - name for name in values - if name in item_columns and name not in ("id", "title", "created_at") - ] - assignments = ", ".join(f"{quote_identifier(name)} = ?" for name in update_columns) - params = [values[name] for name in update_columns] - params.append(existing[0]) - conn.execute(f'update "items" set {assignments} where "id" = ?', params) - ensure_dashboard_tag(conn, existing[0], home_dashboard_tag, now) - continue - - insert_columns = [ - name for name in values - if name in item_columns and name != "deleted_at" - ] - placeholders = ", ".join("?" for _ in insert_columns) - quoted_insert_columns = ", ".join(quote_identifier(name) for name in insert_columns) - conn.execute( - f'insert into "items" ({quoted_insert_columns}) values ({placeholders})', - [values[name] for name in insert_columns], - ) - new_item_id = conn.execute("select last_insert_rowid()").fetchone()[0] - ensure_dashboard_tag(conn, new_item_id, home_dashboard_tag, now) - - conn.commit() - - with open(LINKS_PATH, encoding="utf-8") as links_file: - links = json.load(links_file) - - while True: - db_path = find_database() - connection = wait_for_items_table(db_path) - try: - upsert_links(connection, links) - time.sleep(3600) - except sqlite3.Error as error: - print(f"Failed to seed Heimdall links: {error}", flush=True) - time.sleep(10) - finally: - connection.close() ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: heimdall - namespace: heimdall - labels: - app: heimdall -spec: - replicas: 1 - strategy: - type: Recreate - selector: - matchLabels: - app: heimdall - template: - metadata: - labels: - app: heimdall - spec: - nodeSelector: - kubernetes.io/os: linux - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: OnRootMismatch - containers: - - name: heimdall - image: lscr.io/linuxserver/heimdall:v2.7.6-ls347 - imagePullPolicy: IfNotPresent - env: - - name: PUID - value: "1000" - - name: PGID - value: "1000" - - name: TZ - value: America/Mexico_City - - name: APP_URL - value: "https://heimdall.lab2025.duckdns.org" - - name: ASSET_URL - value: "https://heimdall.lab2025.duckdns.org" - ports: - - containerPort: 80 - name: http - readinessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 20 - periodSeconds: 10 - livenessProbe: - httpGet: - path: / - port: http - initialDelaySeconds: 60 - periodSeconds: 30 - resources: - requests: - cpu: 25m - memory: 128Mi - limits: - memory: 512Mi - volumeMounts: - - name: heimdall-config - mountPath: /config - - name: link-seeder - image: python:3.12-alpine - imagePullPolicy: IfNotPresent - command: - - python - - /seed/seed.py - resources: - requests: - cpu: 5m - memory: 32Mi - limits: - memory: 96Mi - volumeMounts: - - name: heimdall-config - mountPath: /config - - name: link-seed - mountPath: /seed - readOnly: true - volumes: - - name: heimdall-config - persistentVolumeClaim: - claimName: heimdall-config - - name: link-seed - configMap: - name: heimdall-link-seed ---- -apiVersion: v1 -kind: Service -metadata: - name: heimdall - namespace: heimdall -spec: - ports: - - port: 80 - targetPort: http - selector: - app: heimdall diff --git a/bootstrap/apps/variables.tf b/bootstrap/apps/variables.tf index d91e3f5..fbcde01 100644 --- a/bootstrap/apps/variables.tf +++ b/bootstrap/apps/variables.tf @@ -66,14 +66,16 @@ variable "applications" { self_heal = true create_namespace = true } + # Retained as the old Argo app identity so pruning removes the former + # in-cluster Heimdall workload; now only carries platform ingress routes. heimdall = { project = "default" path = "apps/heimdall" - namespace = "heimdall" + namespace = "monitoring" target_revision = "main" prune = true self_heal = true - create_namespace = true + create_namespace = false } n8n = { project = "default" diff --git a/bootstrap/edge/main.tf b/bootstrap/edge/main.tf index 76ba5cf..ca49e3d 100644 --- a/bootstrap/edge/main.tf +++ b/bootstrap/edge/main.tf @@ -9,16 +9,23 @@ terraform { } locals { - compose_file = templatefile("${path.module}/templates/docker-compose.yml.tftpl", {}) - server_names = distinct(concat([var.server_name], var.additional_server_names)) + compose_file = templatefile("${path.module}/templates/docker-compose.yml.tftpl", {}) + heimdall_server_name = "heimdall.${var.server_name}" + heimdall_backend_host = var.heimdall_backend_host != "" ? var.heimdall_backend_host : var.gitea_backend_host + server_names = distinct(concat([var.server_name, local.heimdall_server_name], var.additional_server_names)) + main_server_names = [for name in local.server_names : name if name != local.heimdall_server_name] default_conf = templatefile("${path.module}/templates/default.conf.tftpl", { - server_name = var.server_name - server_names = join(" ", local.server_names) - backend_host = var.backend_host - backend_port = tostring(var.backend_port) - demos_backend_port = var.demos_backend_port - gitea_backend_host = var.gitea_backend_host - gitea_backend_port = var.gitea_backend_port + server_name = var.server_name + server_names = join(" ", local.server_names) + main_server_names = join(" ", local.main_server_names) + backend_host = var.backend_host + backend_port = tostring(var.backend_port) + demos_backend_port = var.demos_backend_port + gitea_backend_host = var.gitea_backend_host + gitea_backend_port = var.gitea_backend_port + heimdall_server_name = local.heimdall_server_name + heimdall_backend_host = local.heimdall_backend_host + heimdall_backend_port = tostring(var.heimdall_backend_port) }) default_vcl = templatefile("${path.module}/templates/default.vcl.tftpl", { backend_host = var.backend_host @@ -54,6 +61,8 @@ resource "null_resource" "edge_services" { backend_port = tostring(var.backend_port) gitea_backend_host = var.gitea_backend_host gitea_backend_port = tostring(var.gitea_backend_port) + heimdall_host = local.heimdall_backend_host + heimdall_port = tostring(var.heimdall_backend_port) enable_letsencrypt = tostring(var.enable_letsencrypt) letsencrypt_email = var.letsencrypt_email letsencrypt_staging = tostring(var.letsencrypt_staging) @@ -114,6 +123,8 @@ backend_host="${self.triggers.backend_host}" backend_port="${self.triggers.backend_port}" gitea_backend_host="${self.triggers.gitea_backend_host}" gitea_backend_port="${self.triggers.gitea_backend_port}" +heimdall_host="${self.triggers.heimdall_host}" +heimdall_port="${self.triggers.heimdall_port}" enable_letsencrypt="${self.triggers.enable_letsencrypt}" letsencrypt_email="${self.triggers.letsencrypt_email}" letsencrypt_staging="${self.triggers.letsencrypt_staging}" @@ -145,6 +156,11 @@ if ! curl -sS --connect-timeout 10 "http://$gitea_backend_host:$gitea_backend_po exit 1 fi +if ! curl -sS --connect-timeout 10 "http://$heimdall_host:$heimdall_port/" >/dev/null; then + echo "Warning: cannot reach Heimdall backend http://$heimdall_host:$heimdall_port/ from the edge host." >&2 + echo "Run ./jeannie deploy-heimdall on the Debian host if the service is missing." >&2 +fi + if ! command -v docker >/dev/null 2>&1; then if [ "$install_docker" = "true" ]; then curl -fsSL https://get.docker.com | sudo sh diff --git a/bootstrap/edge/templates/default.conf.tftpl b/bootstrap/edge/templates/default.conf.tftpl index 032c2ea..1fad3ec 100644 --- a/bootstrap/edge/templates/default.conf.tftpl +++ b/bootstrap/edge/templates/default.conf.tftpl @@ -13,6 +13,11 @@ upstream homelab_backend { keepalive 16; } +upstream heimdall_backend { + server ${heimdall_backend_host}:${heimdall_backend_port}; + keepalive 8; +} + set_real_ip_from 173.245.48.0/20; set_real_ip_from 103.21.244.0/22; set_real_ip_from 103.22.200.0/22; @@ -32,7 +37,22 @@ real_ip_header CF-Connecting-IP; server { listen 80; - server_name ${server_names}; + server_name ${main_server_names}; + + location ^~ /.well-known/acme-challenge/ { + root /var/www/certbot; + default_type "text/plain"; + try_files $uri =404; + } + + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 80; + server_name ${heimdall_server_name}; location ^~ /.well-known/acme-challenge/ { root /var/www/certbot; @@ -48,7 +68,55 @@ server { server { listen 443 ssl; http2 on; - server_name ${server_names}; + server_name ${heimdall_server_name}; + + ssl_certificate /etc/nginx/certs/current.crt; + ssl_certificate_key /etc/nginx/certs/current.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_session_cache shared:homelab_ssl:10m; + ssl_session_timeout 1d; + + keepalive_timeout 65s; + keepalive_requests 1000; + + if ($blocked_uris) { return 403; } + + add_header X-Frame-Options SAMEORIGIN; + add_header X-XSS-Protection "1; mode=block"; + add_header X-Content-Type-Options nosniff; + add_header Referrer-Policy "strict-origin-when-cross-origin"; + add_header Permissions-Policy "geolocation=(), microphone=()"; + + gzip on; + gzip_comp_level 2; + gzip_min_length 1000; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + location = /edge-health { + access_log off; + return 204; + } + + location / { + limit_req zone=one burst=20 nodelay; + + proxy_pass http://heimdall_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header CF-Connecting-IP $http_cf_connecting_ip; + add_header Cache-Control "no-store"; + } +} + +server { + listen 443 ssl; + http2 on; + server_name ${main_server_names}; ssl_certificate /etc/nginx/certs/current.crt; ssl_certificate_key /etc/nginx/certs/current.key; diff --git a/bootstrap/edge/variables.tf b/bootstrap/edge/variables.tf index 3e1b03f..b5c210c 100644 --- a/bootstrap/edge/variables.tf +++ b/bootstrap/edge/variables.tf @@ -93,6 +93,16 @@ variable "gitea_backend_host" { } } +variable "heimdall_backend_host" { + type = string + default = "" +} + +variable "heimdall_backend_port" { + type = number + default = 8082 +} + variable "haproxy_stats_user" { type = string default = "admin" diff --git a/docs/jeannie.1.md b/docs/jeannie.1.md index 0ab4448..53adfe7 100644 --- a/docs/jeannie.1.md +++ b/docs/jeannie.1.md @@ -264,6 +264,9 @@ needed. `deploy-gitea` : Deploy the external Gitea Docker Compose service on the Debian host. +`deploy-heimdall` +: Deploy the external Heimdall Docker Compose dashboard on the Debian host. + `rpi-services` : Deploy Pi-hole, Unbound, and Uptime Kuma on the Raspberry Pi. @@ -586,6 +589,20 @@ to `false`; SSH is the default reachability check. : Allow `deploy-gitea` to install Docker through `get.docker.com` when Docker is missing. Default is to fail and ask for host bootstrap. +`LAB_HEIMDALL_DEPLOY=false` +: Skip the Debian-hosted Heimdall Compose deploy during `up`. + +`LAB_HEIMDALL_INSTALL_DIR` +: Install root for the Debian-hosted Heimdall Compose stack. Defaults to +`/data/homelab-heimdall`. + +`LAB_HEIMDALL_HTTP_PORT` +: Local HTTP port for the Debian-hosted Heimdall container. Defaults to `8082`. + +`LAB_HEIMDALL_INSTALL_DOCKER=true` +: Allow `deploy-heimdall` to install Docker through `get.docker.com` when Docker +is missing. Default is to fail and ask for host bootstrap. + `LAB_RPI_INSTALL_DOCKER=true` : Allow `rpi-services` to install Docker through `get.docker.com` on the Raspberry Pi when Docker is missing. Default is to fail. @@ -594,6 +611,10 @@ Raspberry Pi when Docker is missing. Default is to fail. : Allow the edge OpenTofu stack to install Docker on the OCI host when Docker is missing. Default is to fail. +`TF_VAR_heimdall_backend_host`, `TF_VAR_heimdall_backend_port` +: Override the edge backend used for `heimdall.`. Defaults to the +Debian Tailscale IP and `LAB_HEIMDALL_HTTP_PORT`. + `LAB_EDGE_CONFIGURE_TAILSCALE_ROUTES` : Configure the Tailscale subnet-route advertiser and set the OCI edge host to accept routes before applying the edge stack. Defaults to `true`. diff --git a/docs/runbooks/edge-failures.md b/docs/runbooks/edge-failures.md index be094dd..9de58e5 100644 --- a/docs/runbooks/edge-failures.md +++ b/docs/runbooks/edge-failures.md @@ -13,6 +13,7 @@ Current important backends: - Website and Kubernetes apps: Traefik MetalLB IP `192.168.100.240:80` - Gitea: Debian Tailscale IP `100.85.138.30:3000` +- Heimdall: Debian Tailscale IP `100.85.138.30:8082` - OCI edge host: `132.145.170.74` ## Quick Triage @@ -32,6 +33,7 @@ Check backend reachability from the OCI edge host: ssh ubuntu@132.145.170.74 ' curl -I --max-time 5 http://192.168.100.240:80/ || true curl -I --max-time 5 http://100.85.138.30:3000/ || true + curl -I --max-time 5 http://100.85.138.30:8082/ || true tailscale status ' ``` @@ -67,6 +69,12 @@ OCI cannot reach `192.168.100.240:80`: - Check Traefik service and MetalLB IP assignment. - Check Tailscale subnet route acceptance on the OCI edge host. +Only `heimdall.lab2025.duckdns.org` is broken: + +- The dashboard is outside Kubernetes on the Debian host. +- Run `./jeannie deploy-heimdall`. +- Check `sudo docker compose -f /data/homelab-heimdall/docker-compose.yml ps`. + ## Recovery Refresh the edge stack from the repo: @@ -103,9 +111,11 @@ start the cluster: ```bash curl -I https://lab2025.duckdns.org/ curl -I https://lab2025.duckdns.org/git/ +curl -I https://heimdall.lab2025.duckdns.org/ ssh ubuntu@132.145.170.74 ' curl -I --max-time 5 http://192.168.100.240:80/ || true curl -I --max-time 5 http://100.85.138.30:3000/ || true + curl -I --max-time 5 http://100.85.138.30:8082/ || true ' ``` diff --git a/docs/service-catalog.md b/docs/service-catalog.md index 6ac0552..3556ef8 100644 --- a/docs/service-catalog.md +++ b/docs/service-catalog.md @@ -32,7 +32,7 @@ Update `homelab.yml`, then run `scripts/render-service-catalog`. | PXE Rescue Toolkit | debian (control-plane, 192.168.100.73) | OpenTofu / bootstrap/provisioning | [local](http://192.168.100.73:8088/rescue/) | | `bootstrap/provisioning/README.md` | | Ollama | debian (control-plane, 192.168.100.73) | Local Debian service | [local](http://192.168.100.73:11434) | | `apps/website/ollama/README.md` | | OCI Edge | oci_edge | OpenTofu / bootstrap/edge | `/opt/homelab-edge` | [public](https://lab2025.duckdns.org) | `docs/runbooks/edge-failures.md` | -| Heimdall | Kubernetes | Argo CD / apps/heimdall | [local](http://192.168.100.240/) | [public](https://heimdall.lab2025.duckdns.org/) | `apps/heimdall/` | +| Heimdall | debian (control-plane, 192.168.100.73) | Docker Compose / jeannie deploy-heimdall | [local](http://192.168.100.73:8082/) | [public](https://heimdall.lab2025.duckdns.org/) | `infra/heimdall/README.md` | | n8n | Kubernetes | Argo CD / apps/n8n | [local](http://192.168.100.240/) | [public](https://n8n.lab2025.duckdns.org/) | `apps/n8n/` | | Argo CD | Kubernetes | OpenTofu / bootstrap/platform | [local](http://192.168.100.240/) | [public](https://argocd.lab2025.duckdns.org/) | `bootstrap/platform/` | | Grafana | Kubernetes | OpenTofu / bootstrap/platform | [local](http://192.168.100.240/) | [public](https://grafana.lab2025.duckdns.org/) | `bootstrap/platform/` | diff --git a/homelab.yml b/homelab.yml index 6bb5dbc..bc22a18 100644 --- a/homelab.yml +++ b/homelab.yml @@ -72,6 +72,11 @@ services: ssh_port: 32222 root_url: https://lab2025.duckdns.org/git/ ssh_remote: ssh://git@192.168.100.73:32222/jv/my-homelab-configs.git + heimdall: + host: debian + install_dir: /data/homelab-heimdall + http_port: 8082 + public_url: https://heimdall.lab2025.duckdns.org/ gitops_mirror: host: debian path: /home/jv/git-server/my-homelab-configs.git diff --git a/infra/heimdall/README.md b/infra/heimdall/README.md new file mode 100644 index 0000000..49ec0ba --- /dev/null +++ b/infra/heimdall/README.md @@ -0,0 +1,20 @@ +# Heimdall + +Heimdall runs on the Debian host as an always-on Docker Compose service. It is +intentionally outside Kubernetes so the lab navigation dashboard remains +available during cluster rebuilds or outages. + +Deploy or refresh it from the Debian server: + +```bash +./jeannie deploy-heimdall +``` + +The default install root is `/data/homelab-heimdall`. Runtime config is stored +under `/data/homelab-heimdall/data/config`, and the dashboard is published on +port `8082` before the OCI edge proxy exposes it as +`https://heimdall.lab2025.duckdns.org/`. + +`links.json` is the repo-managed dashboard seed. The sidecar container reruns +`seed-heimdall.py` periodically and upserts links into Heimdall's SQLite +database without deleting manually added tiles. diff --git a/infra/heimdall/docker-compose.yml b/infra/heimdall/docker-compose.yml new file mode 100644 index 0000000..30c158b --- /dev/null +++ b/infra/heimdall/docker-compose.yml @@ -0,0 +1,37 @@ +services: + heimdall: + image: ${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347} + container_name: ${HEIMDALL_CONTAINER_NAME:-homelab-heimdall} + restart: unless-stopped + environment: + PUID: ${HEIMDALL_UID:-1000} + PGID: ${HEIMDALL_GID:-1000} + TZ: ${TZ:-America/Mexico_City} + APP_URL: ${HEIMDALL_PUBLIC_URL:-https://heimdall.lab2025.duckdns.org} + ASSET_URL: ${HEIMDALL_PUBLIC_URL:-https://heimdall.lab2025.duckdns.org} + ports: + - "${HEIMDALL_BIND_IP:-0.0.0.0}:${HEIMDALL_HTTP_PORT:-8082}:80/tcp" + volumes: + - ${HEIMDALL_CONFIG_DIR:-/data/homelab-heimdall/data/config}:/config + networks: + - heimdall + + link-seeder: + image: ${HEIMDALL_SEEDER_IMAGE:-python:3.12-alpine} + container_name: ${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder} + restart: unless-stopped + depends_on: + - heimdall + command: + - python + - /seed/seed-heimdall.py + volumes: + - ${HEIMDALL_CONFIG_DIR:-/data/homelab-heimdall/data/config}:/config + - ./links.json:/seed/links.json:ro + - ./seed-heimdall.py:/seed/seed-heimdall.py:ro + networks: + - heimdall + +networks: + heimdall: + name: ${HEIMDALL_NETWORK:-homelab-heimdall} diff --git a/infra/heimdall/links.json b/infra/heimdall/links.json new file mode 100644 index 0000000..2c92cc7 --- /dev/null +++ b/infra/heimdall/links.json @@ -0,0 +1,142 @@ +[ + { + "title": "Website", + "url": "https://lab2025.duckdns.org/", + "description": "Public portfolio website", + "colour": "#0f766e", + "class": "Website" + }, + { + "title": "Demo Apps", + "url": "https://demos.lab2025.duckdns.org/", + "description": "Static browser-side demo catalog", + "colour": "#2563eb", + "class": "Demos" + }, + { + "title": "Service Catalog", + "url": "https://demos.lab2025.duckdns.org/homelab-catalog/", + "description": "Generated host, service, route, and runbook map from homelab.yml", + "colour": "#0891b2", + "class": "Catalog" + }, + { + "title": "Gitea", + "url": "https://lab2025.duckdns.org/git/", + "description": "External Git service on the Debian host", + "colour": "#609926", + "class": "Gitea" + }, + { + "title": "Grafana", + "url": "https://grafana.lab2025.duckdns.org/", + "description": "Monitoring dashboards", + "colour": "#f97316", + "class": "Grafana" + }, + { + "title": "Prometheus", + "url": "https://prometheus.lab2025.duckdns.org/", + "description": "Prometheus query UI", + "colour": "#dc2626", + "class": "Prometheus" + }, + { + "title": "Alertmanager", + "url": "https://alertmanager.lab2025.duckdns.org/", + "description": "Alert routing and silences", + "colour": "#b91c1c", + "class": "Alertmanager" + }, + { + "title": "Argo CD", + "url": "https://argocd.lab2025.duckdns.org/", + "description": "GitOps application sync status", + "colour": "#0ea5e9", + "class": "ArgoCD" + }, + { + "title": "Container Registry", + "url": "http://100.77.80.72:30500/v2/_catalog", + "description": "Local image registry catalog endpoint", + "colour": "#334155", + "class": "Docker" + }, + { + "title": "Heimdall", + "url": "https://heimdall.lab2025.duckdns.org/", + "description": "Homelab service dashboard", + "colour": "#7c3aed", + "class": "Heimdall" + }, + { + "title": "Pi-hole", + "url": "http://192.168.100.89:8081/admin/", + "description": "LAN DNS and ad blocking on the Raspberry Pi", + "colour": "#dc2626", + "class": "PiHole" + }, + { + "title": "Uptime Kuma", + "url": "http://192.168.100.89:3001/", + "description": "Service uptime checks on the Raspberry Pi", + "colour": "#16a34a", + "class": "UptimeKuma" + }, + { + "title": "n8n", + "url": "https://n8n.lab2025.duckdns.org/", + "description": "Workflow automation for translation jobs and homelab tasks", + "colour": "#ea4b71", + "class": "N8N" + }, + { + "title": "Prowlarr", + "url": "https://prowlarr.lab2025.duckdns.org/", + "description": "Indexer management for the media automation stack", + "colour": "#2563eb", + "class": "Prowlarr" + }, + { + "title": "Sonarr", + "url": "https://sonarr.lab2025.duckdns.org/", + "description": "TV library automation", + "colour": "#22c55e", + "class": "Sonarr" + }, + { + "title": "Radarr", + "url": "https://radarr.lab2025.duckdns.org/", + "description": "Movie library automation", + "colour": "#f59e0b", + "class": "Radarr" + }, + { + "title": "qBittorrent", + "url": "https://qbittorrent.lab2025.duckdns.org/", + "description": "Torrent download client", + "colour": "#3b82f6", + "class": "Qbittorrent" + }, + { + "title": "Kapowarr", + "url": "https://kapowarr.lab2025.duckdns.org/", + "description": "Comics library automation", + "colour": "#ef4444", + "class": "Kapowarr" + }, + { + "title": "Suwayomi", + "url": "https://suwayomi.lab2025.duckdns.org/", + "description": "Manga server and scraper engine", + "colour": "#8b5cf6", + "class": "Suwayomi" + }, + { + "title": "Maintainerr", + "url": "https://maintainerr.lab2025.duckdns.org/", + "description": "Media list maintenance automation", + "colour": "#14b8a6", + "class": "Maintainerr" + } +] diff --git a/infra/heimdall/seed-heimdall.py b/infra/heimdall/seed-heimdall.py new file mode 100644 index 0000000..7615b91 --- /dev/null +++ b/infra/heimdall/seed-heimdall.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +import json +import os +import sqlite3 +import time +from datetime import datetime, timezone + + +DB_CANDIDATES = ( + "/config/www/app.sqlite", + "/config/app.sqlite", +) +LINKS_PATH = os.environ.get("HEIMDALL_LINKS_PATH", "/seed/links.json") +RESYNC_SECONDS = int(os.environ.get("HEIMDALL_LINK_RESYNC_SECONDS", "3600")) + + +def find_database(): + while True: + for path in DB_CANDIDATES: + if os.path.exists(path): + return path + time.sleep(5) + + +def table_exists(conn, name): + row = conn.execute( + "select name from sqlite_master where type = 'table' and name = ?", + (name,), + ).fetchone() + return row is not None + + +def columns(conn, table): + return {row[1] for row in conn.execute(f"pragma table_info({table})")} + + +def quote_identifier(name): + return '"' + name.replace('"', '""') + '"' + + +def ensure_public_dashboard(conn, user_id): + if not table_exists(conn, "users"): + return + user_columns = columns(conn, "users") + if "public_front" not in user_columns: + return + conn.execute( + 'update "users" set "public_front" = 1 where "id" = ?', + (user_id,), + ) + + +def ensure_dashboard_tag(conn, item_id, tag_id, now): + if not table_exists(conn, "item_tag"): + return + item_tag_columns = columns(conn, "item_tag") + if not {"item_id", "tag_id"}.issubset(item_tag_columns): + return + existing = conn.execute( + 'select 1 from "item_tag" where "item_id" = ? and "tag_id" = ?', + (item_id, tag_id), + ).fetchone() + if existing: + return + values = { + "item_id": item_id, + "tag_id": tag_id, + "created_at": now, + "updated_at": now, + } + insert_columns = [name for name in values if name in item_tag_columns] + placeholders = ", ".join("?" for _ in insert_columns) + quoted_insert_columns = ", ".join(quote_identifier(name) for name in insert_columns) + conn.execute( + f'insert into "item_tag" ({quoted_insert_columns}) values ({placeholders})', + [values[name] for name in insert_columns], + ) + + +def home_dashboard_tag_id(conn): + item_columns = columns(conn, "items") + if not {"id", "title"}.issubset(item_columns): + return 0 + row = conn.execute( + 'select "id" from "items" where "title" = ? order by "id" limit 1', + ("app.dashboard",), + ).fetchone() + if row: + return row[0] + return 0 + + +def wait_for_items_table(db_path): + while True: + try: + conn = sqlite3.connect(db_path) + if table_exists(conn, "items"): + return conn + conn.close() + except sqlite3.Error: + pass + time.sleep(5) + + +def upsert_links(conn, links): + item_columns = columns(conn, "items") + seed_user_id = 1 + home_dashboard_tag = home_dashboard_tag_id(conn) + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S") + + ensure_public_dashboard(conn, seed_user_id) + + for order, link in enumerate(links): + existing = conn.execute( + 'select "id" from "items" where "title" = ?', + (link["title"],), + ).fetchone() + values = { + "title": link["title"], + "url": link["url"], + "description": None, + "colour": link.get("colour"), + "pinned": 1, + "order": order, + "type": 0, + "user_id": seed_user_id, + "class": None, + "appid": None, + "appdescription": link.get("description"), + "created_at": now, + "updated_at": now, + "deleted_at": None, + } + + if existing: + update_columns = [ + name + for name in values + if name in item_columns and name not in ("id", "title", "created_at") + ] + assignments = ", ".join(f"{quote_identifier(name)} = ?" for name in update_columns) + params = [values[name] for name in update_columns] + params.append(existing[0]) + conn.execute(f'update "items" set {assignments} where "id" = ?', params) + ensure_dashboard_tag(conn, existing[0], home_dashboard_tag, now) + continue + + insert_columns = [name for name in values if name in item_columns and name != "deleted_at"] + placeholders = ", ".join("?" for _ in insert_columns) + quoted_insert_columns = ", ".join(quote_identifier(name) for name in insert_columns) + conn.execute( + f'insert into "items" ({quoted_insert_columns}) values ({placeholders})', + [values[name] for name in insert_columns], + ) + new_item_id = conn.execute("select last_insert_rowid()").fetchone()[0] + ensure_dashboard_tag(conn, new_item_id, home_dashboard_tag, now) + + conn.commit() + + +def load_links(): + with open(LINKS_PATH, encoding="utf-8") as links_file: + return json.load(links_file) + + +while True: + db_path = find_database() + connection = wait_for_items_table(db_path) + try: + upsert_links(connection, load_links()) + time.sleep(RESYNC_SECONDS) + except (OSError, sqlite3.Error, json.JSONDecodeError) as error: + print(f"Failed to seed Heimdall links: {error}", flush=True) + time.sleep(10) + finally: + connection.close() diff --git a/infra/resource-recommendations.yml b/infra/resource-recommendations.yml index a24fc8e..5078ef1 100644 --- a/infra/resource-recommendations.yml +++ b/infra/resource-recommendations.yml @@ -18,11 +18,6 @@ namespaces: request_memory_mib: 64 limit_cpu_multiplier: 4 limit_memory_multiplier: 2 - heimdall: - request_cpu_m: 50 - request_memory_mib: 128 - limit_cpu_multiplier: 4 - limit_memory_multiplier: 2 argocd: request_cpu_m: 100 request_memory_mib: 256 diff --git a/jeannie b/jeannie index cddcd0e..fc4619f 100755 --- a/jeannie +++ b/jeannie @@ -66,6 +66,9 @@ mapping = { "services.gitea.ssh_port": "LAB_GITEA_SSH_PORT", "services.gitea.root_url": "LAB_GITEA_ROOT_URL", "services.gitea.ssh_remote": "LAB_GITOPS_REPO_URL", + "services.heimdall.install_dir": "LAB_HEIMDALL_INSTALL_DIR", + "services.heimdall.http_port": "LAB_HEIMDALL_HTTP_PORT", + "services.heimdall.public_url": "LAB_HEIMDALL_PUBLIC_URL", "services.local_registry.endpoint": "LAB_REGISTRY_ENDPOINT", "services.ollama.bind_address": "LAB_OLLAMA_BIND_ADDRESS", "services.ollama.models_dir": "LAB_OLLAMA_MODELS_DIR", @@ -199,6 +202,8 @@ export_homelab_inventory_tf_vars() { export_if_unset TF_VAR_backend_host "${LAB_TRAEFIK_LB_IP:-}" export_if_unset TF_VAR_gitea_backend_host "${LAB_DEBIAN_TAILSCALE_IP:-}" export_if_unset TF_VAR_gitea_backend_port "${LAB_GITEA_HTTP_PORT:-}" + export_if_unset TF_VAR_heimdall_backend_host "${LAB_HEIMDALL_BACKEND_HOST:-${LAB_DEBIAN_TAILSCALE_IP:-}}" + export_if_unset TF_VAR_heimdall_backend_port "${LAB_HEIMDALL_HTTP_PORT:-}" export_if_unset TF_VAR_gitops_repo_url "${LAB_GITOPS_REPO_URL:-}" export_if_unset TF_VAR_worker_tailscale_enabled "${LAB_PIMOX_WORKER_TAILSCALE_ENABLED:-}" export_if_unset TF_VAR_worker_tailscale_accept_routes "${LAB_PIMOX_WORKER_TAILSCALE_ACCEPT_ROUTES:-}" @@ -3731,6 +3736,142 @@ sudo docker compose ps install_gitea_backup_timer } +install_missing_debian_packages() { + local missing_packages=() + local package + + for package in "$@"; do + if ! dpkg-query -W -f='${Status}' "${package}" 2>/dev/null | grep -q "install ok installed"; then + missing_packages+=("${package}") + fi + done + + if ((${#missing_packages[@]})); then + sudo apt-get update + sudo apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi +} + +ensure_local_docker_compose() { + local command_name="$1" + local install_docker="$2" + + install_missing_debian_packages ca-certificates curl iptables + + if ! command -v docker >/dev/null 2>&1; then + if [[ "${install_docker}" == "true" ]]; then + curl -fsSL https://get.docker.com | sudo sh + else + echo "Docker is not installed on the Debian host. Install Docker through host bootstrap first, or rerun with LAB_${command_name}_INSTALL_DOCKER=true to allow get.docker.com installation." >&2 + exit 1 + fi + fi + + if ! sudo docker compose version >/dev/null 2>&1; then + install_missing_debian_packages docker-compose-plugin + fi +} + +repair_local_docker_iptables() { + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo "Docker NAT chain is missing on the Debian host; restarting Docker once to restore iptables state..." + sudo systemctl restart docker + sleep 3 + + if sudo iptables -t nat -S DOCKER >/dev/null 2>&1; then + return 0 + fi + + echo "Docker NAT chain is still missing after restarting Docker." >&2 + sudo iptables -t nat -S >&2 || true + sudo systemctl status docker --no-pager -l >&2 || true + exit 1 +} + +deploy_heimdall() { + local mode="${LAB_HEIMDALL_DEPLOY:-true}" + local install_dir="${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" + local data_dir="${LAB_HEIMDALL_DATA_DIR:-${install_dir}/data}" + local config_dir="${LAB_HEIMDALL_CONFIG_DIR:-${data_dir}/config}" + local http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" + local bind_ip="${LAB_HEIMDALL_BIND_IP:-0.0.0.0}" + local public_url="${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:-lab2025.duckdns.org}/}" + local image="${HEIMDALL_IMAGE:-lscr.io/linuxserver/heimdall:v2.7.6-ls347}" + local container_name="${HEIMDALL_CONTAINER_NAME:-homelab-heimdall}" + local seeder_container_name="${HEIMDALL_SEEDER_CONTAINER_NAME:-homelab-heimdall-link-seeder}" + local network="${HEIMDALL_NETWORK:-homelab-heimdall}" + local install_docker="${LAB_HEIMDALL_INSTALL_DOCKER:-false}" + local source_dir="${REPO_ROOT}/infra/heimdall" + local elapsed=0 + local value_name + local value + + require_debian_server "deploy-heimdall" + + if disabled_value "${mode}"; then + return 0 + fi + + for value_name in install_dir data_dir config_dir http_port bind_ip public_url image container_name seeder_container_name network install_docker; do + value="${!value_name}" + if [[ "${value}" == *$'\n'* ]]; then + echo "${value_name} cannot contain a newline." >&2 + exit 1 + fi + done + + if [[ ! -s "${source_dir}/docker-compose.yml" || ! -s "${source_dir}/links.json" || ! -s "${source_dir}/seed-heimdall.py" ]]; then + echo "Missing Heimdall source files in ${source_dir}" >&2 + exit 1 + fi + + echo "Deploying external Heimdall on the Debian host at ${install_dir}..." + + ensure_local_docker_compose "HEIMDALL" "${install_docker}" + repair_local_docker_iptables + + sudo mkdir -p "${install_dir}" "${config_dir}" + sudo cp "${source_dir}/docker-compose.yml" "${install_dir}/docker-compose.yml" + sudo cp "${source_dir}/links.json" "${install_dir}/links.json" + sudo cp "${source_dir}/seed-heimdall.py" "${install_dir}/seed-heimdall.py" + sudo chown -R 1000:1000 "${data_dir}" + sudo tee "${install_dir}/.env" >/dev/null </dev/null; do + if ((elapsed >= 180)); then + echo "Timed out waiting for Heimdall on http://127.0.0.1:${http_port}/" >&2 + (cd "${install_dir}" && sudo docker compose ps && sudo docker compose logs --tail=120) >&2 || true + exit 1 + fi + sleep 5 + elapsed=$((elapsed + 5)) + done + + echo "Heimdall is reachable at http://127.0.0.1:${http_port}/ and ${public_url%/}/." +} + deploy_rpi_services() { local mode="${LAB_RPI_SERVICES_DEPLOY:-true}" local rpi_host="${LAB_RPI_HOST:-${LAB_RASPBERRY_HOST:-192.168.100.89}}" @@ -4905,9 +5046,10 @@ apps() { fi refresh_argocd_application heimdall - wait_for_namespace heimdall heimdall 300 - wait_for_namespaced_resource heimdall deployment heimdall heimdall 300 - wait_for_deployment_ready heimdall heimdall heimdall 300 + wait_for_namespaced_resource monitoring ingress grafana heimdall 300 + wait_for_namespaced_resource monitoring ingress prometheus heimdall 300 + wait_for_namespaced_resource monitoring ingress alertmanager heimdall 300 + wait_for_namespaced_resource argocd ingress argocd-server heimdall 300 refresh_argocd_application n8n wait_for_namespace n8n n8n 300 @@ -5144,6 +5286,11 @@ spec = { "needs": ["gitea-deploy"], "command": [jeannie, "__up-task", "gitea-bootstrap"], }, + { + "id": "heimdall-deploy", + "title": "Heimdall deploy", + "command": [jeannie, "__up-task", "heimdall-deploy"], + }, { "id": "rpi-services", "title": "RPi services", @@ -5188,6 +5335,9 @@ up_task() { gitea-bootstrap) bootstrap_gitea_repo ;; + heimdall-deploy) + deploy_heimdall + ;; rpi-services) deploy_rpi_services ;; @@ -6550,7 +6700,9 @@ status_report() { if [[ -n "${heal_mode}" ]]; then status_json="$(mktemp "${TMPDIR:-/tmp}/jeannie-status-json.XXXXXX")" + # shellcheck disable=SC2034 REPORT_UI_JSON=true + # shellcheck disable=SC2034 REPORT_UI_ONLY=problems report_ui_begin JEANNIE_REPORT_MODE=true status_cascade_report >/dev/null || cascade_status=$? @@ -6596,6 +6748,7 @@ status_report() { printf 'docker not installed\n' fi status_compose_stack "gitea" "/data/homelab-gitea" + status_compose_stack "heimdall" "${LAB_HEIMDALL_INSTALL_DIR:-/data/homelab-heimdall}" status_compose_stack "arr-stack" "${REPO_ROOT}/infra/arr-stack" status_kubernetes @@ -6613,7 +6766,9 @@ status_report() { status_section "HTTP" status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" status_http "gitea local" "http://127.0.0.1:${LAB_GITEA_HTTP_PORT:-3000}/" + status_http "heimdall local" "http://127.0.0.1:${LAB_HEIMDALL_HTTP_PORT:-8082}/" status_http "traefik lb" "http://${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}/" status_http "arr radarr" "http://127.0.0.1:7878/" status_http "arr sonarr" "http://127.0.0.1:8989/" @@ -6641,6 +6796,16 @@ check_edge_gitea_backend() { "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${gitea_ts_ip}:${gitea_http_port}/' | grep -Eq '^[234]'" } +check_edge_heimdall_backend() { + local edge_host="${LAB_EDGE_HOST:?LAB_EDGE_HOST is required from homelab.yml}" + local edge_user="${LAB_EDGE_USER:-ubuntu}" + local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}}" + local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" + + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" \ + "curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}' 'http://${heimdall_host}:${heimdall_http_port}/' | grep -Eq '^[234]'" +} + check_gitea_git_remote() { git ls-remote gitea main >/dev/null } @@ -6659,6 +6824,8 @@ doctor_edge() { local traefik_ip="${LAB_TRAEFIK_LB_IP:?LAB_TRAEFIK_LB_IP is required from homelab.yml}" local gitea_ts_ip="${LAB_GITEA_TAILSCALE_IP:?LAB_GITEA_TAILSCALE_IP is required from homelab.yml}" local gitea_http_port="${LAB_GITEA_HTTP_PORT:-3000}" + local heimdall_host="${LAB_HEIMDALL_BACKEND_HOST:-${gitea_ts_ip}}" + local heimdall_http_port="${LAB_HEIMDALL_HTTP_PORT:-8082}" local failures=0 require_debian_server "doctor-edge" @@ -6673,6 +6840,7 @@ doctor_edge() { report_ui_check fail "Edge" "OCI edge SSH" "./jeannie preflight" "" "" check_edge_ssh || failures=$((failures + 1)) report_ui_check fail "Backends" "Traefik from edge" "./jeannie doctor-cluster" "" "" check_edge_traefik_backend || failures=$((failures + 1)) report_ui_check fail "Backends" "Gitea from edge" "./jeannie doctor-gitea" "" "" check_edge_gitea_backend || failures=$((failures + 1)) + report_ui_check fail "Backends" "Heimdall from edge" "./jeannie deploy-heimdall" "" "" check_edge_heimdall_backend || failures=$((failures + 1)) report_ui_render "Doctor Edge" report_ui_cleanup return "${failures}" @@ -6681,6 +6849,7 @@ doctor_edge() { status_section "Doctor Edge" status_http "website public" "${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/" status_http "gitea public" "${LAB_GITEA_ROOT_URL:-${LAB_PUBLIC_URL:?LAB_PUBLIC_URL is required from homelab.yml}/git/}" + status_http "heimdall public" "${LAB_HEIMDALL_PUBLIC_URL:-https://heimdall.${LAB_DOMAIN:?LAB_DOMAIN is required from homelab.yml}/}" printf '\n-- edge backend reachability\n' if ! ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${edge_user}@${edge_host}" " @@ -6689,6 +6858,8 @@ printf '%-28s ' 'traefik backend' curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${traefik_ip}:80/' || echo unreachable printf '%-28s ' 'gitea backend' curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${gitea_ts_ip}:${gitea_http_port}/' || echo unreachable +printf '%-28s ' 'heimdall backend' +curl -sS -o /dev/null --connect-timeout 5 --max-time 10 -w '%{http_code}\n' 'http://${heimdall_host}:${heimdall_http_port}/' || echo unreachable printf '\n-- tailscale\n' tailscale status || true "; then @@ -6700,6 +6871,7 @@ tailscale status || true Next steps: - If only Traefik is unreachable, check cluster/MetalLB or run ./jeannie doctor-cluster. - If only Gitea is unreachable, run ./jeannie doctor-gitea. +- If only Heimdall is unreachable, run ./jeannie deploy-heimdall. - If both are unreachable from OCI, check Tailscale routes and bootstrap/edge. - Runbook: docs/runbooks/edge-failures.md EOF @@ -7576,6 +7748,7 @@ Base, Inventory, And Planning Build And Bootstrap up Deploy the full homelab pipeline. deploy-gitea Deploy the Debian-hosted Gitea container. + deploy-heimdall Deploy the Debian-hosted Heimdall dashboard. bootstrap-gitea-repo Ensure the Gitea repo and SSH key wiring exist. rpi-services Deploy Pi-hole, Unbound, and Uptime Kuma on RPi4. ollama-setup Install/configure Ollama on the Debian host. @@ -7850,6 +8023,10 @@ case "${1:-}" in record_change_journal "deploy-gitea" "$@" deploy_gitea ;; + deploy-heimdall) + record_change_journal "deploy-heimdall" "$@" + deploy_heimdall + ;; rpi-services) record_change_journal "rpi-services" "$@" deploy_rpi_services diff --git a/scripts/capacity-advisor b/scripts/capacity-advisor index 2d21869..369dcfb 100755 --- a/scripts/capacity-advisor +++ b/scripts/capacity-advisor @@ -47,7 +47,7 @@ for line in control_plane_section.splitlines(): namespace, pod_name, status = match.groups() if namespace == "arr-stack": legacy_control_plane_pods.append((namespace, pod_name, status)) - elif namespace in {"argocd", "website-production", "demos-static", "heimdall", "n8n", "security-lab", "monitoring"}: + elif namespace in {"argocd", "website-production", "demos-static", "n8n", "security-lab", "monitoring"}: app_control_plane_pods.append((namespace, pod_name, status)) print("Jeannie Capacity Advisor") diff --git a/scripts/render-service-catalog b/scripts/render-service-catalog index 95db758..cc292df 100755 --- a/scripts/render-service-catalog +++ b/scripts/render-service-catalog @@ -92,6 +92,7 @@ def service_rows(values: dict[str, object]) -> list[dict[str, str]]: rpi_ip = value(values, "hosts.rpi4.lan_ip") traefik_ip = value(values, "network.metallb.traefik_ip") registry = value(values, "services.local_registry.endpoint") + heimdall_host = value(values, "services.heimdall.host", "debian") rows = [ { @@ -208,11 +209,11 @@ def service_rows(values: dict[str, object]) -> list[dict[str, str]]: }, { "name": "Heimdall", - "host": "kubernetes", - "managed_by": "Argo CD / apps/heimdall", - "local": f"http://{traefik_ip}/", + "host": heimdall_host, + "managed_by": "Docker Compose / jeannie deploy-heimdall", + "local": f"http://{host_ip(values, heimdall_host)}:{value(values, 'services.heimdall.http_port', '8082')}/", "public": f"https://heimdall.{domain}/", - "docs": "apps/heimdall/", + "docs": "infra/heimdall/README.md", }, { "name": "n8n", diff --git a/scripts/validate-homelab-inventory b/scripts/validate-homelab-inventory index a8322fd..f18c267 100755 --- a/scripts/validate-homelab-inventory +++ b/scripts/validate-homelab-inventory @@ -69,11 +69,13 @@ else: "hosts.opi5_pimox.worker_storage", "hosts.oci_edge.public_ip", "services.gitea.root_url", + "services.heimdall.public_url", "services.local_registry.endpoint", "services.traefik.load_balancer_ip", ): require(path) require("pimox.default_worker_count", None) + require("services.heimdall.http_port", None) subdomains = require("domain.subdomains", list) if isinstance(subdomains, list): @@ -161,6 +163,8 @@ required_paths = %w[ hosts.opi5_pimox.worker_storage hosts.oci_edge.public_ip services.gitea.root_url + services.heimdall.http_port + services.heimdall.public_url services.local_registry.endpoint services.traefik.load_balancer_ip pimox.default_worker_count