From 8c96d4ce10765d4787c5343a241b01d7ef346678 Mon Sep 17 00:00:00 2001 From: juvdiaz Date: Sat, 23 May 2026 21:00:28 -0600 Subject: [PATCH] redesign --- .gitignore | 3 +- README.md | 70 ++++ apps/container-registry/namespace.yaml | 4 + .../registry-deployment.yaml | 28 ++ apps/container-registry/registry-storage.yaml | 23 +- apps/gitea/deployment.yaml | 74 +++++ apps/gitea/docker_gitea_backup.tf | 33 -- apps/gitea/host_prep.tf | 18 -- apps/gitea/k8s_gitea.tf | 134 -------- apps/gitea/namespace.yaml | 4 + apps/gitea/providers.tf | 31 -- apps/gitea/service.yaml | 18 ++ apps/gitea/storage.yaml | 36 +++ apps/gitea/variables.tf | 26 -- apps/website/buildkitd.toml | 12 +- apps/website/web-app.yaml | 21 +- bootstrap/apps/.terraform.lock.hcl | 27 ++ bootstrap/apps/main.tf | 60 +--- bootstrap/apps/variables.tf | 56 ++++ bootstrap/cluster/.terraform.lock.hcl | 76 +++++ bootstrap/cluster/main.tf | 158 ++++++--- bootstrap/cluster/variables.tf | 55 ++++ bootstrap/platform/.terraform.lock.hcl | 91 ++++++ bootstrap/platform/main.tf | 281 ++++++++++++---- bootstrap/platform/variables.tf | 82 +++++ lab.sh | 306 +++++++++++++----- 26 files changed, 1241 insertions(+), 486 deletions(-) create mode 100644 README.md create mode 100644 apps/container-registry/namespace.yaml create mode 100644 apps/gitea/deployment.yaml delete mode 100644 apps/gitea/docker_gitea_backup.tf delete mode 100644 apps/gitea/host_prep.tf delete mode 100644 apps/gitea/k8s_gitea.tf create mode 100644 apps/gitea/namespace.yaml delete mode 100644 apps/gitea/providers.tf create mode 100644 apps/gitea/service.yaml create mode 100644 apps/gitea/storage.yaml delete mode 100644 apps/gitea/variables.tf create mode 100644 bootstrap/apps/.terraform.lock.hcl create mode 100644 bootstrap/apps/variables.tf create mode 100644 bootstrap/cluster/.terraform.lock.hcl create mode 100644 bootstrap/cluster/variables.tf create mode 100644 bootstrap/platform/.terraform.lock.hcl create mode 100644 bootstrap/platform/variables.tf diff --git a/.gitignore b/.gitignore index 0196cce..8b1abc6 100644 --- a/.gitignore +++ b/.gitignore @@ -2,12 +2,11 @@ *.tfstate *.tfstate.backup .terraform/ -.terraform.lock.hcl # Ignore local archive dumps and backups *.tar *.zip -gittea/gittea-docker-backup +apps/gitea/gitea-docker-backup # Ignore older source iterations *.old diff --git a/README.md b/README.md new file mode 100644 index 0000000..526e481 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# Homelab Kubernetes Pipeline + +This repo bootstraps a hybrid kubeadm cluster and then hands app delivery to +Argo CD. + +## Flow + +1. `bootstrap/cluster` + - creates the kubeadm control plane on the Debian amd64 node + - joins worker nodes such as Raspberry Pi arm64 nodes + - configures Calico-compatible pod CIDR + - configures containerd to pull from the in-cluster NodePort registry + - creates retained host directories under `/var/openebs/local` + +2. `bootstrap/platform` + - installs a minimal Calico deployment through the Tigera operator + - installs OpenEBS + - creates `openebs-hostpath-retain` + - installs Argo CD + - registers the private GitOps repo without storing the SSH private key in + Terraform state + +3. `bootstrap/apps` + - registers Argo CD Applications from the `applications` map + - default apps are `container-registry`, `gitea`, and + `website-production` + +## Adding Nodes + +Add entries to `bootstrap/cluster/variables.tf` or a `.tfvars` file: + +```hcl +worker_nodes = { + raspberrypi = { + host = "192.168.100.89" + user = "jv" + node_name = "raspberry" + ssh_key_path = "/home/jv/.ssh/id_ed25519" + } +} +``` + +Stateful apps currently pin retained local PVs to the `debian` node. Move or +duplicate those PV manifests when you want storage on another node. + +For `./lab.sh nuke`, set `WORKER_SSH_TARGETS` to a space-separated list of +remote SSH targets when more worker nodes exist. Set it to an empty string for a +single-node rebuild. + +## Adding Platform Tools + +Add Helm releases through `bootstrap/platform`'s `extra_helm_releases` map. + +## Adding Apps + +Add Kubernetes manifests under `apps/` and register them in +`bootstrap/apps`'s `applications` map. Argo CD will own sync, pruning, and +self-healing for the app. + +## Storage + +OpenEBS provides the platform storage provisioner. Stateful homelab apps use +retained local PV paths such as `/var/openebs/local/gitea` and +`/var/openebs/local/registry`; these paths are intentionally outside kubeadm +reset paths so data can survive cluster destroy/create cycles. Those critical +volumes are declared explicitly as retained local PVs so a rebuilt cluster binds +back to the same host paths instead of creating fresh directories. + +Keep the `.terraform.lock.hcl` files committed. They pin provider selections and +make bootstrap behavior reproducible across nodes and rebuilds. diff --git a/apps/container-registry/namespace.yaml b/apps/container-registry/namespace.yaml new file mode 100644 index 0000000..83531ae --- /dev/null +++ b/apps/container-registry/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: container-registry diff --git a/apps/container-registry/registry-deployment.yaml b/apps/container-registry/registry-deployment.yaml index 4bb0e3f..b36ff59 100644 --- a/apps/container-registry/registry-deployment.yaml +++ b/apps/container-registry/registry-deployment.yaml @@ -15,11 +15,39 @@ spec: labels: app: local-registry spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - debian containers: - name: registry image: registry:2 ports: - containerPort: 5000 + name: http + readinessProbe: + httpGet: + path: /v2/ + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /v2/ + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi volumeMounts: - name: registry-vol mountPath: /var/lib/registry diff --git a/apps/container-registry/registry-storage.yaml b/apps/container-registry/registry-storage.yaml index c3eee61..f6e222a 100644 --- a/apps/container-registry/registry-storage.yaml +++ b/apps/container-registry/registry-storage.yaml @@ -1,17 +1,25 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: registry-pv - labels: - type: local + name: registry-data-debian spec: - storageClassName: manual capacity: storage: 20Gi + volumeMode: Filesystem accessModes: - ReadWriteOnce - hostPath: - path: "/home/k8s-storage/registry-data" + persistentVolumeReclaimPolicy: Retain + storageClassName: openebs-hostpath-retain + local: + path: /var/openebs/local/registry + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - debian --- apiVersion: v1 kind: PersistentVolumeClaim @@ -19,7 +27,8 @@ metadata: name: registry-pvc namespace: container-registry spec: - storageClassName: manual + storageClassName: openebs-hostpath-retain + volumeName: registry-data-debian accessModes: - ReadWriteOnce resources: diff --git a/apps/gitea/deployment.yaml b/apps/gitea/deployment.yaml new file mode 100644 index 0000000..871c364 --- /dev/null +++ b/apps/gitea/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitea + namespace: gitea-system + labels: + app: gitea +spec: + replicas: 1 + selector: + matchLabels: + app: gitea + template: + metadata: + labels: + app: gitea + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - debian + containers: + - name: gitea + image: gitea/gitea:1.21.7 + ports: + - containerPort: 3000 + name: http + - containerPort: 22 + name: ssh + env: + - name: USER_UID + value: "1000" + - name: USER_GID + value: "1000" + - name: GITEA__database__DB_TYPE + value: sqlite3 + - name: GITEA__repository__ENABLE_PUSH_MIRROR + value: "true" + - name: GITEA__migrations__ALLOW_LOCALNETWORKS + value: "true" + - name: GITEA__server__SSH_PORT + value: "32222" + - name: GITEA__server__SSH_LISTEN_PORT + value: "22" + volumeMounts: + - name: gitea-data + mountPath: /data + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 20 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 60 + periodSeconds: 30 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 1Gi + volumes: + - name: gitea-data + persistentVolumeClaim: + claimName: gitea-data diff --git a/apps/gitea/docker_gitea_backup.tf b/apps/gitea/docker_gitea_backup.tf deleted file mode 100644 index 92e16c7..0000000 --- a/apps/gitea/docker_gitea_backup.tf +++ /dev/null @@ -1,33 +0,0 @@ -# Pull down the Gitea image locally via the Pi's Docker engine -resource "docker_image" "gitea_backup" { - name = "gitea/gitea:1.21.7" - keep_locally = true -} - -# Fire up the standalone container wrapper -resource "docker_container" "gitea_backup" { - name = "gitea-backup" - image = docker_image.gitea_backup.image_id - restart = "always" - - env = [ - "USER_UID=1000", - "USER_GID=1000", - "GITEA__database__DB_TYPE=sqlite3" - ] - - ports { - internal = 3000 - external = 3000 - } - - ports { - internal = 22 - external = 2222 - } - - volumes { - host_path = "/home/pi/gitea-backup-data" - container_path = "/data" - } -} diff --git a/apps/gitea/host_prep.tf b/apps/gitea/host_prep.tf deleted file mode 100644 index 9c328d3..0000000 --- a/apps/gitea/host_prep.tf +++ /dev/null @@ -1,18 +0,0 @@ -resource "null_resource" "node_prep" { - for_each = var.nodes - - connection { - type = "ssh" - user = each.value.user - host = each.value.ip - private_key = file(var.ssh_key_path) - } - - provisioner "remote-exec" { - inline = [ - "sudo apt-get update", - "sudo apt-get install -y open-iscsi util-linux", - "sudo systemctl enable --now iscsid" - ] - } -} diff --git a/apps/gitea/k8s_gitea.tf b/apps/gitea/k8s_gitea.tf deleted file mode 100644 index 85df5e2..0000000 --- a/apps/gitea/k8s_gitea.tf +++ /dev/null @@ -1,134 +0,0 @@ -resource "kubernetes_namespace" "gitea" { - metadata { - name = "gitea-system" - } -} - -resource "null_resource" "install_longhorn" { - depends_on = [null_resource.node_prep] - - provisioner "local-exec" { - command = "kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.2/deploy/longhorn.yaml" - } -} - -resource "kubernetes_persistent_volume_claim" "gitea_pvc" { - depends_on = [null_resource.install_longhorn] - - metadata { - name = "gitea-pvc" - namespace = kubernetes_namespace.gitea.metadata[0].name - } - spec { - access_modes = ["ReadWriteOnce"] - storage_class_name = "longhorn" - resources { - requests = { - storage = "10Gi" - } - } - } -} - -resource "kubernetes_deployment" "gitea_prod" { - metadata { - name = "gitea-prod" - namespace = kubernetes_namespace.gitea.metadata[0].name - } - spec { - replicas = 1 - selector { - match_labels = { - app = "gitea" - } - } - template { - metadata { - labels = { - app = "gitea" - } - } - spec { - container { - name = "gitea" - image = "gitea/gitea:1.21.7" - - port { - container_port = 3000 - name = "http" - } - port { - container_port = 22 - name = "ssh" - } - - env { - name = "USER_UID" - value = "1000" - } - env { - name = "USER_GID" - value = "1000" - } - env { - name = "GITEA__database__DB_TYPE" - value = "sqlite3" - } - - env { - name = "GITEA__repository__ENABLE_PUSH_MIRROR" - value = "true" - } - - env { - name = "GITEA__repository__ENABLE_PUSH_MIRROR" - value = "true" - } - - # ADD THIS NEW BLOCK: - env { - name = "GITEA__migrations__ALLOW_LOCALNETWORKS" - value = "true" - } - - volume_mount { - mount_path = "/data" - name = "gitea-storage" - } - } - - volume { - name = "gitea-storage" - persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim.gitea_pvc.metadata[0].name - } - } - } - } - } -} - -resource "kubernetes_service" "gitea_service" { - metadata { - name = "gitea-service" - namespace = kubernetes_namespace.gitea.metadata[0].name - } - spec { - type = "NodePort" - selector = { - app = "gitea" - } - port { - name = "http" - port = 3000 - target_port = 3000 - node_port = 30300 - } - port { - name = "ssh" - port = 22 - target_port = 22 - node_port = 32222 - } - } -} diff --git a/apps/gitea/namespace.yaml b/apps/gitea/namespace.yaml new file mode 100644 index 0000000..a2b56dc --- /dev/null +++ b/apps/gitea/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea-system diff --git a/apps/gitea/providers.tf b/apps/gitea/providers.tf deleted file mode 100644 index 1cf17c0..0000000 --- a/apps/gitea/providers.tf +++ /dev/null @@ -1,31 +0,0 @@ -terraform { - required_providers { - null = { - source = "hashicorp/null" - version = "~> 3.2" - } - docker = { - source = "kreuzwerker/docker" - version = "~> 3.0" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.23" - } - } -} - -# Core Cluster API access -provider "kubernetes" { - config_path = "~/.kube/config" - config_context = "kubernetes-admin@kubernetes" -} - -# Remote Docker control over the Raspberry Pi -provider "docker" { - host = "ssh://jv@192.168.100.89:22" - ssh_opts = [ - "-o", "StrictHostKeyChecking=no", - "-i", var.ssh_key_path - ] -} diff --git a/apps/gitea/service.yaml b/apps/gitea/service.yaml new file mode 100644 index 0000000..069f5e3 --- /dev/null +++ b/apps/gitea/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: gitea + namespace: gitea-system +spec: + type: NodePort + selector: + app: gitea + ports: + - name: http + port: 3000 + targetPort: http + nodePort: 30300 + - name: ssh + port: 22 + targetPort: ssh + nodePort: 32222 diff --git a/apps/gitea/storage.yaml b/apps/gitea/storage.yaml new file mode 100644 index 0000000..8a9fdf9 --- /dev/null +++ b/apps/gitea/storage.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitea-data-debian +spec: + capacity: + storage: 20Gi + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Retain + storageClassName: openebs-hostpath-retain + local: + path: /var/openebs/local/gitea + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - debian +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gitea-data + namespace: gitea-system +spec: + accessModes: + - ReadWriteOnce + storageClassName: openebs-hostpath-retain + volumeName: gitea-data-debian + resources: + requests: + storage: 20Gi diff --git a/apps/gitea/variables.tf b/apps/gitea/variables.tf deleted file mode 100644 index f1339ec..0000000 --- a/apps/gitea/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "home_dir" { - type = string - default = "/home/jv" -} - -variable "ssh_key_path" { - type = string - default = "/home/jv/.ssh/id_ed25519" -} - -variable "nodes" { - type = map(object({ - ip = string - user = string - })) - default = { - raspberrypi = { - ip = "192.168.100.89" - user = "jv" - } - debian_laptop = { - ip = "192.168.100.68" - user = "jv" - } - } -} diff --git a/apps/website/buildkitd.toml b/apps/website/buildkitd.toml index 4328ace..8b92fa1 100644 --- a/apps/website/buildkitd.toml +++ b/apps/website/buildkitd.toml @@ -1,7 +1,11 @@ -[registry."host.docker.internal:30500"] - http = true - insecure = true - [registry."192.168.100.68:30500"] http = true insecure = true + +[registry."127.0.0.1:30500"] + http = true + insecure = true + +[registry."localhost:30500"] + http = true + insecure = true diff --git a/apps/website/web-app.yaml b/apps/website/web-app.yaml index 51260c1..12b7065 100644 --- a/apps/website/web-app.yaml +++ b/apps/website/web-app.yaml @@ -34,10 +34,29 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: php-app - image: local-registry-svc.container-registry.svc.cluster.local:5000/php-website:latest + image: 192.168.100.68:30500/php-website:latest imagePullPolicy: Always ports: - containerPort: 80 + name: http + readinessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: / + port: http + initialDelaySeconds: 30 + periodSeconds: 30 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 256Mi --- apiVersion: v1 kind: Service diff --git a/bootstrap/apps/.terraform.lock.hcl b/bootstrap/apps/.terraform.lock.hcl new file mode 100644 index 0000000..561e68c --- /dev/null +++ b/bootstrap/apps/.terraform.lock.hcl @@ -0,0 +1,27 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.26" + hashes = [ + "h1:3VVgWmwdwXFT54fjrplnq+N4+4LZ3ZeLHAlr/0jhPiA=", + "h1:9LfHMXiMOboc6PhcEEelKjA3VL94l3MCj7RlbKO1PQM=", + "h1:AkW6iAcMGHS+P2BIc2nvQe3PZCHtDL4m6+80tEDLge0=", + "h1:HGkB9bCmUqMRcR5/bAUOSqPBsx6DAIEnbT1fZ8vzI78=", + "h1:eCV78xGlh9eay+62U4gAgCEMohuiBJXN9XTIZNn+rX4=", + "h1:ems+O2dA7atxMWpbtqIrsH7Oa+u+ERWSfpMaFnZPbh0=", + "h1:iEDf790HE0h3kz58zfj5IhTVODCDqWF1hISPHz210Uw=", + "h1:nY7J9jFXcsRINog0KYagiWZw1GVYF9D2JmtIB7Wnrao=", + "h1:yw6JHRONXmiTumaQfJBxl1ierFnNNT/Qio8+ttfMcG4=", + "zh:1096b41c4e5b2ee6c1980916fb9a8579bc1892071396f7a9432be058aabf3cbc", + "zh:2959fde9ae3d1deb5e317df0d7b02ea4977951ee6b9c4beb083c148ca8f3681c", + "zh:5082f98fcb3389c73339365f7df39fc6912bf2bd1a46d5f97778f441a67fd337", + "zh:620fd5d0fbc2d7a24ac6b420a4922e6093020358162a62fa8cbd37b2bac1d22e", + "zh:7f47c2de179bba35d759147c53082cad6c3449d19b0ec0c5a4ca8db5b06393e1", + "zh:89c3aa2a87e29febf100fd21cead34f9a4c0e6e7ae5f383b5cef815c677eb52a", + "zh:96eecc9f94938a0bc35b8a63d2c4a5f972395e44206620db06760b730d0471fc", + "zh:e15567c1095f898af173c281b66bffdc4f3068afdd9f84bb5b5b5521d9f29584", + "zh:ecc6b912629734a9a41a7cf1c4c73fb13b4b510afc9e7b2e0011d290bcd6d77f", + ] +} diff --git a/bootstrap/apps/main.tf b/bootstrap/apps/main.tf index cd63f69..d20764a 100644 --- a/bootstrap/apps/main.tf +++ b/bootstrap/apps/main.tf @@ -9,45 +9,11 @@ terraform { } provider "kubernetes" { - config_path = "/home/jv/.kube/config" + config_path = var.kubeconfig_path } -resource "kubernetes_manifest" "container_registry" { - field_manager { - force_conflicts = true - } - - manifest = { - apiVersion = "argoproj.io/v1alpha1" - kind = "Application" - metadata = { - name = "container-registry" - namespace = "argocd" - } - spec = { - project = "default" - source = { - repoURL = "ssh://jv@192.168.100.68/home/jv/git-server/my-homelab-configs.git" - targetRevision = "main" - path = "apps/container-registry" - } - destination = { - server = "https://kubernetes.default.svc" - namespace = "container-registry" - } - syncPolicy = { - automated = { - prune = true - selfHeal = true - } - syncOptions = ["CreateNamespace=true"] - } - } - } -} - -resource "kubernetes_manifest" "production_website" { - depends_on = [kubernetes_manifest.container_registry] +resource "kubernetes_manifest" "argocd_application" { + for_each = var.applications field_manager { force_conflicts = true @@ -57,26 +23,26 @@ resource "kubernetes_manifest" "production_website" { apiVersion = "argoproj.io/v1alpha1" kind = "Application" metadata = { - name = "php-web-app" - namespace = "argocd" + name = each.key + namespace = var.argocd_namespace } spec = { - project = "default" + project = each.value.project source = { - repoURL = "ssh://jv@192.168.100.68/home/jv/git-server/my-homelab-configs.git" - targetRevision = "main" - path = "apps/website" + repoURL = var.gitops_repo_url + targetRevision = each.value.target_revision + path = each.value.path } destination = { server = "https://kubernetes.default.svc" - namespace = "default" + namespace = each.value.namespace } syncPolicy = { automated = { - prune = true - selfHeal = true + prune = each.value.prune + selfHeal = each.value.self_heal } - syncOptions = ["CreateNamespace=true"] + syncOptions = each.value.create_namespace ? ["CreateNamespace=true"] : [] } } } diff --git a/bootstrap/apps/variables.tf b/bootstrap/apps/variables.tf new file mode 100644 index 0000000..d8eada6 --- /dev/null +++ b/bootstrap/apps/variables.tf @@ -0,0 +1,56 @@ +variable "kubeconfig_path" { + type = string + default = "/home/jv/.kube/config" +} + +variable "argocd_namespace" { + type = string + default = "argocd" +} + +variable "gitops_repo_url" { + type = string + default = "ssh://jv@192.168.100.68/home/jv/git-server/my-homelab-configs.git" +} + +variable "applications" { + type = map(object({ + project = string + path = string + namespace = string + target_revision = string + prune = bool + self_heal = bool + create_namespace = bool + })) + + default = { + container-registry = { + project = "default" + path = "apps/container-registry" + namespace = "container-registry" + target_revision = "main" + prune = true + self_heal = true + create_namespace = true + } + gitea = { + project = "default" + path = "apps/gitea" + namespace = "gitea-system" + target_revision = "main" + prune = true + self_heal = true + create_namespace = true + } + website-production = { + project = "default" + path = "apps/website" + namespace = "website-production" + target_revision = "main" + prune = true + self_heal = true + create_namespace = true + } + } +} diff --git a/bootstrap/cluster/.terraform.lock.hcl b/bootstrap/cluster/.terraform.lock.hcl new file mode 100644 index 0000000..8ee8560 --- /dev/null +++ b/bootstrap/cluster/.terraform.lock.hcl @@ -0,0 +1,76 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/hashicorp/external" { + version = "2.4.0" + constraints = "~> 2.3" + hashes = [ + "h1:/+SUsgcSNW/RVmImj/m1Z+hSUU5BUMu8CMbLwyuhMMU=", + "h1:/MpYrEgBd7Gu/lyuyViR4ccaYSpHHIiHZfIxy7AGy2U=", + "h1:0SJyK1GT4ma4uTSUz8Y619fc1xRTzJ9FKFvcpvZlSIo=", + "h1:4WFLgDJ0NXqNlKJ2UFniW8/GMMterkrWBXmi/+yW2do=", + "h1:5vgd2l3wLBjETL3h0npHKNm8efKSE7Xb2uxh+8cysm8=", + "h1:6dSIL7ou4oq3hcuRUx4OLRc31PQ1oAC1d885hnQlOh0=", + "h1:7OLUuwL6BIhAYehzbDX5NVTIoDAIBf2dYZGrpSKe0r0=", + "h1:9PmuE7y6euSGJtnl7W0TIBvZBU1pNojfa615jnJAiIs=", + "h1:ME8EoyLapCOOAzPP8uPTm+OSF7nil1r0wOeJ+Q9KNZ8=", + "h1:PxaDF/WKlkCtfmiqzws5/23jq3QgcoJ5TMeEIimtvDQ=", + "h1:auXuC9e9AsQxrYQj6OyYgqKHgjry0IkJnKHpPShoLHM=", + "h1:eBAcMZAh1wmCyYQMCIPyXljazVRYAHkEu/31RbZD6Sw=", + "h1:jkfHEdIUc6uxIm8zuw0lVZliedeqgzDE9qb1Y608VUI=", + "h1:mC6Rw0cABPfsBZLVXfdOByUUW1hHkfscaQufTzVM+rw=", + "h1:nE1mN6A/5lJVLiNQmVm43m9ME2uHlqwIYUVxlZgblSk=", + "zh:483962b782cee2c970f4bdf6118e4bb665f37a0d488024c660b7a7c9853afc93", + "zh:4a8b42651f6de0ea93854ece3ccdf8e2b21b911145859402a9a6ec6ecf31a23c", + "zh:56836ea1468cb328e98cccc76cccc208fb7336513bc76de76309541b8e5ceef2", + "zh:6d30c2c1aff7e0ddcfffdf815e570f5ed8b77d1ce2d31440c7c50c6f3e7cbe97", + "zh:80a21a23bd9bfafb74e4fc5f2a0e2bc33517c418d5f16dc020b7febba9ef6cd2", + "zh:95d9ce0e6407f199f7e9d3aefc3345a6e67c18f0c8280207417272cbb3f973ad", + "zh:98e46c6504da5f1020489730d23f03a1337e643ed452f271c2a13e6af4687a16", + "zh:a3f59b81da319a87bb3ac4a31e669874a090f082959da29975fa6f11f53b4731", + "zh:a5793f88bb25000d6e6b8b2cd5ca71366a8d4e373e17a247089a80331ccf824e", + "zh:cfc9af183162936b2e9a726c4a68d773b4511ada23b2c764f5add90f080e747d", + "zh:d85e722d771fb7865d9d5b591334d0f2b7598b7e66f534f93923effe6d5134ed", + "zh:de6e5d954ef91bb0ad41f30305ab8c31718ea39308523529f528babd0b27db71", + "zh:fca19dda5e05221e231d400fc39fda4f9e9abfe33e8e833a215eb094fe226ed8", + "zh:fcf67b347f2dc3c609c819ad5f27078fa3d0235311cfe5e33242eb49b3a4a6a8", + "zh:fea69a81ffcd63cb776b0f2c13a99a8a6d64e0b9de111ea1926bc4e713023ece", + ] +} + +provider "registry.opentofu.org/hashicorp/null" { + version = "3.3.0" + constraints = "~> 3.2" + hashes = [ + "h1:0r7+t8CqzjfBgHgEiJGBCw+McEUdRXliMdF+Hk29d8o=", + "h1:EvvCOc4FJY3NitSm6BpzCcUPU53LayVCB/tPOxYmy7U=", + "h1:IDVnZXNCh0u4LfeSazc9z1v/kNz+92Eej7ePWV6SbyE=", + "h1:Iw2c0n9/4fS92N5WnJ3CCSwSUXZO953oHp9gj3pWCaM=", + "h1:JofS1og3hPN0ANjH+gNjxrJyyk6znodpC/F0qhp4eEk=", + "h1:QIBhsJ4+5+t0vFEgJwtezNLT31tsptFHOEyGAAhLR1o=", + "h1:RjjoL9qRPwNTwLdtJsYUaFvunbPM2/oujf2DcUcitOE=", + "h1:SSirA+z2VWTs1s+TCAx8vVKg9jh6cRjxqc8LYi2iQTI=", + "h1:U2XZc7hxcpcWp/C2S9LtuGUimhMOD2UT5xAEJJQQQaU=", + "h1:bPG+xE5UonkJv3y/Yn9Q7OfbP2qHU/QKiS31nwfe7S0=", + "h1:eODLdk/pARc4yxChAFtwseVmBr+r5fF9yGOvUhwGEyM=", + "h1:iFj1oM5ZPENspsPqK1kcvZzyP95jJE/CM0rlu0MfIss=", + "h1:mdu+qpyVmjDDLMrcL1JFy+cSyF58I3TFJwB5NssCZ58=", + "h1:tJmep6aoBeDH77XsYU65HAbi0RAjxtsmbCOXmnqT13U=", + "h1:tdMTn1evBLd6KCeLqWdQXCpF07hBu3n5rY6N3rXw3Rc=", + "zh:083dcc0bec53f8abfa3f2aa2ce9d732a9675338fd60ae7d61162e25db7cb08bf", + "zh:19f7456b5a2ad16595860974714bfdb25b87bc16356ea9d5c7453892aaa27864", + "zh:222c0ed1fed4e4c677ebe626104dbfdba66763e264de0d9c27c58ce60104ee69", + "zh:271711d6caa7dd5a4e9b79fe8c679fab61a840bcf80040a0f5ebb425d1b27d97", + "zh:5adcf35f30baaea13f80c2a2c774deb9369892719493049687e23476c9dff40f", + "zh:5bcfd19df16e73d7f0ad75bd09e2b3b86cf6700d09822d585d68304b71de1d97", + "zh:604edecf263e38674decb35bb4e0e048fdc951f26fa103c33065ff9728f0313b", + "zh:782acbfb4fa4807e273e588fe45b4aaea9dd0fd1136f76ec3200f6f4db3af8d6", + "zh:84411a596d528fe67294e5c1cfd0c2036b08802497bcc4215ce518924f3c9a4a", + "zh:85e79eecf3f5348975cffec3016b0eba3baf605646102d4348796ccd2df2e5f6", + "zh:95669535ca17aeefef307ebfd59ce6930953173baae5637e8cbbf0297ec7ad58", + "zh:d04d9b177747bfd66b4a45b5d911a2a7822aa8451f5e35621971fb7a4206b530", + "zh:e6d9c924475283e90833450a14a732f4deb6d9bb131db8f86ab856e894270836", + "zh:ebcab0c8a1334c86ed7cfa53f571a17ad6d27e9901f27a8854ea622a74b54bb6", + "zh:ef9c757bb2c83d2103811a3d86b6ec5be06b0ffc337b84db1582d023bce7cdcd", + ] +} diff --git a/bootstrap/cluster/main.tf b/bootstrap/cluster/main.tf index 3037668..8ae9678 100644 --- a/bootstrap/cluster/main.tf +++ b/bootstrap/cluster/main.tf @@ -13,63 +13,143 @@ terraform { } resource "null_resource" "kubeadm_control_plane" { - provisioner "local-exec" { - command = </dev/null +fi +sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml +sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml +sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint} +sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null < /tmp/join.sh", - "sudo sh /tmp/join.sh", - "rm -f /tmp/join.sh" - ] - } + </dev/null +fi +sudo sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml +sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml +sudo mkdir -p /etc/containerd/certs.d/${self.triggers.registry_endpoint} +sudo tee /etc/containerd/certs.d/${self.triggers.registry_endpoint}/hosts.toml >/dev/null <&2 + exit 1 +fi + +known_hosts_file="$(mktemp)" +known_hosts_sorted="$(mktemp)" +trap 'rm -f "$${known_hosts_file}" "$${known_hosts_sorted}"' EXIT + +kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" get configmap argocd-ssh-known-hosts-cm \ + -o jsonpath='{.data.ssh_known_hosts}' > "$${known_hosts_file}" 2>/dev/null || true +ssh-keyscan -H "$${repo_host}" >> "$${known_hosts_file}" 2>/dev/null +sort -u "$${known_hosts_file}" > "$${known_hosts_sorted}" +kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" create configmap argocd-ssh-known-hosts-cm \ + --from-file=ssh_known_hosts="$${known_hosts_sorted}" \ + --dry-run=client -o yaml | kubectl --kubeconfig "${self.triggers.kubeconfig_path}" apply -f - + +kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" create secret generic "${self.triggers.secret_name}" \ + --from-literal=type=git \ + --from-literal=url="${self.triggers.repo_url}" \ + --from-file=sshPrivateKey="${self.triggers.ssh_key_path}" \ + --dry-run=client -o yaml | kubectl --kubeconfig "${self.triggers.kubeconfig_path}" apply -f - + +kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" label secret "${self.triggers.secret_name}" \ + argocd.argoproj.io/secret-type=repository --overwrite +EOT + } +} + +resource "helm_release" "extra_tools" { + for_each = var.extra_helm_releases + + depends_on = [null_resource.calico_ready] + name = each.key + repository = each.value.repository + chart = each.value.chart + version = each.value.version != "" ? each.value.version : null + namespace = each.value.namespace + create_namespace = each.value.create_namespace + timeout = each.value.timeout + values = each.value.values_yaml != "" ? [each.value.values_yaml] : [] + + dynamic "set" { + for_each = each.value.set_values + content { + name = set.key + value = set.value + } } } diff --git a/bootstrap/platform/variables.tf b/bootstrap/platform/variables.tf new file mode 100644 index 0000000..245961b --- /dev/null +++ b/bootstrap/platform/variables.tf @@ -0,0 +1,82 @@ +variable "kubeconfig_path" { + type = string + default = "/home/jv/.kube/config" +} + +variable "pod_network_cidr" { + type = string + default = "10.244.0.0/16" +} + +variable "gitops_repo_url" { + type = string + default = "ssh://jv@192.168.100.68/home/jv/git-server/my-homelab-configs.git" +} + +variable "gitops_ssh_key_path" { + type = string + default = "/home/jv/.ssh/id_ed25519" +} + +variable "calico" { + type = object({ + repository = string + version = string + namespace = string + }) + + default = { + repository = "https://docs.tigera.io/calico/charts" + version = "v3.32.0" + namespace = "tigera-operator" + } +} + +variable "openebs" { + type = object({ + repository = string + version = string + namespace = string + retain_storage_class = string + base_path = string + }) + + default = { + repository = "https://openebs.github.io/openebs" + version = "4.3.3" + namespace = "openebs" + retain_storage_class = "openebs-hostpath-retain" + base_path = "/var/openebs/local" + } +} + +variable "argocd" { + type = object({ + repository = string + version = string + namespace = string + repo_secret_name = string + }) + + default = { + repository = "https://argoproj.github.io/argo-helm" + version = "8.5.8" + namespace = "argocd" + repo_secret_name = "homelab-configs-repo" + } +} + +variable "extra_helm_releases" { + type = map(object({ + repository = string + chart = string + version = string + namespace = string + create_namespace = bool + timeout = number + values_yaml = string + set_values = map(string) + })) + + default = {} +} diff --git a/lab.sh b/lab.sh index c9911f5..dc8571a 100755 --- a/lab.sh +++ b/lab.sh @@ -1,117 +1,263 @@ +#!/usr/bin/env bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +BUILDX_CONFIG="/tmp/buildx-config.toml" +KUBECONFIG_PATH="${KUBECONFIG_PATH:-${TF_VAR_kubeconfig_path:-/home/jv/.kube/config}}" + +trap 'rm -f "${BUILDX_CONFIG}"' EXIT + +run_tofu_stack() { + local stack="$1" + + tofu -chdir="${REPO_ROOT}/${stack}" init + tofu -chdir="${REPO_ROOT}/${stack}" apply -auto-approve +} + +cleanup_calico_links() { + ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true + sudo ip link delete vxlan.calico 2>/dev/null || true + sudo ip link delete tunl0 2>/dev/null || true + sudo ip link delete cni0 2>/dev/null || true + sudo ip link delete kube-ipvs0 2>/dev/null || true + ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true +} + +cleanup_iptables() { + sudo iptables -F || true + sudo iptables -X || true + sudo iptables -t nat -F || true + sudo iptables -t nat -X || true + sudo iptables -t mangle -F || true + sudo iptables -t mangle -X || true + sudo iptables -t raw -F || true + sudo iptables -t raw -X || true + if command -v ipvsadm >/dev/null 2>&1; then + sudo ipvsadm --clear || true + fi +} + +cleanup_mounts() { + if command -v findmnt >/dev/null 2>&1; then + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(findmnt -Rno TARGET /var/lib/kubelet /var/lib/containerd 2>/dev/null | sort -r) + fi + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) + sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true +} + +cleanup_node() { + sudo kubeadm reset --force || true + sudo systemctl stop kubelet 2>/dev/null || true + sudo systemctl stop containerd 2>/dev/null || true + sudo killall containerd-shim-runc-v2 2>/dev/null || true + + cleanup_mounts + + sudo rm -rf \ + /etc/kubernetes/ \ + /var/lib/etcd/ \ + /var/lib/kubelet/ \ + /var/lib/cni/ \ + /etc/cni/net.d \ + /run/flannel \ + /run/calico \ + /var/run/calico \ + /var/lib/calico \ + /var/log/calico \ + /var/lib/containerd/* \ + /run/containerd/* \ + /etc/containerd/certs.d \ + /etc/containerd/config.toml + sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam + + cleanup_iptables + cleanup_calico_links + + sudo mkdir -p /etc/containerd/certs.d + sudo systemctl reset-failed kubelet containerd 2>/dev/null || true + sudo systemctl start containerd 2>/dev/null || true +} + +website_registry_endpoint() { + local image + + image="$(awk '$1 == "image:" && $2 ~ /php-website/ {print $2; exit}' "${REPO_ROOT}/apps/website/web-app.yaml")" + if [[ -z "${image}" || "${image}" != */* ]]; then + echo "Could not determine website registry endpoint from apps/website/web-app.yaml" >&2 + exit 1 + fi + + printf '%s\n' "${image%%/*}" +} + up() { + local registry_endpoint + + registry_endpoint="$(website_registry_endpoint)" + export TF_VAR_registry_endpoint="${TF_VAR_registry_endpoint:-${registry_endpoint}}" + export TF_VAR_kubeconfig_path="${TF_VAR_kubeconfig_path:-${KUBECONFIG_PATH}}" + export KUBECONFIG="${TF_VAR_kubeconfig_path}" + + if [[ "${TF_VAR_registry_endpoint}" != "${registry_endpoint}" ]]; then + echo "TF_VAR_registry_endpoint must match apps/website/web-app.yaml (${registry_endpoint})" >&2 + exit 1 + fi + echo "Deploying the homelab infrastructure..." docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - cat < /tmp/buildx-config.toml + cat < "${BUILDX_CONFIG}" +[registry."${registry_endpoint}"] + http = true + insecure = true [registry."127.0.0.1:30500"] http = true + insecure = true [registry."localhost:30500"] http = true + insecure = true EOF docker buildx rm lab-builder 2>/dev/null || true - - docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config /tmp/buildx-config.toml --use + docker buildx create --name lab-builder --driver docker-container --driver-opt network=host --config "${BUILDX_CONFIG}" --use docker buildx inspect --bootstrap - cd bootstrap/cluster - tofu init - tofu apply -auto-approve + run_tofu_stack "bootstrap/cluster" + run_tofu_stack "bootstrap/platform" + run_tofu_stack "bootstrap/apps" - cd ../platform - tofu init - tofu apply -auto-approve - - cd ../apps - tofu init - tofu apply -auto-approve - - cd ../.. - - until kubectl get deployment local-registry -n container-registry -o jsonpath='{.status.availableReplicas}' 2>/dev/null | grep -q '^[1-9]'; do - echo "Waiting for local-registry pods to initialize..." - sleep 5 - done + kubectl --kubeconfig "${KUBECONFIG}" -n container-registry rollout status deployment/local-registry --timeout=300s docker buildx build \ --network host \ --platform linux/amd64,linux/arm64 \ - -t "127.0.0.1:30500/php-website:latest" \ - -f apps/website/Dockerfile \ - apps/website/ \ + -t "${registry_endpoint}/php-website:latest" \ + -f "${REPO_ROOT}/apps/website/Dockerfile" \ + "${REPO_ROOT}/apps/website/" \ --push - kubectl patch application php-web-app -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"sync"}}}' + kubectl --kubeconfig "${KUBECONFIG}" patch application website-production -n argocd --type merge -p '{"metadata":{"annotations":{"argocd.argoproj.io/refresh":"sync"}}}' - echo "Deployment successfully completed!" + echo "Deployment successfully completed." } nuke() { + local worker_ssh_targets + local worker_targets + local target + echo "Brutally nuking the homelab infrastructure..." + worker_ssh_targets="${WORKER_SSH_TARGETS-jv@192.168.100.89}" + read -r -a worker_targets <<< "${worker_ssh_targets}" echo "--> Terminating local OpenTofu tasks..." killall tofu terraform 2>/dev/null || true - echo "--> Eviscerating local Kubernetes components (Laptop)..." - sudo kubeadm reset --force || true - sudo systemctl stop containerd 2>/dev/null || true - sudo killall containerd-shim-runc-v2 2>/dev/null || true + echo "--> Eviscerating local Kubernetes components..." + cleanup_node + sudo rm -f "${KUBECONFIG_PATH}" - sudo umount /var/lib/containerd/srun/* 2>/dev/null || true - sudo rm -rf /var/lib/containerd/* /run/containerd/* - sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d /home/jv/.kube/ + for target in "${worker_targets[@]}"; do + echo "--> Eviscerating remote Kubernetes components (${target})..." + if ! ssh -o ConnectTimeout=5 "${target}" "bash -s" <<'EOF' +set -euo pipefail - sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X - sudo ip link delete cilium_host 2>/dev/null || true - sudo ip link delete cilium_net 2>/dev/null || true - sudo ip link delete cilium_vxlan 2>/dev/null || true - - sudo systemctl start containerd - - echo "--> Eviscerating remote Kubernetes components (Raspberry Pi)..." - ssh -o ConnectTimeout=5 jv@192.168.100.89 << 'EOF' 2>/dev/null || true - # 1. Force reset kubeadm configurations - sudo kubeadm reset --force || true - - # 2. Halt the container runtime engine to drop file descriptor and socket locks - sudo systemctl stop containerd 2>/dev/null || true - sudo killall containerd-shim-runc-v2 2>/dev/null || true - - # 3. Unmount any lingering ephemeral pod volumes, secrets, or token rings - sudo umount -f /var/lib/kubelet/pods/*/*/*/* 2>/dev/null || true - - # 4. Completely wipe the cluster file configurations and runtime data tracks - sudo rm -rf /etc/kubernetes/ /var/lib/kubelet/ /var/lib/cni/ /etc/cni/net.d - sudo rm -rf /var/lib/containerd/* /run/containerd/* - - # 5. Reset network routing policies left over by the CNI - sudo iptables -F && sudo iptables -t nat -F && sudo iptables -t mangle -F && sudo iptables -X - - # 6. Bring the container engine back online with a completely clean state slate - sudo systemctl start containerd -EOF - - docker buildx rm lab-builder 2>/dev/null || true - rm -f /tmp/buildx-config.toml || true - - echo "--> Deleting OpenTofu tracking state files..." - rm -rf bootstrap/cluster/terraform.tfstate* - rm -rf bootstrap/cluster/.terraform/ - rm -rf bootstrap/cluster/.terraform.lock.hcl - - rm -rf bootstrap/platform/terraform.tfstate* - rm -rf bootstrap/platform/.terraform/ - rm -rf bootstrap/platform/.terraform.lock.hcl - - rm -rf bootstrap/apps/terraform.tfstate* - rm -rf bootstrap/apps/.terraform/ - rm -rf bootstrap/apps/.terraform.lock.hcl - - echo "Destruction complete! Your hardware is completely sanitized." +cleanup_calico_links() { + ip link show | awk -F: '/^[0-9]+: cali/ {print $2}' | cut -d@ -f1 | xargs -r -n1 sudo ip link delete 2>/dev/null || true + sudo ip link delete vxlan.calico 2>/dev/null || true + sudo ip link delete tunl0 2>/dev/null || true + sudo ip link delete cni0 2>/dev/null || true + sudo ip link delete kube-ipvs0 2>/dev/null || true + ip netns list | awk '/^(cni-|calico)/ {print $1}' | xargs -r -n1 sudo ip netns delete 2>/dev/null || true } -case "$1" in +cleanup_iptables() { + sudo iptables -F || true + sudo iptables -X || true + sudo iptables -t nat -F || true + sudo iptables -t nat -X || true + sudo iptables -t mangle -F || true + sudo iptables -t mangle -X || true + sudo iptables -t raw -F || true + sudo iptables -t raw -X || true + if command -v ipvsadm >/dev/null 2>&1; then + sudo ipvsadm --clear || true + fi +} + +cleanup_mounts() { + if command -v findmnt >/dev/null 2>&1; then + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(findmnt -Rno TARGET /var/lib/kubelet /var/lib/containerd 2>/dev/null | sort -r) + fi + while IFS= read -r mountpoint; do + sudo umount -f "${mountpoint}" 2>/dev/null || sudo umount -l "${mountpoint}" 2>/dev/null || true + done < <(find /var/lib/kubelet/pods -mindepth 2 -maxdepth 5 -type d 2>/dev/null || true) + sudo umount -f /var/lib/containerd/srun/* 2>/dev/null || sudo umount -l /var/lib/containerd/srun/* 2>/dev/null || true +} + +sudo kubeadm reset --force || true +sudo systemctl stop kubelet 2>/dev/null || true +sudo systemctl stop containerd 2>/dev/null || true +sudo killall containerd-shim-runc-v2 2>/dev/null || true + +cleanup_mounts + +sudo rm -rf \ + /etc/kubernetes/ \ + /var/lib/etcd/ \ + /var/lib/kubelet/ \ + /var/lib/cni/ \ + /etc/cni/net.d \ + /run/flannel \ + /run/calico \ + /var/run/calico \ + /var/lib/calico \ + /var/log/calico \ + /var/lib/containerd/* \ + /run/containerd/* \ + /etc/containerd/certs.d \ + /etc/containerd/config.toml +sudo rm -f /opt/cni/bin/calico /opt/cni/bin/calico-ipam + +cleanup_iptables +cleanup_calico_links + +sudo mkdir -p /etc/containerd/certs.d +sudo systemctl reset-failed kubelet containerd 2>/dev/null || true +sudo systemctl start containerd 2>/dev/null || true +EOF + then + echo "Remote cleanup failed for ${target}; not deleting OpenTofu state." >&2 + exit 1 + fi + done + + docker buildx rm lab-builder 2>/dev/null || true + docker rm -f buildx_buildkit_lab-builder0 2>/dev/null || true + rm -f "${BUILDX_CONFIG}" || true + + echo "--> Deleting OpenTofu tracking state files..." + rm -rf "${REPO_ROOT}"/bootstrap/cluster/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/cluster/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/cluster/.terraform/ + rm -rf "${REPO_ROOT}"/bootstrap/platform/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/platform/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/platform/.terraform/ + rm -rf "${REPO_ROOT}"/bootstrap/apps/terraform.tfstate* + rm -f "${REPO_ROOT}"/bootstrap/apps/.terraform.tfstate.lock.info + rm -rf "${REPO_ROOT}"/bootstrap/apps/.terraform/ + + echo "Destruction complete. Retained data under /var/openebs/local was left intact." +} + +case "${1:-}" in up) up ;;