Adding k8s metrics server

README.md

@@ -455,6 +455,7 @@ Useful checks after a rebuild:
 export KUBECONFIG=/home/jv/.kube/config
 
 kubectl get nodes
+kubectl top nodes
 kubectl -n argocd get applications
 kubectl -n container-registry get pods
 kubectl -n website-production get pods -o wide

README.md.tmpl

@@ -455,6 +455,7 @@ Useful checks after a rebuild:
 export KUBECONFIG=/home/jv/.kube/config
 
 kubectl get nodes
+kubectl top nodes
 kubectl -n argocd get applications
 kubectl -n container-registry get pods
 kubectl -n website-production get pods -o wide

bootstrap/platform/main.tf

@@ -425,6 +425,73 @@ EOT
   }
 }
 
+resource "helm_release" "metrics_server" {
+  for_each = var.metrics_server.enabled ? { enabled = true } : {}
+
+  depends_on       = [null_resource.calico_ready]
+  name             = "metrics-server"
+  repository       = var.metrics_server.repository
+  chart            = var.metrics_server.chart
+  version          = var.metrics_server.version
+  namespace        = var.metrics_server.namespace
+  create_namespace = false
+  timeout          = 600
+  wait             = true
+
+  values = [
+    yamlencode({
+      args = [
+        "--kubelet-insecure-tls",
+      ]
+      nodeSelector = local.prometheus_stack_node_selector
+      resources = {
+        requests = {
+          cpu    = "50m"
+          memory = "128Mi"
+        }
+        limits = {
+          memory = "256Mi"
+        }
+      }
+    })
+  ]
+}
+
+resource "null_resource" "metrics_server_ready" {
+  for_each = var.metrics_server.enabled ? { enabled = true } : {}
+
+  depends_on = [helm_release.metrics_server]
+
+  triggers = {
+    kubeconfig_path = var.kubeconfig_path
+    namespace       = var.metrics_server.namespace
+    version         = var.metrics_server.version
+  }
+
+  provisioner "local-exec" {
+    interpreter = ["/bin/bash", "-lc"]
+    command     = <<EOT
+set -euo pipefail
+
+kubectl --kubeconfig "${self.triggers.kubeconfig_path}" wait --for=condition=Available --timeout=180s apiservice/v1beta1.metrics.k8s.io
+kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" rollout status deployment/metrics-server --timeout=300s
+
+elapsed=0
+until kubectl --kubeconfig "${self.triggers.kubeconfig_path}" top nodes >/dev/null 2>&1; do
+  if [ "$elapsed" -ge 180 ]; then
+    echo "Timed out waiting for metrics-server to serve node metrics." >&2
+    kubectl --kubeconfig "${self.triggers.kubeconfig_path}" get apiservice v1beta1.metrics.k8s.io -o yaml || true
+    kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" get pods -l app.kubernetes.io/name=metrics-server -o wide || true
+    kubectl --kubeconfig "${self.triggers.kubeconfig_path}" -n "${self.triggers.namespace}" logs deployment/metrics-server --tail=120 || true
+    exit 1
+  fi
+  sleep 10
+  elapsed=$((elapsed + 10))
+done
+EOT
+  }
+}
+
 resource "kubernetes_manifest" "nodelocal_dns_service_account" {
   for_each = var.nodelocal_dns.enabled ? { enabled = true } : {}
 

bootstrap/platform/variables.tf

@@ -112,6 +112,24 @@ variable "nodelocal_dns" {
   }
 }
 
+variable "metrics_server" {
+  type = object({
+    enabled    = bool
+    repository = string
+    chart      = string
+    version    = string
+    namespace  = string
+  })
+
+  default = {
+    enabled    = true
+    repository = "https://kubernetes-sigs.github.io/metrics-server/"
+    chart      = "metrics-server"
+    version    = "3.13.1"
+    namespace  = "kube-system"
+  }
+}
+
 variable "metallb" {
   type = object({
     enabled                  = bool