feat: app to node pool selection

* introduces app_node_selector
* enables devops stack apps to node pools using node labels

Co-authored-by: modridi <mohamed-amine.dridi@camptocamp.com>

Author: fsismondi

argocd/aad-pod-identity/templates/namespace.yaml

@@ -19,6 +19,11 @@ locals {
       principal_id = azurerm_user_assigned_identity.this[format("%s.%s", v.namespace, v.name)].principal_id
     }
   }
+
+  namespaces = merge(
+    { for i in local.azureidentities : i.namespace => null },
+    var.app_node_selectors
+  )
 }
 
 provider "helm" {
@@ -65,7 +70,7 @@ module "cluster" {
   network_policy      = var.network_policy
   vnet_subnet_id      = var.vnet_subnet_id
   agents_pool_name    = var.agents_pool_name
-  agents_labels       = var.agents_labels
+  agents_labels       = merge({ "devops-stack.io/nodepool" = var.agents_pool_name }, var.agents_labels)
   agents_count        = var.agents_count
   agents_size         = var.agents_size
   agents_max_pods     = var.agents_max_pods
@@ -100,7 +105,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "this" {
   os_disk_size_gb     = lookup(each.value, "os_disk_size_gb", null)
   os_type             = lookup(each.value, "os_type", "Linux")
   vnet_subnet_id      = lookup(each.value, "vnet_subnet_id", var.vnet_subnet_id)
-  node_labels         = lookup(each.value, "node_labels", null)
+  node_labels         = merge({ "devops-stack.io/nodepool" = each.key }, lookup(each.value, "node_labels", null))
   mode                = lookup(each.value, "mode", null)
 }
 
@@ -150,13 +155,14 @@ module "argocd" {
         loki_account_name                            = azurerm_storage_account.this.name
         loki_account_key                             = azurerm_storage_account.this.primary_access_key
         azureidentities                              = local.azureidentities
+        namespaces                                   = local.namespaces
       }
     ),
     var.app_of_apps_values_overrides,
   ]
 
   depends_on = [
-    module.cluster,
+    azurerm_kubernetes_cluster_node_pool.this, # node pools creation must precede apps creation for the pod to node assignation
   ]
 }
 

modules/aks/azure/main.tf

@@ -30,6 +30,10 @@ output "azureidentities" {
   value       = local.azureidentities
 }
 
+output "namespaces" {
+  value = local.namespaces
+}
+
 output "kubelet_identity" {
   value = module.cluster.kubelet_identity
 }

modules/aks/azure/outputs.tf

@@ -7,9 +7,11 @@ apps:
   metrics-server:
     enabled: false
 
+${yamlencode({"namespaces": namespaces})}
+
 aad-pod-identity:
   azureIdentities:
-%{for k, v in azureidentities~}
+%{ for k, v in azureidentities ~}
     ${k}:
       namespace: "${v.namespace}"
       name: "${v.name}"
@@ -19,7 +21,21 @@ aad-pod-identity:
       binding:
         name: "${v.name}"
         selector: "${v.name}"
-%{endfor~}
+%{ endfor ~}
+%{ if length(lookup(namespaces, "aad-pod-identity", {})) > 0 ~}
+  mic:
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+%{ for k, v in lookup(namespaces, "aad-pod-identity") ~}
+            - key: ${k}
+              operator: In
+              values:
+              - ${v}
+%{ endfor ~}
+%{ endif ~}
 
 argo-cd:
   controller:
@@ -125,6 +141,19 @@ kube-prometheus-stack:
       requests:
         cpu: 100m
         memory: 32Mi
+%{ if length(lookup(namespaces, "kube-prometheus-stack", {})) > 0 ~}
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+%{ for k, v in lookup(namespaces, "kube-prometheus-stack") ~}
+            - key: ${k}
+              operator: In
+              values:
+              - ${v}
+%{ endfor ~}
+%{ endif ~}
 
   alertmanager:
     alertmanagerSpec:
@@ -134,6 +163,19 @@ kube-prometheus-stack:
         requests:
           cpu: 10m
           memory: 16Mi
+%{ if length(lookup(namespaces, "kube-prometheus-stack", {})) > 0 ~}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+%{ for k, v in lookup(namespaces, "kube-prometheus-stack") ~}
+              - key: ${k}
+                operator: In
+                values:
+                - ${v}
+%{ endfor ~}
+%{ endif ~}
 
   prometheus:
     azureIdentity:
@@ -157,6 +199,19 @@ kube-prometheus-stack:
           requests:
             memory: 32Mi
             cpu: 10m
+%{ if length(lookup(namespaces, "kube-prometheus-stack",{})) > 0 ~}
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+%{ for k, v in lookup(namespaces, "kube-prometheus-stack") ~}
+              - key: ${k}
+                operator: In
+                values:
+                - ${v}
+%{ endfor ~}
+%{ endif ~}
 
   kube-state-metrics:
     resources:
@@ -167,6 +222,19 @@ kube-prometheus-stack:
         memory: 16Mi
     podSecurityPolicy:
       enabled: false
+%{ if length(lookup(namespaces, "kube-prometheus-stack", {})) > 0 ~}
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+%{ for k, v in lookup(namespaces, "kube-prometheus-stack") ~}
+              - key: ${k}
+                operator: In
+                values:
+                - ${v}
+%{ endfor ~}
+%{ endif ~}
 
   prometheus-node-exporter:
     resources:
@@ -194,6 +262,19 @@ kube-prometheus-stack:
         requests:
           cpu: 10m
           memory: 96Mi
+%{ if length(lookup(namespaces, "kube-prometheus-stack", {})) > 0 ~}
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+          - matchExpressions:
+%{ for k, v in lookup(namespaces, "kube-prometheus-stack") ~}
+            - key: ${k}
+              operator: In
+              values:
+              - ${v}
+%{ endfor ~}
+%{ endif ~}
 
 loki-stack:
   loki:
@@ -207,7 +288,6 @@ loki-stack:
             index:
               prefix: index_
               period: 24h
-
       storage_config:
         azure:
           container_name: ${loki_container_name}
@@ -218,6 +298,19 @@ loki-stack:
           active_index_directory: /data/loki/index
           shared_store: azure
           cache_location: /data/loki/boltdb-cache
+%{ if length(lookup(namespaces, "loki-stack", {})) > 0 ~}
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+%{ for k, v in lookup(namespaces, "loki-stack") ~}
+          - matchExpressions:
+            - key: ${k}
+              operator: In
+              values:
+              - ${v}
+%{ endfor ~}
+%{ endif ~}
 
 traefik:
   resources:

modules/aks/azure/values.tmpl.yaml

@@ -85,7 +85,7 @@ variable "network_policy" {
 
 variable "node_pools" {
   default     = {}
-  description = "List of node pools with minimal configuration"
+  description = "Map of node pools"
   type        = map(any)
 }
 
@@ -106,3 +106,29 @@ variable "sku_tier" {
   default     = "Free"
   type        = string
 }
+
+variable "app_node_selectors" {
+  /* Example:
+  app_node_selectors = {
+    aad-pod-identity = {
+      "kubernetes.azure.com/agentpool" = "default"
+    }
+    argocd = {
+      "odoo.camptocamp.io/nodepool"    = "mutualized"
+      "odoo.camptocamp.io/environment" = "prod"
+    }
+    non-existing-app = {
+      "odoo.camptocamp.io/nodepool"    = "mutualized"
+      "odoo.camptocamp.io/environment" = "misc-apps"
+    }
+    kube-prometheus-stack = {
+      "kubernetes.azure.com/agentpool" = "default"
+    }
+    loki-stack = {}
+    #cert-manager = {}
+  }
+*/
+  description = "Map of argoCD apps to node selector"
+  default     = {}
+  type        = map(map(string))
+}

modules/aks/azure/variables.tf

@@ -37,8 +37,6 @@ apps:
     enabled: ${metrics_server.enable}
   minio:
     enabled: ${minio.enable}
-  namespaces:
-    enabled: true
   secrets-store-csi-driver:
     enabled: false
   traefik: