aks setup complete

pagopa · May 22, 2024 · f708c6a · f708c6a
1 parent 5cf9682
commit f708c6a
Show file tree

Hide file tree

Showing 13 changed files with 248 additions and 226 deletions.
diff --git a/src/aks-platform/.terraform.lock.hcl b/src/aks-platform/.terraform.lock.hcl
diff --git a/src/aks-platform/00_key_vault.tf b/src/aks-platform/00_key_vault.tf
@@ -1,4 +1,4 @@
-data "azurerm_key_vault" "kv_core" {
-  name                = "dvopla-d-neu-kv"
-  resource_group_name = "dvopla-d-sec-rg"
+data "azurerm_key_vault" "kv_core_ita" {
+  name                = "dvopla-d-itn-core-kv"
+  resource_group_name = "dvopla-d-itn-sec-rg"
 }
diff --git a/src/aks-platform/00_network.tf b/src/aks-platform/00_network.tf
@@ -34,25 +34,25 @@ data "azurerm_public_ip" "pip_aks_outboud" {
 #
 # Subnet
 #
-data "azurerm_subnet" "private_endpoint_subnet" {
-  name                 = "${local.product}-private-endpoints-snet"
-  resource_group_name  = data.azurerm_resource_group.vnet_core_rg.name
-  virtual_network_name = data.azurerm_virtual_network.vnet_core.name
-}
+# data "azurerm_subnet" "private_endpoint_subnet" {
+#   name                 = "${local.product}-private-endpoints-snet"
+#   resource_group_name  = data.azurerm_resource_group.vnet_core_rg.name
+#   virtual_network_name = data.azurerm_virtual_network.vnet_core.name
+# }
 
-data "azurerm_subnet" "private_endpoint_italy_subnet" {
-  name                 = "${local.product}-private-endpoints-italy-snet"
-  resource_group_name  = data.azurerm_resource_group.vnet_italy_rg.name
-  virtual_network_name = data.azurerm_virtual_network.vnet_italy.name
-}
+# data "azurerm_subnet" "private_endpoint_italy_subnet" {
+#   name                 = "${local.product}-private-endpoints-italy-snet"
+#   resource_group_name  = data.azurerm_resource_group.vnet_italy_rg.name
+#   virtual_network_name = data.azurerm_virtual_network.vnet_italy.name
+# }
 
 #
 # Dns
 #
-data "azurerm_private_dns_zone" "storage_account_private_dns_zone" {
-  name                = "privatelink.blob.core.windows.net"
-  resource_group_name = data.azurerm_resource_group.vnet_core_rg.name
-}
+# data "azurerm_private_dns_zone" "storage_account_private_dns_zone" {
+#   name                = "privatelink.blob.core.windows.net"
+#   resource_group_name = data.azurerm_resource_group.vnet_core_rg.name
+# }
 
 data "azurerm_private_dns_zone" "internal" {
   name                = local.internal_dns_zone_name

diff --git a/src/aks-platform/01_network_aks.tf b/src/aks-platform/01_network_aks.tf
@@ -1,17 +1,18 @@
-# k8s cluster subnet
-module "snet_aks" {
-  source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet?ref=v7.69.1"
-
-  name = "${local.project}-aks-snet"
+resource "azurerm_subnet" "system_aks_subnet" {
+  name                                          = "${local.project}-system-aks"
+  resource_group_name                           = data.azurerm_resource_group.vnet_italy_rg.name
+  virtual_network_name                          = data.azurerm_virtual_network.vnet_italy.name
+  address_prefixes                              = var.cidr_subnet_system_aks
+  private_endpoint_network_policies_enabled     = true
+  private_link_service_network_policies_enabled = true
+}
 
+resource "azurerm_subnet" "user_aks_subnet" {
+  name                 = "${local.project}-user-aks"
   resource_group_name  = data.azurerm_resource_group.vnet_italy_rg.name
   virtual_network_name = data.azurerm_virtual_network.vnet_italy.name
+  address_prefixes     = var.cidr_subnet_user_aks
 
-  address_prefixes                          = var.cidr_subnet_aks
-  private_endpoint_network_policies_enabled = var.aks_private_cluster_enabled
-
-  service_endpoints = [
-    "Microsoft.Web",
-    "Microsoft.Storage"
-  ]
+  private_endpoint_network_policies_enabled     = true
+  private_link_service_network_policies_enabled = true
 }
diff --git a/src/aks-platform/02_aks_0.tf b/src/aks-platform/02_aks_0.tf
@@ -14,9 +14,7 @@ resource "azurerm_resource_group" "rg_aks_backup" {
 
 
 module "aks" {
-  source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v7.70.1"
-
-  count = var.aks_enabled ? 1 : 0
+  source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v8.14.0"
 
   name                       = local.aks_cluster_name
   resource_group_name        = azurerm_resource_group.rg_aks.name
@@ -37,50 +35,34 @@ module "aks" {
   system_node_pool_node_count_min  = var.aks_system_node_pool.node_count_min
   system_node_pool_node_count_max  = var.aks_system_node_pool.node_count_max
   ### K8s node configuration
-  system_node_pool_node_labels = var.aks_system_node_pool.node_labels
-  system_node_pool_tags        = var.aks_system_node_pool.node_tags
-
-  #
-  # 👤 User node pool
-  #
-  user_node_pool_enabled = var.aks_user_node_pool.enabled
-  user_node_pool_name    = var.aks_user_node_pool.name
-  ### vm configuration
-  user_node_pool_vm_size         = var.aks_user_node_pool.vm_size
-  user_node_pool_os_disk_type    = var.aks_user_node_pool.os_disk_type
-  user_node_pool_os_disk_size_gb = var.aks_user_node_pool.os_disk_size_gb
-  user_node_pool_node_count_min  = var.aks_user_node_pool.node_count_min
-  user_node_pool_node_count_max  = var.aks_user_node_pool.node_count_max
-  ### K8s node configuration
-  user_node_pool_node_labels = var.aks_user_node_pool.node_labels
-  user_node_pool_node_taints = var.aks_user_node_pool.node_taints
-  user_node_pool_tags        = var.aks_user_node_pool.node_tags
-  # end user node pool
+  system_node_pool_node_labels                  = var.aks_system_node_pool.node_labels
+  system_node_pool_tags                         = var.aks_system_node_pool.node_tags
 
   #
   # ☁️ Network
   #
   vnet_id        = data.azurerm_virtual_network.vnet_italy.id
-  vnet_subnet_id = module.snet_aks.id
+  vnet_subnet_id = azurerm_subnet.system_aks_subnet.id
 
   outbound_ip_address_ids = [data.azurerm_public_ip.pip_aks_outboud.id]
   private_cluster_enabled = var.aks_private_cluster_enabled
   network_profile = {
-    docker_bridge_cidr = "172.17.0.1/16"
-    dns_service_ip     = "10.250.0.10"
-    network_plugin     = "azure"
-    network_policy     = "azure"
-    outbound_type      = "loadBalancer"
-    service_cidr       = "10.250.0.0/16"
+    docker_bridge_cidr  = "172.17.0.1/16"
+    dns_service_ip      = "10.0.0.10"
+    network_plugin      = "azure"
+    network_plugin_mode = "overlay"
+    network_policy      = "azure"
+    outbound_type       = "loadBalancer"
+    service_cidr        = "10.0.0.0/16"
   }
   # end network
   oidc_issuer_enabled = true
 
   aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id, data.azuread_group.adgroup_externals.object_id] : [data.azuread_group.adgroup_admin.object_id]
 
-  addon_azure_policy_enabled                     = var.aks_addons.azure_policy
-  addon_azure_key_vault_secrets_provider_enabled = var.aks_addons.azure_key_vault_secrets_provider
-  addon_azure_pod_identity_enabled               = var.aks_addons.pod_identity_enabled
+  addon_azure_policy_enabled                     = true
+  addon_azure_key_vault_secrets_provider_enabled = true
+  addon_azure_pod_identity_enabled               = true
 
   default_metric_alerts = var.aks_metric_alerts_default
   custom_metric_alerts  = var.aks_metric_alerts_custom
@@ -101,18 +83,57 @@ module "aks" {
   ]
 
   tags = var.tags
+}
 
-  depends_on = [
-    module.snet_aks,
-    data.azurerm_public_ip.pip_aks_outboud,
-    data.azurerm_virtual_network.vnet_italy
-  ]
+resource "azurerm_kubernetes_cluster_node_pool" "user_nodepool_default" {
+  count = var.aks_user_node_pool.enabled ? 1 : 0
+
+  kubernetes_cluster_id = module.aks.id
+
+  name = var.aks_user_node_pool.name
+
+  ### vm configuration
+  vm_size = var.aks_user_node_pool.vm_size
+  # https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-general
+  os_disk_type           = var.aks_user_node_pool.os_disk_type # Managed or Ephemeral
+  os_disk_size_gb        = var.aks_user_node_pool.os_disk_size_gb
+  zones                  = var.aks_user_node_pool.zones
+  ultra_ssd_enabled      = var.aks_user_node_pool.ultra_ssd_enabled
+  enable_host_encryption = var.aks_user_node_pool.enable_host_encryption
+  os_type                = "Linux"
+
+  ### autoscaling
+  enable_auto_scaling = true
+  node_count          = var.aks_user_node_pool.node_count_min
+  min_count           = var.aks_user_node_pool.node_count_min
+  max_count           = var.aks_user_node_pool.node_count_max
+
+  ### K8s node configuration
+  max_pods    = var.aks_user_node_pool.max_pods
+  node_labels = var.aks_user_node_pool.node_labels
+  node_taints = var.aks_user_node_pool.node_taints
+
+  ### networking
+  vnet_subnet_id        = azurerm_subnet.user_aks_subnet.id
+  enable_node_public_ip = false
+
+  upgrade_settings {
+    max_surge = var.aks_user_node_pool.upgrade_settings_max_surge
+  }
+
+  tags = merge(var.tags, var.aks_user_node_pool.node_tags)
+
+  lifecycle {
+    ignore_changes = [
+      node_count
+    ]
+  }
 }
 
 resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" {
   count = var.aks_spot_user_node_pool.enabled ? 1 : 0
 
-  kubernetes_cluster_id = module.aks[0].id
+  kubernetes_cluster_id = module.aks.id
 
   name = var.aks_spot_user_node_pool.name
 
@@ -140,7 +161,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" {
   node_taints = var.aks_spot_user_node_pool.node_taints
 
   ### networking
-  vnet_subnet_id        = module.snet_aks.id
+  vnet_subnet_id        = azurerm_subnet.user_aks_subnet.id
   enable_node_public_ip = false
 
   tags = merge(var.tags, var.aks_spot_user_node_pool.node_tags)
@@ -150,13 +171,15 @@ resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" {
       node_count
     ]
   }
+
+  depends_on = [module.aks]
 }
 
 
 resource "azurerm_role_assignment" "managed_identity_operator_vs_aks_managed_identity" {
   scope                = azurerm_resource_group.rg_aks.id
   role_definition_name = "Managed Identity Operator"
-  principal_id         = module.aks[0].identity_principal_id
+  principal_id         = module.aks.identity_principal_id
 }
 
 #
@@ -166,7 +189,7 @@ resource "azurerm_role_assignment" "managed_identity_operator_vs_aks_managed_ide
 resource "azurerm_role_assignment" "aks_to_acr" {
   scope                = data.azurerm_container_registry.acr.id
   role_definition_name = "AcrPull"
-  principal_id         = module.aks[0].kubelet_identity_id
+  principal_id         = module.aks.kubelet_identity_id
 
   depends_on = [module.aks]
 }
@@ -181,7 +204,7 @@ resource "null_resource" "create_vnet_core_aks_link" {
 
   count = var.aks_enabled && var.aks_private_cluster_enabled ? 1 : 0
   triggers = {
-    cluster_name = module.aks[0].name
+    cluster_name = module.aks.name
     vnet_id      = data.azurerm_virtual_network.vnet_core.id
     vnet_name    = data.azurerm_virtual_network.vnet_core.name
   }

diff --git a/src/aks-platform/02_aks_storage.tf b/src/aks-platform/02_aks_storage.tf
@@ -1,3 +1,5 @@
 module "aks_storage_class" {
   source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_storage_class?ref=v7.69.1"
+
+  depends_on = [module.aks]
 }
diff --git a/src/aks-platform/03_monitoring.tf b/src/aks-platform/03_monitoring.tf
@@ -9,4 +9,6 @@ module "aks_prometheus_install" {
   source               = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_prometheus_install?ref=v7.69.1"
   prometheus_namespace = kubernetes_namespace.monitoring.metadata[0].name
   storage_class_name   = "default-zrs"
+
+  depends_on = [module.aks_storage_class]
 }