diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5f8fd64..7d96eae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,7 @@ repos: # - --args=-platform=darwin_amd64 # - --args=-platform=darwin_arm64 # - --args=-platform=linux_amd64 + # - --args=-platform=linux_arm64 ## general - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 diff --git a/README.md b/README.md index 3e0cf1f..e06efb3 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,8 @@ terraform providers lock \ -platform=windows_amd64 \ -platform=darwin_amd64 \ -platform=darwin_arm64 \ - -platform=linux_amd64 + -platform=linux_amd64 \ + -platform=linux_arm64 ``` ## Precommit checks diff --git a/src/aks-platform/.terraform.lock.hcl b/src/aks-platform/.terraform.lock.hcl index 7c94e56..fef2536 100644 --- a/src/aks-platform/.terraform.lock.hcl +++ b/src/aks-platform/.terraform.lock.hcl @@ -5,7 +5,11 @@ provider "registry.terraform.io/alekc/kubectl" { version = "2.0.4" constraints = "~> 2.0" hashes = [ + "h1:1Ence3VDSQ7BNO+IFD6QoGBiBf6rJgCbygkATSdjcTA=", + "h1:6xRO3WlBsOTbeJ90QFjxGbc4BjnoGdEaeSCdWI/B1jU=", "h1:TUeUq1UdVkHTxcgq7CJWWXBrc8VEQTufmgU18qDmfGE=", + "h1:V9WCWj18ygdOE0h4AycrJd9MtKG1W3RzGP636LfCgEo=", + "h1:mCz0lOwNsFCZEcFf7DBSe6b4hZgn5piiy0mZDwRGUIU=", "zh:15c227886bac78c8b8827f85595648212574ec81febc39e1055e1a6bf048fe65", "zh:2211ebeeb0918dbb3587d206e32adca9e1f343a93bbffcd37d8d99bf4d8dea9a", "zh:2303836cdea12ece8dbe39c2d7d30a9378fd06e9c2ebda66cbe5e01cc096ee2e", @@ -27,7 +31,11 @@ provider "registry.terraform.io/hashicorp/azuread" { version = "2.47.0" constraints = "> 2.10.0" hashes = [ + "h1:8J74v92UvtqVNucugAtB+Sd44oTgnhfct+Xf8ObOZug=", + "h1:KB9BNRNStbdsfdRmVXUwXtN77qgX5VjBy2UALcqp218=", + "h1:g8+gBFM4QVOEQFqAEs5pR6iXpbGvgPvcEi1evHwziyw=", "h1:iRwDQBdXBpVBoYwM9au2RG01RQuJSm3TGQ2kioFVAas=", + "h1:zYMGokLn44KSWir7Nr4t8lEAPMB6JuXd2LlP2Ac2tMY=", "zh:1372d81eb24ef3b4b00ea350fe87219f22da51691b8e42ce91d662f6c2a8af5e", "zh:1c3e89cf19118fc07d7b04257251fc9897e722c16e0a0df7b07fcd261f8c12e7", "zh:1e654a74d171d6ff8f9f6f67e3ff1421d4c5e56a18607703626bf12cd23ba001", @@ -45,8 +53,12 @@ provider "registry.terraform.io/hashicorp/azuread" { provider "registry.terraform.io/hashicorp/azurerm" { version = "3.93.0" - constraints = ">= 3.30.0, >= 3.76.0, <= 3.93.0, <= 3.94.0" + constraints = ">= 3.30.0, ~> 3.30, <= 3.93.0, <= 3.94.0" hashes = [ + "h1:BYtgd2o7DTesxgB53hmhqzNkMEeYM355aCgECmnSc8k=", + "h1:SKhtBNyVOKexC8XTioruXEQYHFBF1n8yiP8LfRMl96M=", + "h1:SV0mvtJqXyRMw4pDhVeBGUOgmnJr6Klgh7hWa1xiuu8=", + "h1:bByD6VHJV+QU6JHEtkSh3inxqVMRgjxedFPe51C305c=", "h1:lUioDFuE2xonpcH5QP55F1WZu7oAOGbhXbSaQSBGIR8=", "zh:0fc3169c32e43e44a34308856ebf4209c8c13e2526993bb4d9dc81fedc91f60a", "zh:10ed7bc146480d4261c5a899032947c3908fa3661efe797128fb961c0a0a4eb6", @@ -68,6 +80,10 @@ provider "registry.terraform.io/hashicorp/helm" { constraints = ">= 2.0.0, <= 2.12.1" hashes = [ "h1:7wfYOAeSEchHB8idNl+2jf+OkFi9zFSOLWkEZFuTCik=", + "h1:aBfcqM4cbywa7TAxfT1YoFS+Cst9waerlm4XErFmJlk=", + "h1:sgYI7lwGqJqPopY3NGmhb1eQ0YbH8PIXaAZAmnJrAvw=", + "h1:sjzfyNQAjtF9zXHxB67geryjGkHaPDMMVw9iqPP5pkE=", + "h1:xwHVa6ab/XVfDrZ3h35OzLJ6g0Zte4VAvSnyKw3f9AI=", "zh:1d623fb1662703f2feb7860e3c795d849c77640eecbc5a776784d08807b15004", "zh:253a5bc62ba2c4314875139e3fbd2feaad5ef6b0fb420302a474ab49e8e51a38", "zh:282358f4ad4f20d0ccaab670b8645228bfad1c03ac0d0df5889f0aea8aeac01a", @@ -87,7 +103,11 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.25.2" constraints = "<= 2.25.2, <= 2.27.0" hashes = [ + "h1:+Yi+ho+dpgEmMz6Mt/9O/kDQw9HTcrLWkMyTBFN9yIE=", + "h1:QlTKoO0efmkzgX/9y0DQCEkg7VeidOSQW8epF6B4cEQ=", + "h1:T1WAQt40cAk721H0AM/eZ5YuodJaIfS8r3Tu7rKCJJE=", "h1:o/+UcYEaEHrQzq2kkWw2MohCK033u6vY+T6cmHd46QU=", + "h1:vrpxWZfnmJ7t9gDff1/z4h+UhewGBpDB52EIwhygn6A=", "zh:044788ac936e0e8ece8f78a2e4e366ecd435ea8235388eaf2cbc8e7975d9d970", "zh:24f5ff01df91f51f00ee7ff39430adeb63bb2ca4ea0042e68f06d6b65808c02f", "zh:49984aa0aa1faa8c4f01e8faa039322f1e6fdaeab0b7e32f5c6e96edfde36a38", @@ -107,6 +127,10 @@ provider "registry.terraform.io/hashicorp/local" { version = "2.5.1" constraints = ">= 2.4.0" hashes = [ + "h1:/GAVA/xheGQcbOZEq0qxANOg+KVLCA7Wv8qluxhTjhU=", + "h1:8oTPe2VUL6E2d3OcrvqyjI4Nn/Y/UEQN26WLk5O/B0g=", + "h1:Np4kERf9SMrqUi7DJ1rK3soMK14k49nfgE7l/ipQ5xw=", + "h1:fm2EuMlsdPTuv2tKwx3PMJzWJUh7aMtU9Eky7t4fMys=", "h1:tjcGlQAFA0kmQ4vKkIPPUC4it1UYxLbg4YvHOWRAJHA=", "zh:0af29ce2b7b5712319bf6424cb58d13b852bf9a777011a545fac99c7fdcdf561", "zh:126063ea0d79dad1f68fa4e4d556793c0108ce278034f101d1dbbb2463924561", @@ -127,7 +151,11 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.2.1" constraints = "<= 3.2.1" hashes = [ + "h1:FbGfc+muBsC17Ohy5g806iuI1hQc4SIexpYCrQHQd8w=", "h1:tSj1mL6OQ8ILGqR2mDu7OYYYWf+hoir0pf9KAQ8IzO8=", + "h1:vUW21lLLsKlxtBf0QF7LKJreKxs0CM7YXGzqW1N/ODY=", + "h1:wqgRvlyVIbkCeCQs+5jj6zVuQL0KDxZZtNofGqqlSdI=", + "h1:ydA0/SNRVB1o95btfshvYsmxA+jZFRZcvKzZSB+4S1M=", "zh:58ed64389620cc7b82f01332e27723856422820cfd302e304b5f6c3436fb9840", "zh:62a5cc82c3b2ddef7ef3a6f2fedb7b9b3deff4ab7b414938b08e51d6e8be87cb", "zh:63cff4de03af983175a7e37e52d4bd89d990be256b16b5c7f919aff5ad485aa5", @@ -142,22 +170,3 @@ provider "registry.terraform.io/hashicorp/null" { "zh:fca01a623d90d0cad0843102f9b8b9fe0d3ff8244593bd817f126582b52dd694", ] } - -provider "registry.terraform.io/hashicorp/random" { - version = "3.6.0" - hashes = [ - "h1:p6WG1IPHnqx1fnJVKNjv733FBaArIugqy58HRZnpPCk=", - "zh:03360ed3ecd31e8c5dac9c95fe0858be50f3e9a0d0c654b5e504109c2159287d", - "zh:1c67ac51254ba2a2bb53a25e8ae7e4d076103483f55f39b426ec55e47d1fe211", - "zh:24a17bba7f6d679538ff51b3a2f378cedadede97af8a1db7dad4fd8d6d50f829", - "zh:30ffb297ffd1633175d6545d37c2217e2cef9545a6e03946e514c59c0859b77d", - "zh:454ce4b3dbc73e6775f2f6605d45cee6e16c3872a2e66a2c97993d6e5cbd7055", - "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", - "zh:91df0a9fab329aff2ff4cf26797592eb7a3a90b4a0c04d64ce186654e0cc6e17", - "zh:aa57384b85622a9f7bfb5d4512ca88e61f22a9cea9f30febaa4c98c68ff0dc21", - "zh:c4a3e329ba786ffb6f2b694e1fd41d413a7010f3a53c20b432325a94fa71e839", - "zh:e2699bc9116447f96c53d55f2a00570f982e6f9935038c3810603572693712d0", - "zh:e747c0fd5d7684e5bfad8aa0ca441903f15ae7a98a737ff6aca24ba223207e2c", - "zh:f1ca75f417ce490368f047b63ec09fd003711ae48487fba90b4aba2ccf71920e", - ] -} diff --git a/src/aks-platform/00_key_vault.tf b/src/aks-platform/00_key_vault.tf index 25213f2..96f924b 100644 --- a/src/aks-platform/00_key_vault.tf +++ b/src/aks-platform/00_key_vault.tf @@ -1,4 +1,4 @@ -data "azurerm_key_vault" "kv_core" { - name = "dvopla-d-neu-kv" - resource_group_name = "dvopla-d-sec-rg" +data "azurerm_key_vault" "kv_core_ita" { + name = "dvopla-d-itn-core-kv" + resource_group_name = "dvopla-d-itn-sec-rg" } diff --git a/src/aks-platform/00_network.tf b/src/aks-platform/00_network.tf index 5cf1a87..06ab051 100644 --- a/src/aks-platform/00_network.tf +++ b/src/aks-platform/00_network.tf @@ -34,25 +34,25 @@ data "azurerm_public_ip" "pip_aks_outboud" { # # Subnet # -data "azurerm_subnet" "private_endpoint_subnet" { - name = "${local.product}-private-endpoints-snet" - resource_group_name = data.azurerm_resource_group.vnet_core_rg.name - virtual_network_name = data.azurerm_virtual_network.vnet_core.name -} +# data "azurerm_subnet" "private_endpoint_subnet" { +# name = "${local.product}-private-endpoints-snet" +# resource_group_name = data.azurerm_resource_group.vnet_core_rg.name +# virtual_network_name = data.azurerm_virtual_network.vnet_core.name +# } -data "azurerm_subnet" "private_endpoint_italy_subnet" { - name = "${local.product}-private-endpoints-italy-snet" - resource_group_name = data.azurerm_resource_group.vnet_italy_rg.name - virtual_network_name = data.azurerm_virtual_network.vnet_italy.name -} +# data "azurerm_subnet" "private_endpoint_italy_subnet" { +# name = "${local.product}-private-endpoints-italy-snet" +# resource_group_name = data.azurerm_resource_group.vnet_italy_rg.name +# virtual_network_name = data.azurerm_virtual_network.vnet_italy.name +# } # # Dns # -data "azurerm_private_dns_zone" "storage_account_private_dns_zone" { - name = "privatelink.blob.core.windows.net" - resource_group_name = data.azurerm_resource_group.vnet_core_rg.name -} +# data "azurerm_private_dns_zone" "storage_account_private_dns_zone" { +# name = "privatelink.blob.core.windows.net" +# resource_group_name = data.azurerm_resource_group.vnet_core_rg.name +# } data "azurerm_private_dns_zone" "internal" { name = local.internal_dns_zone_name diff --git a/src/aks-platform/01_network_aks.tf b/src/aks-platform/01_network_aks.tf index 71a43aa..d731537 100644 --- a/src/aks-platform/01_network_aks.tf +++ b/src/aks-platform/01_network_aks.tf @@ -1,17 +1,18 @@ -# k8s cluster subnet -module "snet_aks" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet?ref=v7.69.1" - - name = "${local.project}-aks-snet" +resource "azurerm_subnet" "system_aks_subnet" { + name = "${local.project}-system-aks" + resource_group_name = data.azurerm_resource_group.vnet_italy_rg.name + virtual_network_name = data.azurerm_virtual_network.vnet_italy.name + address_prefixes = var.cidr_subnet_system_aks + private_endpoint_network_policies_enabled = true + private_link_service_network_policies_enabled = true +} +resource "azurerm_subnet" "user_aks_subnet" { + name = "${local.project}-user-aks" resource_group_name = data.azurerm_resource_group.vnet_italy_rg.name virtual_network_name = data.azurerm_virtual_network.vnet_italy.name + address_prefixes = var.cidr_subnet_user_aks - address_prefixes = var.cidr_subnet_aks - private_endpoint_network_policies_enabled = var.aks_private_cluster_enabled - - service_endpoints = [ - "Microsoft.Web", - "Microsoft.Storage" - ] + private_endpoint_network_policies_enabled = true + private_link_service_network_policies_enabled = true } diff --git a/src/aks-platform/02_aks_0.tf b/src/aks-platform/02_aks_0.tf index d84bc3d..27b723f 100644 --- a/src/aks-platform/02_aks_0.tf +++ b/src/aks-platform/02_aks_0.tf @@ -14,9 +14,7 @@ resource "azurerm_resource_group" "rg_aks_backup" { module "aks" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v7.70.1" - - count = var.aks_enabled ? 1 : 0 + source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster?ref=v8.14.0" name = local.aks_cluster_name resource_group_name = azurerm_resource_group.rg_aks.name @@ -40,47 +38,31 @@ module "aks" { system_node_pool_node_labels = var.aks_system_node_pool.node_labels system_node_pool_tags = var.aks_system_node_pool.node_tags - # - # 👤 User node pool - # - user_node_pool_enabled = var.aks_user_node_pool.enabled - user_node_pool_name = var.aks_user_node_pool.name - ### vm configuration - user_node_pool_vm_size = var.aks_user_node_pool.vm_size - user_node_pool_os_disk_type = var.aks_user_node_pool.os_disk_type - user_node_pool_os_disk_size_gb = var.aks_user_node_pool.os_disk_size_gb - user_node_pool_node_count_min = var.aks_user_node_pool.node_count_min - user_node_pool_node_count_max = var.aks_user_node_pool.node_count_max - ### K8s node configuration - user_node_pool_node_labels = var.aks_user_node_pool.node_labels - user_node_pool_node_taints = var.aks_user_node_pool.node_taints - user_node_pool_tags = var.aks_user_node_pool.node_tags - # end user node pool - # # ☁️ Network # vnet_id = data.azurerm_virtual_network.vnet_italy.id - vnet_subnet_id = module.snet_aks.id + vnet_subnet_id = azurerm_subnet.system_aks_subnet.id outbound_ip_address_ids = [data.azurerm_public_ip.pip_aks_outboud.id] private_cluster_enabled = var.aks_private_cluster_enabled network_profile = { - docker_bridge_cidr = "172.17.0.1/16" - dns_service_ip = "10.250.0.10" - network_plugin = "azure" - network_policy = "azure" - outbound_type = "loadBalancer" - service_cidr = "10.250.0.0/16" + docker_bridge_cidr = "172.17.0.1/16" + dns_service_ip = "10.0.0.10" + network_plugin = "azure" + network_plugin_mode = "overlay" + network_policy = "azure" + outbound_type = "loadBalancer" + service_cidr = "10.0.0.0/16" } # end network oidc_issuer_enabled = true aad_admin_group_ids = var.env_short == "d" ? [data.azuread_group.adgroup_admin.object_id, data.azuread_group.adgroup_developers.object_id, data.azuread_group.adgroup_externals.object_id] : [data.azuread_group.adgroup_admin.object_id] - addon_azure_policy_enabled = var.aks_addons.azure_policy - addon_azure_key_vault_secrets_provider_enabled = var.aks_addons.azure_key_vault_secrets_provider - addon_azure_pod_identity_enabled = var.aks_addons.pod_identity_enabled + addon_azure_policy_enabled = true + addon_azure_key_vault_secrets_provider_enabled = true + addon_azure_pod_identity_enabled = true default_metric_alerts = var.aks_metric_alerts_default custom_metric_alerts = var.aks_metric_alerts_custom @@ -101,18 +83,57 @@ module "aks" { ] tags = var.tags +} - depends_on = [ - module.snet_aks, - data.azurerm_public_ip.pip_aks_outboud, - data.azurerm_virtual_network.vnet_italy - ] +resource "azurerm_kubernetes_cluster_node_pool" "user_nodepool_default" { + count = var.aks_user_node_pool.enabled ? 1 : 0 + + kubernetes_cluster_id = module.aks.id + + name = var.aks_user_node_pool.name + + ### vm configuration + vm_size = var.aks_user_node_pool.vm_size + # https://docs.microsoft.com/en-us/azure/virtual-machines/sizes-general + os_disk_type = var.aks_user_node_pool.os_disk_type # Managed or Ephemeral + os_disk_size_gb = var.aks_user_node_pool.os_disk_size_gb + zones = var.aks_user_node_pool.zones + ultra_ssd_enabled = var.aks_user_node_pool.ultra_ssd_enabled + enable_host_encryption = var.aks_user_node_pool.enable_host_encryption + os_type = "Linux" + + ### autoscaling + enable_auto_scaling = true + node_count = var.aks_user_node_pool.node_count_min + min_count = var.aks_user_node_pool.node_count_min + max_count = var.aks_user_node_pool.node_count_max + + ### K8s node configuration + max_pods = var.aks_user_node_pool.max_pods + node_labels = var.aks_user_node_pool.node_labels + node_taints = var.aks_user_node_pool.node_taints + + ### networking + vnet_subnet_id = azurerm_subnet.user_aks_subnet.id + enable_node_public_ip = false + + upgrade_settings { + max_surge = var.aks_user_node_pool.upgrade_settings_max_surge + } + + tags = merge(var.tags, var.aks_user_node_pool.node_tags) + + lifecycle { + ignore_changes = [ + node_count + ] + } } resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" { count = var.aks_spot_user_node_pool.enabled ? 1 : 0 - kubernetes_cluster_id = module.aks[0].id + kubernetes_cluster_id = module.aks.id name = var.aks_spot_user_node_pool.name @@ -140,7 +161,7 @@ resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" { node_taints = var.aks_spot_user_node_pool.node_taints ### networking - vnet_subnet_id = module.snet_aks.id + vnet_subnet_id = azurerm_subnet.user_aks_subnet.id enable_node_public_ip = false tags = merge(var.tags, var.aks_spot_user_node_pool.node_tags) @@ -150,13 +171,15 @@ resource "azurerm_kubernetes_cluster_node_pool" "spot_node_pool" { node_count ] } + + depends_on = [module.aks] } resource "azurerm_role_assignment" "managed_identity_operator_vs_aks_managed_identity" { scope = azurerm_resource_group.rg_aks.id role_definition_name = "Managed Identity Operator" - principal_id = module.aks[0].identity_principal_id + principal_id = module.aks.identity_principal_id } # @@ -166,7 +189,7 @@ resource "azurerm_role_assignment" "managed_identity_operator_vs_aks_managed_ide resource "azurerm_role_assignment" "aks_to_acr" { scope = data.azurerm_container_registry.acr.id role_definition_name = "AcrPull" - principal_id = module.aks[0].kubelet_identity_id + principal_id = module.aks.kubelet_identity_id depends_on = [module.aks] } @@ -181,7 +204,7 @@ resource "null_resource" "create_vnet_core_aks_link" { count = var.aks_enabled && var.aks_private_cluster_enabled ? 1 : 0 triggers = { - cluster_name = module.aks[0].name + cluster_name = module.aks.name vnet_id = data.azurerm_virtual_network.vnet_core.id vnet_name = data.azurerm_virtual_network.vnet_core.name } diff --git a/src/aks-platform/02_aks_storage.tf b/src/aks-platform/02_aks_storage.tf index ec41a25..1eb21a8 100644 --- a/src/aks-platform/02_aks_storage.tf +++ b/src/aks-platform/02_aks_storage.tf @@ -1,3 +1,5 @@ module "aks_storage_class" { source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_storage_class?ref=v7.69.1" + + depends_on = [module.aks] } diff --git a/src/aks-platform/03_monitoring.tf b/src/aks-platform/03_monitoring.tf index 53f06cb..f11ef87 100644 --- a/src/aks-platform/03_monitoring.tf +++ b/src/aks-platform/03_monitoring.tf @@ -9,4 +9,6 @@ module "aks_prometheus_install" { source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_prometheus_install?ref=v7.69.1" prometheus_namespace = kubernetes_namespace.monitoring.metadata[0].name storage_class_name = "default-zrs" + + depends_on = [module.aks_storage_class] } diff --git a/src/aks-platform/05_argocd.tf b/src/aks-platform/05_argocd.tf index aeb18c3..f679b88 100644 --- a/src/aks-platform/05_argocd.tf +++ b/src/aks-platform/05_argocd.tf @@ -1,103 +1,103 @@ -resource "kubernetes_namespace" "namespace_argocd" { - metadata { - name = "argocd" - } - - depends_on = [ - module.aks - ] -} - -resource "helm_release" "argocd" { - name = "argo" - chart = "https://github.com/argoproj/argo-helm/releases/download/argo-cd-6.7.11/argo-cd-6.7.11.tgz" - namespace = kubernetes_namespace.namespace_argocd.metadata[0].name - wait = false - - values = [ - file("argocd/argocd_helm_setup_values.yaml") - ] - - depends_on = [ - module.aks - ] -} - -resource "random_password" "argocd_admin_password" { - length = 12 - special = true - override_special = "_%@" - - depends_on = [helm_release.argocd] -} - -resource "null_resource" "argocd_change_admin_password" { - - triggers = { - helm_revision = helm_release.argocd.metadata[0].revision, - argocd_password = random_password.argocd_admin_password.result - } - - provisioner "local-exec" { - command = "kubectl -n argocd patch secret argocd-secret -p '{\"stringData\": {\"admin.password\": \"${bcrypt(random_password.argocd_admin_password.result)}\", \"admin.passwordMtime\": \"'$(date +%FT%T%Z)'\"}}'" - } -} - -resource "azurerm_key_vault_secret" "argocd_admin_password" { - key_vault_id = data.azurerm_key_vault.kv_core.id - name = "argocd-admin-password" - value = random_password.argocd_admin_password.result -} - -# -# tools -# - -module "argocd_pod_identity" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v7.69.1" - - cluster_name = module.aks[0].name - resource_group_name = azurerm_resource_group.rg_aks.name - location = var.location - tenant_id = data.azurerm_subscription.current.tenant_id - - identity_name = "argocd-pod-identity" - namespace = kubernetes_namespace.namespace_argocd.metadata[0].name - key_vault_id = data.azurerm_key_vault.kv_core.id - - secret_permissions = ["Get"] - certificate_permissions = ["Get"] -} - -resource "helm_release" "reloader_argocd" { - name = "reloader" - repository = "https://stakater.github.io/stakater-charts" - chart = "reloader" - version = "v1.0.30" - namespace = kubernetes_namespace.namespace_argocd.metadata[0].name - - set { - name = "reloader.watchGlobally" - value = "false" - } -} - -module "cert_mounter_argocd_internal" { - source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//cert_mounter?ref=v7.69.1" - namespace = "argocd" - certificate_name = replace(local.argocd_internal_url, ".", "-") - kv_name = data.azurerm_key_vault.kv_core.name - tenant_id = data.azurerm_subscription.current.tenant_id - - depends_on = [ - module.argocd_pod_identity - ] -} - -resource "azurerm_private_dns_a_record" "argocd_ingress" { - name = local.ingress_hostname_prefix - zone_name = data.azurerm_private_dns_zone.internal.name - resource_group_name = local.internal_dns_zone_resource_group_name - ttl = 3600 - records = [var.ingress_load_balancer_ip] -} +# resource "kubernetes_namespace" "namespace_argocd" { +# metadata { +# name = "argocd" +# } +# +# depends_on = [ +# module.aks +# ] +# } +# +# resource "helm_release" "argocd" { +# name = "argo" +# chart = "https://github.com/argoproj/argo-helm/releases/download/argo-cd-6.7.11/argo-cd-6.7.11.tgz" +# namespace = kubernetes_namespace.namespace_argocd.metadata[0].name +# wait = false +# +# values = [ +# file("argocd/argocd_helm_setup_values.yaml") +# ] +# +# depends_on = [ +# module.aks +# ] +# } +# +# resource "random_password" "argocd_admin_password" { +# length = 12 +# special = true +# override_special = "_%@" +# +# depends_on = [helm_release.argocd] +# } +# +# resource "null_resource" "argocd_change_admin_password" { +# +# triggers = { +# helm_revision = helm_release.argocd.metadata[0].revision, +# argocd_password = random_password.argocd_admin_password.result +# } +# +# provisioner "local-exec" { +# command = "kubectl -n argocd patch secret argocd-secret -p '{\"stringData\": {\"admin.password\": \"${bcrypt(random_password.argocd_admin_password.result)}\", \"admin.passwordMtime\": \"'$(date +%FT%T%Z)'\"}}'" +# } +# } +# +# resource "azurerm_key_vault_secret" "argocd_admin_password" { +# key_vault_id = data.azurerm_key_vault.kv_core_ita.id +# name = "argocd-admin-password" +# value = random_password.argocd_admin_password.result +# } +# +# # +# # tools +# # +# +# module "argocd_pod_identity" { +# source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity?ref=v7.69.1" +# +# cluster_name = module.aks.name +# resource_group_name = azurerm_resource_group.rg_aks.name +# location = var.location +# tenant_id = data.azurerm_subscription.current.tenant_id +# +# identity_name = "argocd-pod-identity" +# namespace = kubernetes_namespace.namespace_argocd.metadata[0].name +# key_vault_id = data.azurerm_key_vault.kv_core_ita.id +# +# secret_permissions = ["Get"] +# certificate_permissions = ["Get"] +# } +# +# resource "helm_release" "reloader_argocd" { +# name = "reloader" +# repository = "https://stakater.github.io/stakater-charts" +# chart = "reloader" +# version = "v1.0.30" +# namespace = kubernetes_namespace.namespace_argocd.metadata[0].name +# +# set { +# name = "reloader.watchGlobally" +# value = "false" +# } +# } +# +# module "cert_mounter_argocd_internal" { +# source = "git::https://github.com/pagopa/terraform-azurerm-v3.git//cert_mounter?ref=v7.69.1" +# namespace = "argocd" +# certificate_name = replace(local.argocd_internal_url, ".", "-") +# kv_name = data.azurerm_key_vault.kv_core_ita.name +# tenant_id = data.azurerm_subscription.current.tenant_id +# +# depends_on = [ +# module.argocd_pod_identity +# ] +# } +# +# resource "azurerm_private_dns_a_record" "argocd_ingress" { +# name = local.ingress_hostname_prefix +# zone_name = data.azurerm_private_dns_zone.internal.name +# resource_group_name = local.internal_dns_zone_resource_group_name +# ttl = 3600 +# records = [var.ingress_load_balancer_ip] +# } diff --git a/src/aks-platform/05_ingress.tf b/src/aks-platform/05_ingress.tf index 8a3cf7e..fc7d744 100644 --- a/src/aks-platform/05_ingress.tf +++ b/src/aks-platform/05_ingress.tf @@ -27,7 +27,7 @@ module "nginx_ingress" { values = [ templatefile("${path.module}/ingress/loadbalancer.yaml.tpl", { load_balancer_ip = var.ingress_load_balancer_ip - private_subnet_name = module.snet_aks.name + private_subnet_name = azurerm_subnet.user_aks_subnet.name }) ] diff --git a/src/aks-platform/05_keda.tf b/src/aks-platform/05_keda.tf index eb0c89d..7c1c73d 100644 --- a/src/aks-platform/05_keda.tf +++ b/src/aks-platform/05_keda.tf @@ -21,7 +21,7 @@ module "keda_pod_identity" { identity_name = "${local.keda_namespace_name}-pod-identity" tenant_id = data.azurerm_subscription.current.tenant_id - cluster_name = module.aks[0].name + cluster_name = module.aks.name namespace = kubernetes_namespace.keda.metadata[0].name depends_on = [ diff --git a/src/aks-platform/99_locals.tf b/src/aks-platform/99_locals.tf index 784819c..deb4bb4 100644 --- a/src/aks-platform/99_locals.tf +++ b/src/aks-platform/99_locals.tf @@ -1,6 +1,7 @@ locals { - product = "${var.prefix}-${var.env_short}" - project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}" + product = "${var.prefix}-${var.env_short}" + product_ita = "${var.prefix}-${var.env_short}-${var.location_short}" + project = "${var.prefix}-${var.env_short}-${var.location_short}-${var.env}" # AKS aks_rg_name = "${local.project}-aks-rg" @@ -15,18 +16,17 @@ locals { ingress_hostname_prefix = "argocd" internal_dns_zone_name = "${var.dns_zone_internal_prefix}.${var.external_domain}" - internal_dns_zone_resource_group_name = "${local.product}-vnet-rg" + internal_dns_zone_resource_group_name = "${local.product_ita}-vnet-rg" # ACR DOCKER - docker_rg_name = "dvopla-d-dockerreg-rg" - docker_registry_name = "dvopladneuacr" + docker_rg_name = "dvopla-d-docker-registry-rg" + docker_registry_name = "dvopladitnacr" # monitor - monitor_rg_name = "${local.product}-monitor-rg" - monitor_log_analytics_workspace_name = "${local.product}-law" - monitor_log_analytics_workspace_prometheus_name = "${local.product}-prometheus-law" - monitor_appinsights_name = "${local.product}-appinsights" - monitor_security_storage_name = replace("${local.product}-sec-monitor-st", "-", "") + monitor_rg_name = "${local.product_ita}-monitor-rg" + monitor_log_analytics_workspace_name = "${local.product_ita}-law" + monitor_appinsights_name = "${local.product_ita}-appinsights" + monitor_security_storage_name = replace("${local.product}-sec-monitor-st", "-", "") monitor_action_group_slack_name = "SlackPagoPA" monitor_action_group_email_name = "PagoPA" diff --git a/src/aks-platform/99_variables.tf b/src/aks-platform/99_variables.tf index 376ba86..f564d95 100644 --- a/src/aks-platform/99_variables.tf +++ b/src/aks-platform/99_variables.tf @@ -83,7 +83,12 @@ variable "public_ip_aksoutbound_name" { description = "Public IP AKS outbound" } -variable "cidr_subnet_aks" { +variable "cidr_subnet_system_aks" { + type = list(string) + description = "Subnet cluster kubernetes." +} + +variable "cidr_subnet_user_aks" { type = list(string) description = "Subnet cluster kubernetes." } @@ -445,30 +450,37 @@ variable "aks_alerts_enabled" { variable "aks_system_node_pool" { type = object({ - name = string, - vm_size = string, - os_disk_type = string, - os_disk_size_gb = string, - node_count_min = number, - node_count_max = number, - node_labels = map(any), - node_tags = map(any) + name = string, + vm_size = string, + os_disk_type = string, + os_disk_size_gb = string, + node_count_min = number, + node_count_max = number, + node_labels = map(any), + node_tags = map(any), + only_critical_addons_enabled = optional(bool, true) + zones = optional(list(any), [1, 2, 3]) }) description = "AKS node pool system configuration" } variable "aks_user_node_pool" { type = object({ - enabled = bool, - name = string, - vm_size = string, - os_disk_type = string, - os_disk_size_gb = string, - node_count_min = number, - node_count_max = number, - node_labels = map(any), - node_taints = list(string), - node_tags = map(any), + enabled = optional(bool, true), + name = string, + vm_size = string, + os_disk_type = string, + os_disk_size_gb = string, + node_count_min = number, + node_count_max = number, + node_labels = map(any), + node_taints = list(string), + node_tags = map(any), + ultra_ssd_enabled = optional(bool, false), + enable_host_encryption = optional(bool, true), + max_pods = optional(number, 250), + upgrade_settings_max_surge = optional(string, "30%"), + zones = optional(list(any), [1, 2, 3]), }) description = "AKS node pool user configuration" } diff --git a/src/aks-platform/README.md b/src/aks-platform/README.md index a897525..ca351a2 100644 --- a/src/aks-platform/README.md +++ b/src/aks-platform/README.md @@ -40,30 +40,26 @@ Re-enable all the resource, commented before to complete the procedure | Name | Source | Version | |------|--------|---------| -| [aks](#module\_aks) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster | v7.70.1 | +| [aks](#module\_aks) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_cluster | v8.14.0 | | [aks\_prometheus\_install](#module\_aks\_prometheus\_install) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_prometheus_install | v7.69.1 | | [aks\_storage\_class](#module\_aks\_storage\_class) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_storage_class | v7.69.1 | -| [argocd\_pod\_identity](#module\_argocd\_pod\_identity) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity | v7.69.1 | -| [cert\_mounter\_argocd\_internal](#module\_cert\_mounter\_argocd\_internal) | git::https://github.com/pagopa/terraform-azurerm-v3.git//cert_mounter | v7.69.1 | | [keda\_pod\_identity](#module\_keda\_pod\_identity) | git::https://github.com/pagopa/terraform-azurerm-v3.git//kubernetes_pod_identity | v7.69.1 | | [nginx\_ingress](#module\_nginx\_ingress) | terraform-module/release/helm | 2.7.0 | -| [snet\_aks](#module\_snet\_aks) | git::https://github.com/pagopa/terraform-azurerm-v3.git//subnet | v7.69.1 | ## Resources | Name | Type | |------|------| -| [azurerm_key_vault_secret.argocd_admin_password](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/key_vault_secret) | resource | | [azurerm_kubernetes_cluster_node_pool.spot_node_pool](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | -| [azurerm_private_dns_a_record.argocd_ingress](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/private_dns_a_record) | resource | +| [azurerm_kubernetes_cluster_node_pool.user_nodepool_default](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/kubernetes_cluster_node_pool) | resource | | [azurerm_resource_group.rg_aks](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_resource_group.rg_aks_backup](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/resource_group) | resource | | [azurerm_role_assignment.aks_to_acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.keda_monitoring_reader](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | | [azurerm_role_assignment.managed_identity_operator_vs_aks_managed_identity](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_assignment) | resource | -| [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [azurerm_subnet.system_aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | +| [azurerm_subnet.user_aks_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/subnet) | resource | | [helm_release.keda](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | -| [helm_release.reloader_argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_cluster_role.cluster_deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | | [kubernetes_cluster_role.edit_extra](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | | [kubernetes_cluster_role.system_cluster_deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role) | resource | @@ -75,10 +71,7 @@ Re-enable all the resource, commented before to complete the procedure | [kubernetes_namespace.ingress](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | | [kubernetes_namespace.keda](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | | [kubernetes_namespace.monitoring](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [kubernetes_namespace.namespace_argocd](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [null_resource.argocd_change_admin_password](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [null_resource.create_vnet_core_aks_link](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [random_password.argocd_admin_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource | | [azuread_group.adgroup_admin](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | | [azuread_group.adgroup_developers](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | | [azuread_group.adgroup_externals](https://registry.terraform.io/providers/hashicorp/azuread/latest/docs/data-sources/group) | data source | @@ -88,18 +81,15 @@ Re-enable all the resource, commented before to complete the procedure | [azurerm_application_insights.application_insights](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/application_insights) | data source | | [azurerm_client_config.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/client_config) | data source | | [azurerm_container_registry.acr](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/container_registry) | data source | -| [azurerm_key_vault.kv_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | +| [azurerm_key_vault.kv_core_ita](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/key_vault) | data source | | [azurerm_log_analytics_workspace.log_analytics_workspace](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/log_analytics_workspace) | data source | | [azurerm_monitor_action_group.email](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_monitor_action_group.slack](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/monitor_action_group) | data source | | [azurerm_private_dns_zone.internal](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | -| [azurerm_private_dns_zone.storage_account_private_dns_zone](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/private_dns_zone) | data source | | [azurerm_public_ip.pip_aks_outboud](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/public_ip) | data source | | [azurerm_resource_group.rg_monitor](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.vnet_core_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | | [azurerm_resource_group.vnet_italy_rg](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/resource_group) | data source | -| [azurerm_subnet.private_endpoint_italy_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | -| [azurerm_subnet.private_endpoint_subnet](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subnet) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/subscription) | data source | | [azurerm_virtual_network.vnet_core](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | | [azurerm_virtual_network.vnet_italy](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/virtual_network) | data source | @@ -125,10 +115,11 @@ Re-enable all the resource, commented before to complete the procedure | [aks\_reverse\_proxy\_ip](#input\_aks\_reverse\_proxy\_ip) | AKS external ip. Also the ingress-nginx-controller external ip. Value known after installing the ingress controller. | `string` | `"127.0.0.1"` | no | | [aks\_sku\_tier](#input\_aks\_sku\_tier) | The SKU Tier that should be used for this Kubernetes Cluster. | `string` | `"Free"` | no | | [aks\_spot\_user\_node\_pool](#input\_aks\_spot\_user\_node\_pool) | AKS node pool user configuration |
object({
enabled = bool,
name = string,
vm_size = string,
os_disk_type = string,
os_disk_size_gb = string,
node_count_min = number,
node_count_max = number,
node_labels = map(any),
node_taints = list(string),
node_tags = map(any),
})
| n/a | yes | -| [aks\_system\_node\_pool](#input\_aks\_system\_node\_pool) | AKS node pool system configuration |
object({
name = string,
vm_size = string,
os_disk_type = string,
os_disk_size_gb = string,
node_count_min = number,
node_count_max = number,
node_labels = map(any),
node_tags = map(any)
})
| n/a | yes | -| [aks\_user\_node\_pool](#input\_aks\_user\_node\_pool) | AKS node pool user configuration |
object({
enabled = bool,
name = string,
vm_size = string,
os_disk_type = string,
os_disk_size_gb = string,
node_count_min = number,
node_count_max = number,
node_labels = map(any),
node_taints = list(string),
node_tags = map(any),
})
| n/a | yes | +| [aks\_system\_node\_pool](#input\_aks\_system\_node\_pool) | AKS node pool system configuration |
object({
name = string,
vm_size = string,
os_disk_type = string,
os_disk_size_gb = string,
node_count_min = number,
node_count_max = number,
node_labels = map(any),
node_tags = map(any),
only_critical_addons_enabled = optional(bool, true)
zones = optional(list(any), [1, 2, 3])
})
| n/a | yes | +| [aks\_user\_node\_pool](#input\_aks\_user\_node\_pool) | AKS node pool user configuration |
object({
enabled = optional(bool, true),
name = string,
vm_size = string,
os_disk_type = string,
os_disk_size_gb = string,
node_count_min = number,
node_count_max = number,
node_labels = map(any),
node_taints = list(string),
node_tags = map(any),
ultra_ssd_enabled = optional(bool, false),
enable_host_encryption = optional(bool, true),
max_pods = optional(number, 250),
upgrade_settings_max_surge = optional(string, "30%"),
zones = optional(list(any), [1, 2, 3]),
})
| n/a | yes | | [aks\_vm\_size](#input\_aks\_vm\_size) | The size of the AKS Virtual Machine in the Node Pool. | `string` | `"Standard_DS3_v2"` | no | -| [cidr\_subnet\_aks](#input\_cidr\_subnet\_aks) | Subnet cluster kubernetes. | `list(string)` | n/a | yes | +| [cidr\_subnet\_system\_aks](#input\_cidr\_subnet\_system\_aks) | Subnet cluster kubernetes. | `list(string)` | n/a | yes | +| [cidr\_subnet\_user\_aks](#input\_cidr\_subnet\_user\_aks) | Subnet cluster kubernetes. | `list(string)` | n/a | yes | | [default\_service\_port](#input\_default\_service\_port) | n/a | `number` | `8080` | no | | [dns\_zone\_internal\_prefix](#input\_dns\_zone\_internal\_prefix) | The dns subdomain. | `string` | `null` | no | | [domain](#input\_domain) | n/a | `string` | n/a | yes | diff --git a/src/aks-platform/env/itn-dev/terraform.tfvars b/src/aks-platform/env/itn-dev/terraform.tfvars index 4f2dbce..a40970e 100644 --- a/src/aks-platform/env/itn-dev/terraform.tfvars +++ b/src/aks-platform/env/itn-dev/terraform.tfvars @@ -28,13 +28,14 @@ key_vault_rg_name = "dvopla-d-sec-rg" ### Network -cidr_subnet_aks = ["10.3.0.0/23"] +cidr_subnet_system_aks = ["10.3.9.0/24"] +cidr_subnet_user_aks = ["10.3.10.0/24"] ### External resources -monitor_resource_group_name = "dvopla-d-monitor-rg" -log_analytics_workspace_name = "dvopla-d-law" -log_analytics_workspace_resource_group_name = "dvopla-d-monitor-rg" +monitor_resource_group_name = "dvopla-d-itn-monitor-rg" +log_analytics_workspace_name = "dvopla-d-itn-law" +log_analytics_workspace_resource_group_name = "dvopla-d-itn-monitor-rg" ### Aks @@ -118,7 +119,7 @@ aks_addons = { ingress_replica_count = "1" # This is the k8s ingress controller ip. It must be in the aks subnet range. -ingress_load_balancer_ip = "10.3.1.250" +ingress_load_balancer_ip = "10.3.10.250" nginx_helm_version = "4.10.0" keda_helm_version = "2.12.1"