Merge pull request #84 from JDBraun/aws-jd

Update tags, add isolation features, new UC data sources, CMK for UC, cluster update, and other bug fixes

aws-gov/tf/modules/sra/cmk.tf

@@ -63,7 +63,8 @@ resource "aws_kms_key" "workspace_storage" {
   depends_on = [aws_iam_role.cross_account_role]
 
   tags = {
-    Resource = var.resource_prefix
+    Name    = "${var.resource_prefix}-workspace-storage-key"
+    Project = var.resource_prefix
   }
 }
 
@@ -111,7 +112,8 @@ resource "aws_kms_key" "managed_storage" {
   )
 
   tags = {
-    Resource = var.resource_prefix
+    Project = var.resource_prefix
+    Name    = "${var.resource_prefix}-managed-storage-key"
   }
 }
 

aws-gov/tf/modules/sra/credential.tf

@@ -19,10 +19,11 @@ data "aws_iam_policy_document" "passrole_for_cross_account_credential" {
 
 // Cross Account Role
 resource "aws_iam_role" "cross_account_role" {
-  name               = "${var.resource_prefix}-crossaccount"
+  name               = "${var.resource_prefix}-cross-account"
   assume_role_policy = data.aws_iam_policy_document.passrole_for_cross_account_credential.json
   tags = {
-    Name = "${var.resource_prefix}-crossaccount-role"
+    Name    = "${var.resource_prefix}-cross-account"
+    Project = var.resource_prefix
   }
 }
 

aws-gov/tf/modules/sra/data_plane_hardening.tf

@@ -8,18 +8,17 @@ module "harden_firewall" {
     aws = aws
   }
 
-  vpc_id                      = module.vpc[0].vpc_id
-  vpc_cidr_range              = var.vpc_cidr_range
-  public_subnets_cidr         = var.public_subnets_cidr
-  private_subnets_cidr        = module.vpc[0].private_subnets_cidr_blocks
-  private_subnet_rt           = module.vpc[0].private_route_table_ids
-  firewall_subnets_cidr       = var.firewall_subnets_cidr
-  firewall_allow_list         = var.firewall_allow_list
-  firewall_protocol_deny_list = split(",", var.firewall_protocol_deny_list)
-  hive_metastore_fqdn         = var.hive_metastore_fqdn
-  availability_zones          = var.availability_zones
-  region                      = var.region
-  resource_prefix             = var.resource_prefix
+  vpc_id                = module.vpc[0].vpc_id
+  vpc_cidr_range        = var.vpc_cidr_range
+  public_subnets_cidr   = var.public_subnets_cidr
+  private_subnets_cidr  = module.vpc[0].private_subnets_cidr_blocks
+  private_subnet_rt     = module.vpc[0].private_route_table_ids
+  firewall_subnets_cidr = var.firewall_subnets_cidr
+  firewall_allow_list   = var.firewall_allow_list
+  hive_metastore_fqdn   = var.hms_fqdn[var.databricks_gov_shard]
+  availability_zones    = var.availability_zones
+  region                = var.region
+  resource_prefix       = var.resource_prefix
 
   depends_on = [module.databricks_mws_workspace]
 }

aws-gov/tf/modules/sra/data_plane_hardening/firewall/firewall.tf

@@ -8,7 +8,8 @@ resource "aws_subnet" "public" {
   availability_zone       = element(var.availability_zones, count.index)
   map_public_ip_on_launch = true
   tags = {
-    Name = "${var.resource_prefix}-public-${element(var.availability_zones, count.index)}"
+    Name    = "${var.resource_prefix}-public-${element(var.availability_zones, count.index)}"
+    Project = var.resource_prefix
   }
 }
 
@@ -25,7 +26,8 @@ resource "aws_nat_gateway" "ngw" {
   subnet_id     = element(aws_subnet.public.*.id, count.index)
   depends_on    = [aws_internet_gateway.igw]
   tags = {
-    Name = "${var.resource_prefix}-ngw-${element(var.availability_zones, count.index)}"
+    Name    = "${var.resource_prefix}-ngw-${element(var.availability_zones, count.index)}"
+    Project = var.resource_prefix
   }
 }
 
@@ -37,13 +39,13 @@ resource "aws_route" "private" {
   nat_gateway_id         = element(aws_nat_gateway.ngw.*.id, count.index)
 }
 
-
 // Public RT
 resource "aws_route_table" "public_rt" {
   count  = length(var.public_subnets_cidr)
   vpc_id = var.vpc_id
   tags = {
-    Name = "${var.resource_prefix}-public-rt-${element(var.availability_zones, count.index)}"
+    Name    = "${var.resource_prefix}-public-rt-${element(var.availability_zones, count.index)}"
+    Project = var.resource_prefix
   }
 }
 
@@ -63,7 +65,8 @@ resource "aws_subnet" "firewall" {
   availability_zone       = element(var.availability_zones, count.index)
   map_public_ip_on_launch = false
   tags = {
-    Name = "${var.resource_prefix}-firewall-${element(var.availability_zones, count.index)}"
+    Name    = "${var.resource_prefix}-firewall-${element(var.availability_zones, count.index)}"
+    Project = var.resource_prefix
   }
 }
 
@@ -72,7 +75,8 @@ resource "aws_route_table" "firewall_rt" {
   count  = length(var.firewall_subnets_cidr)
   vpc_id = var.vpc_id
   tags = {
-    Name = "${var.resource_prefix}-firewall-rt-${element(var.availability_zones, count.index)}"
+    Name    = "${var.resource_prefix}-firewall-rt-${element(var.availability_zones, count.index)}"
+    Project = var.resource_prefix
   }
 }
 
@@ -87,15 +91,17 @@ resource "aws_route_table_association" "firewall" {
 resource "aws_internet_gateway" "igw" {
   vpc_id = var.vpc_id
   tags = {
-    Name = "${var.resource_prefix}-igw"
+    Name    = "${var.resource_prefix}-igw"
+    Project = var.resource_prefix
   }
 }
 
 // IGW RT
 resource "aws_route_table" "igw_rt" {
   vpc_id = var.vpc_id
   tags = {
-    Name = "${var.resource_prefix}-igw-rt"
+    Name    = "${var.resource_prefix}-igw-rt"
+    Project = var.resource_prefix
   }
 }
 
@@ -105,29 +111,47 @@ resource "aws_route_table_association" "igw" {
   route_table_id = aws_route_table.igw_rt.id
 }
 
+// Local Map for Availability Zone to Index
+locals {
+  az_to_index_map = {
+    for idx, az in var.availability_zones :
+    az => idx
+  }
+
+  firewall_endpoints_by_az = {
+    for sync_state in aws_networkfirewall_firewall.nfw.firewall_status[0].sync_states :
+    sync_state.availability_zone => sync_state.attachment[0].endpoint_id
+  }
+
+  az_to_endpoint_map = {
+    for az in var.availability_zones :
+    az => lookup(local.firewall_endpoints_by_az, az, null)
+  }
+}
+
 // Public Route
 resource "aws_route" "public" {
-  count                  = length(var.public_subnets_cidr)
-  route_table_id         = element(aws_route_table.public_rt.*.id, count.index)
+  for_each               = local.az_to_endpoint_map
+  route_table_id         = aws_route_table.public_rt[local.az_to_index_map[each.key]].id
   destination_cidr_block = "0.0.0.0/0"
-  vpc_endpoint_id        = tolist(aws_networkfirewall_firewall.nfw.firewall_status[0].sync_states)[count.index].attachment[0].endpoint_id
+  vpc_endpoint_id        = each.value
   depends_on             = [aws_networkfirewall_firewall.nfw]
 }
 
-// Firewall Route
+// Firewall Outbound Route
 resource "aws_route" "firewall_outbound" {
   count                  = length(var.firewall_subnets_cidr)
   route_table_id         = element(aws_route_table.firewall_rt.*.id, count.index)
   destination_cidr_block = "0.0.0.0/0"
   gateway_id             = aws_internet_gateway.igw.id
 }
 
-// Add a route back to FW
+// Firewall Inbound Route
 resource "aws_route" "firewall_inbound" {
-  count                  = length(var.public_subnets_cidr)
+  for_each               = local.az_to_endpoint_map
   route_table_id         = aws_route_table.igw_rt.id
-  destination_cidr_block = element(var.public_subnets_cidr, count.index)
-  vpc_endpoint_id        = tolist(aws_networkfirewall_firewall.nfw.firewall_status[0].sync_states)[count.index].attachment[0].endpoint_id
+  destination_cidr_block = element(var.public_subnets_cidr, index(var.availability_zones, each.key))
+  vpc_endpoint_id        = each.value
   depends_on             = [aws_networkfirewall_firewall.nfw]
 }
 
@@ -157,7 +181,8 @@ resource "aws_networkfirewall_rule_group" "databricks_fqdn_allowlist" {
     }
   }
   tags = {
-    Name = "${var.resource_prefix}-${var.region}-databricks-fqdn-allowlist"
+    Name    = "${var.resource_prefix}-${var.region}-databricks-fqdn-allowlist"
+    Project = var.resource_prefix
   }
 }
 
@@ -170,7 +195,6 @@ data "external" "metastore_ip" {
   }
 }
 
-
 // JDBC Firewall group IP allow list
 resource "aws_networkfirewall_rule_group" "databricks_metastore_allowlist" {
   capacity = 100
@@ -199,11 +223,12 @@ resource "aws_networkfirewall_rule_group" "databricks_metastore_allowlist" {
     }
   }
   tags = {
-    Name = "${var.resource_prefix}-${var.region}-databricks-metastore-allowlist"
+    Name    = "${var.resource_prefix}-${var.region}-databricks-metastore-allowlist"
+    Project = var.resource_prefix
   }
 }
 
-# Firewall policy
+// Firewall policy
 resource "aws_networkfirewall_firewall_policy" "databricks_nfw_policy" {
   name = "${var.resource_prefix}-firewall-policy"
 
@@ -229,11 +254,11 @@ resource "aws_networkfirewall_firewall_policy" "databricks_nfw_policy" {
   }
 
   tags = {
-    Name = "${var.resource_prefix}-firewall-policy"
+    Name    = "${var.resource_prefix}-firewall-policy"
+    Project = var.resource_prefix
   }
 }
 
-
 // Firewall
 resource "aws_networkfirewall_firewall" "nfw" {
   name                = "${var.resource_prefix}-nfw"
@@ -246,6 +271,7 @@ resource "aws_networkfirewall_firewall" "nfw" {
     }
   }
   tags = {
-    Name = "${var.resource_prefix}-${var.region}-databricks-nfw"
+    Name    = "${var.resource_prefix}-${var.region}-databricks-nfw"
+    Project = var.resource_prefix
   }
 }

aws-gov/tf/modules/sra/data_plane_hardening/firewall/variables.tf

@@ -1,47 +1,43 @@
-variable "vpc_id" {
-  type = string
+variable "availability_zones" {
+  type = list(string)
 }
 
-variable "vpc_cidr_range" {
-  type = string
+variable "firewall_allow_list" {
+  type = list(string)
 }
 
-variable "public_subnets_cidr" {
+variable "firewall_subnets_cidr" {
   type = list(string)
 }
 
-variable "private_subnets_cidr" {
-  type = list(string)
+variable "hive_metastore_fqdn" {
+  type = string
 }
 
 variable "private_subnet_rt" {
   type = list(string)
 }
 
-variable "firewall_subnets_cidr" {
+variable "private_subnets_cidr" {
   type = list(string)
 }
 
-variable "firewall_allow_list" {
+variable "public_subnets_cidr" {
   type = list(string)
 }
 
-variable "hive_metastore_fqdn" {
+variable "region" {
   type = string
 }
 
-variable "availability_zones" {
-  type = list(string)
-}
-
-variable "region" {
+variable "resource_prefix" {
   type = string
 }
 
-variable "resource_prefix" {
+variable "vpc_cidr_range" {
   type = string
 }
 
-variable "firewall_protocol_deny_list" {
-  type = list(string)
+variable "vpc_id" {
+  type = string
 }

aws-gov/tf/modules/sra/databricks_account/logging_configuration/logging_configuration.tf

@@ -5,7 +5,8 @@ resource "aws_s3_bucket" "log_delivery" {
   bucket        = "${var.resource_prefix}-log-delivery"
   force_destroy = true
   tags = {
-    Name = "${var.resource_prefix}-log-delivery"
+    Name    = "${var.resource_prefix}-log-delivery"
+    Project = var.resource_prefix
   }
 }
 
@@ -106,14 +107,16 @@ resource "aws_iam_role" "log_delivery" {
   description        = "(${var.resource_prefix}) Log Delivery Role"
   assume_role_policy = data.aws_iam_policy_document.passrole_for_log_delivery.json
   tags = {
-    Name = "${var.resource_prefix}-log-delivery-role"
+    Name    = "${var.resource_prefix}-log-delivery-role"
+    Project = var.resource_prefix
   }
 }
 
 // Databricks Configurations
 
 // Databricks Credential Configuration for Logs
 resource "databricks_mws_credentials" "log_writer" {
+  account_id                 = var.databricks_account_id
   credentials_name = "${var.resource_prefix}-log-delivery-credential"
   role_arn         = aws_iam_role.log_delivery.arn
   depends_on = [

aws-gov/tf/modules/sra/databricks_account/logging_configuration/variables.tf

@@ -1,7 +1,3 @@
-variable "resource_prefix" {
-  type = string
-}
-
 variable "databricks_account_id" {
   type = string
 }
@@ -10,10 +6,14 @@ variable "databricks_gov_shard" {
   type = string
 }
 
-variable "log_delivery_role_name" {
+variable "databricks_prod_aws_account_id" {
   type = map(string)
 }
 
-variable "databricks_prod_aws_account_id" {
+variable "log_delivery_role_name" {
   type = map(string)
+}
+
+variable "resource_prefix" {
+  type = string
 }

aws-gov/tf/modules/sra/databricks_account/uc_assignment/variables.tf

@@ -2,10 +2,10 @@ variable "metastore_id" {
   type = string
 }
 
-variable "workspace_id" {
+variable "region" {
   type = string
 }
 
-variable "region" {
+variable "workspace_id" {
   type = string
 }