From adadcc7220aba246b2e1ca644ecd88794fed167f Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Tue, 5 Sep 2023 11:52:18 -0500 Subject: [PATCH 01/11] WIP #213 Started getting Orca Variables and TF info together #213 --- app/stacks/cumulus/main.tf | 1 + app/stacks/cumulus/orca.tf | 64 ++++++++++++++++++++++++++++ app/stacks/cumulus/tfvars/uat.tfvars | 8 ++++ app/stacks/cumulus/variables.tf | 24 +++++++++++ app/stacks/rds-cluster/outputs.tf | 1 + 5 files changed, 98 insertions(+) create mode 100644 app/stacks/cumulus/orca.tf diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf index 58b5f0f..33b5aeb 100644 --- a/app/stacks/cumulus/main.tf +++ b/app/stacks/cumulus/main.tf @@ -59,6 +59,7 @@ locals { rds_security_group = jsondecode("<%= json_output('rds-cluster.security_group_id') %>") rds_user_access_secret_arn = jsondecode("<%= json_output('rds-cluster.user_credentials_secret_arn') %>") + rds_endpoint = jsondecode("<%= json_output('rds-cluster.rds_endpoint') %>") tags = merge(var.tags, { Deployment = var.prefix }) } diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf new file mode 100644 index 0000000..1b52282 --- /dev/null +++ b/app/stacks/cumulus/orca.tf @@ -0,0 +1,64 @@ +## ORCA Module +## ============================================================================= +resource "random_password" "db_password" { + length = 50 + upper = true + special = false +} +module "orca" { + source = "https://github.com/nasa/cumulus-orca/releases/download/v6.0.2/cumulus-orca-terraform.zip" + ## -------------------------- + ## Cumulus Variables + ## -------------------------- + ## REQUIRED + buckets = var.buckets + lambda_subnet_ids = var.lambda_subnet_ids + permissions_boundary_arn = var.permissions_boundary_arn + prefix = var.prefix + system_bucket = var.system_bucket + vpc_id = module.vpc.vpc_id + workflow_config = module.cumulus.workflow_config + + ## OPTIONAL + tags = var.tags + + ## -------------------------- + ## ORCA Variables + ## -------------------------- + ## REQUIRED + db_admin_password = random_password.db_password.result + db_host_endpoint = local.rds_endpoint + db_user_password = random_password.db_password.result + dlq_subscription_email = var.dlq_subscription_email + orca_default_bucket = var.orca_default_bucket + orca_reports_bucket_name = var.orca_reports_bucket_name + rds_security_group_id = var.rds_security_group_id + + ## OPTIONAL + # db_admin_username = "postgres" + # default_multipart_chunksize_mb = 250 + # metadata_queue_message_retention_time = 777600 + # orca_default_recovery_type = "Standard" + # orca_default_storage_class = "GLACIER" + # orca_delete_old_reconcile_jobs_frequency_cron = "cron(0 0 ? * SUN *)" + # orca_ingest_lambda_memory_size = 2240 + # orca_ingest_lambda_timeout = 600 + # orca_internal_reconciliation_expiration_days = 30 + # orca_reconciliation_lambda_memory_size = 128 + # orca_reconciliation_lambda_timeout = 720 + # orca_recovery_buckets = [] + # orca_recovery_complete_filter_prefix = "" + # orca_recovery_expiration_days = 5 + # orca_recovery_lambda_memory_size = 128 + # orca_recovery_lambda_timeout = 720 + # orca_recovery_retry_limit = 3 + # orca_recovery_retry_interval = 1 + # orca_recovery_retry_backoff = 2 + # s3_inventory_queue_message_retention_time_seconds = 432000 + # s3_report_frequency = "Daily" + # sqs_delay_time_seconds = 0 + # sqs_maximum_message_size = 262144 + # staged_recovery_queue_message_retention_time_seconds = 432000 + # status_update_queue_message_retention_time_seconds = 777600 + +} diff --git a/app/stacks/cumulus/tfvars/uat.tfvars b/app/stacks/cumulus/tfvars/uat.tfvars index 25adea3..adbcfc5 100644 --- a/app/stacks/cumulus/tfvars/uat.tfvars +++ b/app/stacks/cumulus/tfvars/uat.tfvars @@ -24,3 +24,11 @@ s3_replicator_target_bucket = "esdis-metrics-inbound-uat-csdap-distribution" # <% end %> s3_replicator_target_prefix = "input/s3_access/csdapuat" + +# Orca Integration +db_admin_password = "" # TODO - Maybe Needs to be done in SSM +db_user_password = "" +dlq_subscription_email = "" +orca_default_bucket = "" +orca_reports_bucket_name = "" +rds_security_group_id = "" diff --git a/app/stacks/cumulus/variables.tf b/app/stacks/cumulus/variables.tf index 0e73f95..0b35504 100644 --- a/app/stacks/cumulus/variables.tf +++ b/app/stacks/cumulus/variables.tf @@ -196,3 +196,27 @@ variable "tags" { type = map(string) default = {} } + +# ORCA Variables +variable "db_admin_password" { + +} + +variable "db_user_password" { + +} +variable "dlq_subscription_email" { + default = "pic8690@gmail.com" +} + +# TODO +# https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket/ +variable "orca_default_bucket" { + default = "TODO__NEED_BUCKET_NAME_CONVENTION" # TODO - Go to Disaster Recovery Account +} +variable "orca_reports_bucket_name" { + default = "" +} +variable "rds_security_group_id" { + default = "" +} diff --git a/app/stacks/rds-cluster/outputs.tf b/app/stacks/rds-cluster/outputs.tf index 4509213..f3b6313 100644 --- a/app/stacks/rds-cluster/outputs.tf +++ b/app/stacks/rds-cluster/outputs.tf @@ -17,3 +17,4 @@ output "security_group_id" { output "user_credentials_secret_arn" { value = module.rds_cluster.user_credentials_secret_arn } + From 1723ae146ff679216045a8531da9fa444be1be50 Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Tue, 19 Sep 2023 13:53:31 -0500 Subject: [PATCH 02/11] further progress on ORCA work - this time we added the bucket definitions and attempted a terraform plan generation. #213 --- app/stacks/cumulus/orca.tf | 4 +++- app/stacks/cumulus/tfvars/base.tfvars | 4 ++++ app/stacks/cumulus/variables.tf | 7 ++----- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf index 1b52282..8d7c0ab 100644 --- a/app/stacks/cumulus/orca.tf +++ b/app/stacks/cumulus/orca.tf @@ -32,7 +32,9 @@ module "orca" { dlq_subscription_email = var.dlq_subscription_email orca_default_bucket = var.orca_default_bucket orca_reports_bucket_name = var.orca_reports_bucket_name - rds_security_group_id = var.rds_security_group_id + rds_security_group_id = local.rds_security_group + + ## OPTIONAL # db_admin_username = "postgres" diff --git a/app/stacks/cumulus/tfvars/base.tfvars b/app/stacks/cumulus/tfvars/base.tfvars index 6fdfea3..0d1ec1f 100644 --- a/app/stacks/cumulus/tfvars/base.tfvars +++ b/app/stacks/cumulus/tfvars/base.tfvars @@ -19,6 +19,10 @@ cmr_environment = "UAT" system_bucket = "<%= bucket('internal') %>" buckets = { + orca_default = { + name = "csda-cumulus-cba-uat-orca-archive" + type = "orca" + } internal = { name = "<%= bucket('internal') %>" type = "internal" diff --git a/app/stacks/cumulus/variables.tf b/app/stacks/cumulus/variables.tf index 0b35504..a724859 100644 --- a/app/stacks/cumulus/variables.tf +++ b/app/stacks/cumulus/variables.tf @@ -212,11 +212,8 @@ variable "dlq_subscription_email" { # TODO # https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket/ variable "orca_default_bucket" { - default = "TODO__NEED_BUCKET_NAME_CONVENTION" # TODO - Go to Disaster Recovery Account + default = "csda-cumulus-cba-uat-orca-archive" # TODO - Go to Disaster Recovery Account } variable "orca_reports_bucket_name" { - default = "" -} -variable "rds_security_group_id" { - default = "" + default = "csda-cumulus-cba-uat-orca-reports" } From 0520ab1407666c7fa0e82102448711e1a1927885 Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Wed, 20 Sep 2023 12:24:43 -0500 Subject: [PATCH 03/11] ORCA Changes WIP #213 --- app/stacks/cumulus/orca.tf | 9 ++++++--- app/stacks/cumulus/tfvars/base.tfvars | 4 ++++ app/stacks/cumulus/tfvars/uat.tfvars | 12 ++++++------ app/stacks/cumulus/variables.tf | 23 +++++++++++++++++------ 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf index 8d7c0ab..4badf34 100644 --- a/app/stacks/cumulus/orca.tf +++ b/app/stacks/cumulus/orca.tf @@ -12,8 +12,8 @@ module "orca" { ## -------------------------- ## REQUIRED buckets = var.buckets - lambda_subnet_ids = var.lambda_subnet_ids - permissions_boundary_arn = var.permissions_boundary_arn + lambda_subnet_ids = module.vpc.subnets.ids + permissions_boundary_arn = local.permissions_boundary_arn prefix = var.prefix system_bucket = var.system_bucket vpc_id = module.vpc.vpc_id @@ -33,8 +33,11 @@ module "orca" { orca_default_bucket = var.orca_default_bucket orca_reports_bucket_name = var.orca_reports_bucket_name rds_security_group_id = local.rds_security_group + s3_access_key = var.s3_access_key + s3_secret_key = var.s3_secret_key + + - ## OPTIONAL # db_admin_username = "postgres" diff --git a/app/stacks/cumulus/tfvars/base.tfvars b/app/stacks/cumulus/tfvars/base.tfvars index 0d1ec1f..a4bfc49 100644 --- a/app/stacks/cumulus/tfvars/base.tfvars +++ b/app/stacks/cumulus/tfvars/base.tfvars @@ -19,6 +19,10 @@ cmr_environment = "UAT" system_bucket = "<%= bucket('internal') %>" buckets = { + orca_reports_bucket_name = { + name = "csda-cumulus-cba-uat-orca-archive" + type = "orca" + } orca_default = { name = "csda-cumulus-cba-uat-orca-archive" type = "orca" diff --git a/app/stacks/cumulus/tfvars/uat.tfvars b/app/stacks/cumulus/tfvars/uat.tfvars index adbcfc5..b130242 100644 --- a/app/stacks/cumulus/tfvars/uat.tfvars +++ b/app/stacks/cumulus/tfvars/uat.tfvars @@ -26,9 +26,9 @@ s3_replicator_target_bucket = "esdis-metrics-inbound-uat-csdap-distribution" s3_replicator_target_prefix = "input/s3_access/csdapuat" # Orca Integration -db_admin_password = "" # TODO - Maybe Needs to be done in SSM -db_user_password = "" -dlq_subscription_email = "" -orca_default_bucket = "" -orca_reports_bucket_name = "" -rds_security_group_id = "" +#db_admin_password = "" # TODO - Maybe Needs to be done in SSM +#db_user_password = "" +#dlq_subscription_email = "" +#orca_default_bucket = "" +#orca_reports_bucket_name = "" +#rds_security_group_id = "" diff --git a/app/stacks/cumulus/variables.tf b/app/stacks/cumulus/variables.tf index a724859..0c06ff1 100644 --- a/app/stacks/cumulus/variables.tf +++ b/app/stacks/cumulus/variables.tf @@ -198,13 +198,13 @@ variable "tags" { } # ORCA Variables -variable "db_admin_password" { +#variable "db_admin_password" { +# +#} -} - -variable "db_user_password" { - -} +#variable "db_user_password" { +# +#} variable "dlq_subscription_email" { default = "pic8690@gmail.com" } @@ -217,3 +217,14 @@ variable "orca_default_bucket" { variable "orca_reports_bucket_name" { default = "csda-cumulus-cba-uat-orca-reports" } + +variable "s3_access_key" { + default = "Axxx" +} +variable "s3_secret_key" { + default = "Axxx" +} + + + + From f1f4d07f92c42a31d564ae74e665a59386004c1d Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Mon, 25 Sep 2023 23:05:34 -0500 Subject: [PATCH 04/11] Added SSM Params for ORCA. Note, these are AWS DR Acct Keys that are stored in the SSM of their matching non-DR aws account. DR is short for Disaster Recovery. As of now, we have 4 accounts which will utilize these. CBA_UAT, CBA_DR_UAT, CBA_PROD, and CBA_DR_PROD. #213 --- app/stacks/cumulus/orca.tf | 4 ++-- app/stacks/cumulus/ssm_parameters.tf | 21 +++++++++++++++++++++ app/stacks/cumulus/variables.tf | 16 ++++++++++------ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf index 4badf34..93d8afb 100644 --- a/app/stacks/cumulus/orca.tf +++ b/app/stacks/cumulus/orca.tf @@ -33,8 +33,8 @@ module "orca" { orca_default_bucket = var.orca_default_bucket orca_reports_bucket_name = var.orca_reports_bucket_name rds_security_group_id = local.rds_security_group - s3_access_key = var.s3_access_key - s3_secret_key = var.s3_secret_key + s3_access_key = data.aws_ssm_parameter.s3_access_key.value + s3_secret_key = data.aws_ssm_parameter.s3_secret_key.value diff --git a/app/stacks/cumulus/ssm_parameters.tf b/app/stacks/cumulus/ssm_parameters.tf index 0321986..05925cb 100644 --- a/app/stacks/cumulus/ssm_parameters.tf +++ b/app/stacks/cumulus/ssm_parameters.tf @@ -54,6 +54,7 @@ data "aws_ssm_parameter" "csdap_client_password" { name = "/shared/cumulus/csdap-client-password" } + #------------------------------------------------------------------------------- # SSM Parameters required across ONLY non-sandbox (non-dev) environments #------------------------------------------------------------------------------- @@ -84,6 +85,26 @@ data "aws_ssm_parameter" "metrics_aws_account_id" { name = "/shared/cumulus/metrics-aws-account-id" } +# ORCA Bucket Access - Note: As of now, the Buckets must be setup on the DR AWS accounts +# There are only DR AWS accounts for CBA UAT and CBA PROD +# +# Unfortunately, This parameter must be refreshed everytime these keys expire. +# To refresh, do the following +# (1) Make new keys +# (2) For each environment, run the following commands +# (2a) make bash +# (2b) aws ssm put-parameter --type SecureString --name NAME --overwrite --value VALUE +# +# Note, for setting the FIRST time, the command is slightly different (no --overwrite) +# # aws ssm put-parameter --type SecureString --name NAME --value VALUE +# TODO - add some of the above stuff to the proper documentation +data "aws_ssm_parameter" "s3_access_key" { + name = "/shared/cumulus/orca/dr/s3-access-key" +} +data "aws_ssm_parameter" "s3_secret_key" { + name = "/shared/cumulus/orca/dr/s3-secret-key" +} + # <% end %> #------------------------------------------------------------------------------- diff --git a/app/stacks/cumulus/variables.tf b/app/stacks/cumulus/variables.tf index 0c06ff1..9e8ed71 100644 --- a/app/stacks/cumulus/variables.tf +++ b/app/stacks/cumulus/variables.tf @@ -218,12 +218,16 @@ variable "orca_reports_bucket_name" { default = "csda-cumulus-cba-uat-orca-reports" } -variable "s3_access_key" { - default = "Axxx" -} -variable "s3_secret_key" { - default = "Axxx" -} +# TODO - Remove these from here all together during the PR +# These have been moved to ssm_parameters.tf +# Leaving these here while this task is still a Work in Progress +# +#variable "s3_access_key" { +# default = "Axxx" +#} +#variable "s3_secret_key" { +# default = "Axxx" +#} From af07cba19d10e32627cf2d654fbd9ff76cfdab2d Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Tue, 26 Sep 2023 12:20:08 -0500 Subject: [PATCH 05/11] Fix for sandbox deployment error #213 #214 --- app/stacks/cumulus/ssm_parameters.tf | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/stacks/cumulus/ssm_parameters.tf b/app/stacks/cumulus/ssm_parameters.tf index 05925cb..a0eaf4b 100644 --- a/app/stacks/cumulus/ssm_parameters.tf +++ b/app/stacks/cumulus/ssm_parameters.tf @@ -54,6 +54,12 @@ data "aws_ssm_parameter" "csdap_client_password" { name = "/shared/cumulus/csdap-client-password" } +data "aws_ssm_parameter" "s3_access_key" { + name = "/shared/cumulus/orca/dr/s3-access-key" +} +data "aws_ssm_parameter" "s3_secret_key" { + name = "/shared/cumulus/orca/dr/s3-secret-key" +} #------------------------------------------------------------------------------- # SSM Parameters required across ONLY non-sandbox (non-dev) environments @@ -98,12 +104,12 @@ data "aws_ssm_parameter" "metrics_aws_account_id" { # Note, for setting the FIRST time, the command is slightly different (no --overwrite) # # aws ssm put-parameter --type SecureString --name NAME --value VALUE # TODO - add some of the above stuff to the proper documentation -data "aws_ssm_parameter" "s3_access_key" { - name = "/shared/cumulus/orca/dr/s3-access-key" -} -data "aws_ssm_parameter" "s3_secret_key" { - name = "/shared/cumulus/orca/dr/s3-secret-key" -} +#data "aws_ssm_parameter" "s3_access_key" { +# name = "/shared/cumulus/orca/dr/s3-access-key" +#} +#data "aws_ssm_parameter" "s3_secret_key" { +# name = "/shared/cumulus/orca/dr/s3-secret-key" +#} # <% end %> From 46f212428a42b04e50a14bf62a126820d4944e0c Mon Sep 17 00:00:00 2001 From: Kris Stanton Date: Wed, 27 Sep 2023 13:29:09 -0500 Subject: [PATCH 06/11] WIP - having trouble getting passed the postrgres connection during a sandbox deploy. #213 #214 --- app/stacks/cumulus/main.tf | 5 +++++ app/stacks/cumulus/orca.tf | 7 +++++-- app/stacks/rds-cluster/outputs.tf | 23 +++++++++++++++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf index 33b5aeb..3640dba 100644 --- a/app/stacks/cumulus/main.tf +++ b/app/stacks/cumulus/main.tf @@ -61,6 +61,11 @@ locals { rds_user_access_secret_arn = jsondecode("<%= json_output('rds-cluster.user_credentials_secret_arn') %>") rds_endpoint = jsondecode("<%= json_output('rds-cluster.rds_endpoint') %>") + # For these two lines, I am trying to give the cumulus module access to the password from the RDS module so it can make the connection + #rds_cluster_db_admin_password_1 = jsondecode("<%= json_output('rds-cluster.rds_cluster_db_admin_password') %>") + #rds_cluster_rds_user_password_1 = jsondecode("<%= json_output('rds-cluster.rds_cluster_rds_user_password') %>") + + tags = merge(var.tags, { Deployment = var.prefix }) } diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf index 93d8afb..5f43471 100644 --- a/app/stacks/cumulus/orca.tf +++ b/app/stacks/cumulus/orca.tf @@ -26,15 +26,18 @@ module "orca" { ## ORCA Variables ## -------------------------- ## REQUIRED - db_admin_password = random_password.db_password.result + # + # These two don't work unless I hardcode the password!!!!! (db_admin_password, and db_user_password) + db_admin_password = random_password.db_password.result # Note, this does not work - it will generate a password that is different from the actual one generated in the rds_cluster (which I cannot seem to access.. If I could access that password programmatically then this will get passed the PW error correctly.. I tested this by hard coding the password in and that worked. db_host_endpoint = local.rds_endpoint - db_user_password = random_password.db_password.result + db_user_password = random_password.db_password.result # Note, this does not work - it will generate a password that is different from the actual one generated in the rds_cluster (which I cannot seem to access.. If I could access that password programmatically then this will get passed the PW error correctly.. I tested this by hard coding the password in and that worked. dlq_subscription_email = var.dlq_subscription_email orca_default_bucket = var.orca_default_bucket orca_reports_bucket_name = var.orca_reports_bucket_name rds_security_group_id = local.rds_security_group s3_access_key = data.aws_ssm_parameter.s3_access_key.value s3_secret_key = data.aws_ssm_parameter.s3_secret_key.value + db_admin_username = "postgres" diff --git a/app/stacks/rds-cluster/outputs.tf b/app/stacks/rds-cluster/outputs.tf index f3b6313..de9ebe3 100644 --- a/app/stacks/rds-cluster/outputs.tf +++ b/app/stacks/rds-cluster/outputs.tf @@ -18,3 +18,26 @@ output "user_credentials_secret_arn" { value = module.rds_cluster.user_credentials_secret_arn } +# How do we output the password so that the Cumulus module can read it via +# +# # jsondecode("<%= json_output('rds-cluster.db_admin_password') %>") +# # jsondecode("<%= json_output('rds-cluster.rds_user_password') %>") +# +# This does not work... why.. +# +# [2023-09-27T17:36:47 #29 terraspace up rds-cluster]: Error: Unsupported attribute +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: on outputs.tf line 27, in output "rds_cluster_db_admin_password": +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: 27: value = module.rds_cluster.db_admin_password +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: This object does not have an attribute named "db_admin_password". +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: +#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: +# +#output "rds_cluster_db_admin_password" { +# value = module.rds_cluster.db_admin_password +#} +## +#output "rds_cluster_rds_user_password" { +# value = module.rds_cluster.rds_user_password +#} From 5d1ed2f11983e49d6a6a2dbaf9e9fb4b6b27ee0d Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 5 Oct 2023 10:00:18 -0400 Subject: [PATCH 07/11] Configure RDS secrets for ORCA module --- .pre-commit-config.yaml | 9 ++++ Makefile | 4 ++ README.md | 32 +++++++++++++ app/stacks/cumulus/main.tf | 5 --- app/stacks/cumulus/orca.tf | 65 ++++++++++++++------------- app/stacks/cumulus/ssm_parameters.tf | 11 +++-- app/stacks/cumulus/tfvars/base.tfvars | 10 +++-- app/stacks/cumulus/tfvars/prod.tfvars | 2 + app/stacks/cumulus/tfvars/uat.tfvars | 10 +---- app/stacks/cumulus/variables.tf | 40 ++--------------- app/stacks/rds-cluster/outputs.tf | 24 ---------- bin/pre-commit-terraspace-fmt.sh | 5 +++ 12 files changed, 107 insertions(+), 110 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100755 bin/pre-commit-terraspace-fmt.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f24bb8a --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: local + hooks: + - id: terraspace-fmt + name: Terraspace Format + entry: bin/pre-commit-terraspace-fmt.sh + language: system + pass_filenames: false + files: \.tf$ diff --git a/Makefile b/Makefile index 657327b..141790f 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,10 @@ create-test-data: docker docker: Dockerfile .dockerignore .terraform-version Gemfile Gemfile.lock package.json yarn.lock $(DOCKER_BUILD) +## fmt: Runs `terraspace fmt` to format all Terraform files +fmt: docker + $(DOCKER_RUN) $(IMAGE) bundle exec 'terraspace fmt 2>/dev/null' + ## init-STACK: Runs `terraform init` for specified STACK init-%: docker $(TERRASPACE) init $* diff --git a/README.md b/README.md index 28ee200..0222e15 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,36 @@ this repository: used within the Docker container to properly configure the AWS CLI and Terraform. +If you also wish to contribute changes, you should also do the following: + +- **Install pre-commit** + + If you don't already have `pre-commit` installed on your development machine, + please [install pre-commit]. + +- **Install the pre-commit hooks** + + Once `pre-commit` is installed, install the pre-commit hooks defined in the + `.pre-commit-config.yaml` file by running the following command: + + ```plain + pre-commit install --install-hooks + ``` + + This will cause the configured hooks to run whenever you run `git commit`. If + any hooks fail, the commit is aborted, requiring you to fix the problem(s) + that caused the hook(s) to fail. Often, hooks automatically fix problems + (such as file formatting), and thus you may simply need to `git add` the + automatically fixed files and run `git commit` again. + + Further, you can run `pre-commit` hooks _without_ running `git commit` if you + wish to, which can be handy when you want to perform actions such as file + formatting prior to adding files to git: + + ```plain + pre-commit run -a + ``` + ## Infrastructure Management This section assumes that you have completed all prerequisite steps as detailed @@ -195,6 +225,8 @@ See [Destroying a Deployment](docs/OPERATING.md#destroying-a-deployment) in [Deploying Cumulus Troubleshooting]: https://nasa.github.io/cumulus/docs/troubleshooting/troubleshooting-deployment#deploying-cumulus +[Install pre-commit]: + https://pre-commit.com/#install [Terraform]: https://www.terraform.io/ [Terraspace]: diff --git a/app/stacks/cumulus/main.tf b/app/stacks/cumulus/main.tf index 3640dba..33b5aeb 100644 --- a/app/stacks/cumulus/main.tf +++ b/app/stacks/cumulus/main.tf @@ -61,11 +61,6 @@ locals { rds_user_access_secret_arn = jsondecode("<%= json_output('rds-cluster.user_credentials_secret_arn') %>") rds_endpoint = jsondecode("<%= json_output('rds-cluster.rds_endpoint') %>") - # For these two lines, I am trying to give the cumulus module access to the password from the RDS module so it can make the connection - #rds_cluster_db_admin_password_1 = jsondecode("<%= json_output('rds-cluster.rds_cluster_db_admin_password') %>") - #rds_cluster_rds_user_password_1 = jsondecode("<%= json_output('rds-cluster.rds_cluster_rds_user_password') %>") - - tags = merge(var.tags, { Deployment = var.prefix }) } diff --git a/app/stacks/cumulus/orca.tf b/app/stacks/cumulus/orca.tf index 5f43471..29d7cae 100644 --- a/app/stacks/cumulus/orca.tf +++ b/app/stacks/cumulus/orca.tf @@ -1,16 +1,25 @@ -## ORCA Module -## ============================================================================= -resource "random_password" "db_password" { - length = 50 - upper = true - special = false +data "aws_secretsmanager_secret" "rds_cluster_admin_db_login_secret" { + arn = "<%= unquoted(output('rds-cluster.admin_db_login_secret_arn')) %>" } + +data "aws_secretsmanager_secret_version" "rds_cluster_admin_db_login_secret_version" { + secret_id = data.aws_secretsmanager_secret.rds_cluster_admin_db_login_secret.id +} + +data "aws_secretsmanager_secret" "rds_cluster_user_credentials_secret" { + arn = "<%= unquoted(output('rds-cluster.user_credentials_secret_arn')) %>" +} + +data "aws_secretsmanager_secret_version" "rds_cluster_user_credentials_secret_version" { + secret_id = data.aws_secretsmanager_secret.rds_cluster_user_credentials_secret.id +} + module "orca" { - source = "https://github.com/nasa/cumulus-orca/releases/download/v6.0.2/cumulus-orca-terraform.zip" - ## -------------------------- - ## Cumulus Variables - ## -------------------------- - ## REQUIRED + source = "https://github.com/nasa/cumulus-orca/releases/download/v6.0.3/cumulus-orca-terraform.zip" + #-------------------------- + # Cumulus variables + #-------------------------- + # REQUIRED buckets = var.buckets lambda_subnet_ids = module.vpc.subnets.ids permissions_boundary_arn = local.permissions_boundary_arn @@ -19,30 +28,27 @@ module "orca" { vpc_id = module.vpc.vpc_id workflow_config = module.cumulus.workflow_config - ## OPTIONAL - tags = var.tags + # OPTIONAL + tags = var.tags - ## -------------------------- - ## ORCA Variables - ## -------------------------- - ## REQUIRED + #-------------------------- + # ORCA variables + #-------------------------- + # REQUIRED # - # These two don't work unless I hardcode the password!!!!! (db_admin_password, and db_user_password) - db_admin_password = random_password.db_password.result # Note, this does not work - it will generate a password that is different from the actual one generated in the rds_cluster (which I cannot seem to access.. If I could access that password programmatically then this will get passed the PW error correctly.. I tested this by hard coding the password in and that worked. db_host_endpoint = local.rds_endpoint - db_user_password = random_password.db_password.result # Note, this does not work - it will generate a password that is different from the actual one generated in the rds_cluster (which I cannot seem to access.. If I could access that password programmatically then this will get passed the PW error correctly.. I tested this by hard coding the password in and that worked. - dlq_subscription_email = var.dlq_subscription_email - orca_default_bucket = var.orca_default_bucket - orca_reports_bucket_name = var.orca_reports_bucket_name + db_admin_username = "postgres" + db_admin_password = jsondecode(data.aws_secretsmanager_secret_version.rds_cluster_admin_db_login_secret_version.secret_string)["password"] + db_user_password = jsondecode(data.aws_secretsmanager_secret_version.rds_cluster_user_credentials_secret_version.secret_string)["password"] + dlq_subscription_email = var.orca_dlq_subscription_email + orca_default_bucket = var.buckets.orca_default.name + orca_reports_bucket_name = var.buckets.orca_reports.name rds_security_group_id = local.rds_security_group - s3_access_key = data.aws_ssm_parameter.s3_access_key.value - s3_secret_key = data.aws_ssm_parameter.s3_secret_key.value - db_admin_username = "postgres" - + s3_access_key = data.aws_ssm_parameter.orca_s3_access_key.value + s3_secret_key = data.aws_ssm_parameter.orca_s3_secret_key.value + # OPTIONAL - - ## OPTIONAL # db_admin_username = "postgres" # default_multipart_chunksize_mb = 250 # metadata_queue_message_retention_time = 777600 @@ -68,5 +74,4 @@ module "orca" { # sqs_maximum_message_size = 262144 # staged_recovery_queue_message_retention_time_seconds = 432000 # status_update_queue_message_retention_time_seconds = 777600 - } diff --git a/app/stacks/cumulus/ssm_parameters.tf b/app/stacks/cumulus/ssm_parameters.tf index a0eaf4b..ce571d8 100644 --- a/app/stacks/cumulus/ssm_parameters.tf +++ b/app/stacks/cumulus/ssm_parameters.tf @@ -54,10 +54,11 @@ data "aws_ssm_parameter" "csdap_client_password" { name = "/shared/cumulus/csdap-client-password" } -data "aws_ssm_parameter" "s3_access_key" { +data "aws_ssm_parameter" "orca_s3_access_key" { name = "/shared/cumulus/orca/dr/s3-access-key" } -data "aws_ssm_parameter" "s3_secret_key" { + +data "aws_ssm_parameter" "orca_s3_secret_key" { name = "/shared/cumulus/orca/dr/s3-secret-key" } @@ -104,10 +105,12 @@ data "aws_ssm_parameter" "metrics_aws_account_id" { # Note, for setting the FIRST time, the command is slightly different (no --overwrite) # # aws ssm put-parameter --type SecureString --name NAME --value VALUE # TODO - add some of the above stuff to the proper documentation -#data "aws_ssm_parameter" "s3_access_key" { + +#data "aws_ssm_parameter" "orca_s3_access_key" { # name = "/shared/cumulus/orca/dr/s3-access-key" #} -#data "aws_ssm_parameter" "s3_secret_key" { + +#data "aws_ssm_parameter" "orca_s3_secret_key" { # name = "/shared/cumulus/orca/dr/s3-secret-key" #} diff --git a/app/stacks/cumulus/tfvars/base.tfvars b/app/stacks/cumulus/tfvars/base.tfvars index a4bfc49..703b682 100644 --- a/app/stacks/cumulus/tfvars/base.tfvars +++ b/app/stacks/cumulus/tfvars/base.tfvars @@ -15,16 +15,20 @@ #<% depends_on("rds-cluster") %> cmr_environment = "UAT" +orca_dlq_subscription_email = "pic8690@gmail.com" system_bucket = "<%= bucket('internal') %>" buckets = { - orca_reports_bucket_name = { - name = "csda-cumulus-cba-uat-orca-archive" + # https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket/ + orca_reports = { + # name = "<%= expand('csda-cumulus-cba-:ENV-orca-reports') %>" + name = "<%= %Q[csda-cumulus-cba-#{Terraspace.env == 'prod' ? 'prod' : 'uat'}-orca-reports] %>" type = "orca" } orca_default = { - name = "csda-cumulus-cba-uat-orca-archive" + # name = "<%= expand('csda-cumulus-cba-:ENV-orca-archive') %>" + name = "<%= %Q[csda-cumulus-cba-#{Terraspace.env == 'prod' ? 'prod' : 'uat'}-orca-archive] %>" type = "orca" } internal = { diff --git a/app/stacks/cumulus/tfvars/prod.tfvars b/app/stacks/cumulus/tfvars/prod.tfvars index 0352275..fd07562 100644 --- a/app/stacks/cumulus/tfvars/prod.tfvars +++ b/app/stacks/cumulus/tfvars/prod.tfvars @@ -17,6 +17,8 @@ cumulus_distribution_url = "https://data.csdap.earthdata.nasa.gov/" cumulus_distribution_url = "https://data.csda.earthdata.nasa.gov/" # <% end %> +#orca_dlq_subscription_email = "" + metrics_es_host = "https://d23fzndssjmbvi.cloudfront.net/" # <% if in_cba? then %> diff --git a/app/stacks/cumulus/tfvars/uat.tfvars b/app/stacks/cumulus/tfvars/uat.tfvars index b130242..a4ab130 100644 --- a/app/stacks/cumulus/tfvars/uat.tfvars +++ b/app/stacks/cumulus/tfvars/uat.tfvars @@ -15,6 +15,8 @@ cumulus_distribution_url = "https://data.csdap.uat.earthdata.nasa.gov/" cumulus_distribution_url = "https://data.csda.uat.earthdata.nasa.gov/" # <% end %> +#orca_dlq_subscription_email = "" + metrics_es_host = "https://dmzza2al43z4f.cloudfront.net/" # <% if in_cba? then %> @@ -24,11 +26,3 @@ s3_replicator_target_bucket = "esdis-metrics-inbound-uat-csdap-distribution" # <% end %> s3_replicator_target_prefix = "input/s3_access/csdapuat" - -# Orca Integration -#db_admin_password = "" # TODO - Maybe Needs to be done in SSM -#db_user_password = "" -#dlq_subscription_email = "" -#orca_default_bucket = "" -#orca_reports_bucket_name = "" -#rds_security_group_id = "" diff --git a/app/stacks/cumulus/variables.tf b/app/stacks/cumulus/variables.tf index 9e8ed71..5a88e3a 100644 --- a/app/stacks/cumulus/variables.tf +++ b/app/stacks/cumulus/variables.tf @@ -156,6 +156,10 @@ variable "metrics_es_username" { default = null } +variable "orca_dlq_subscription_email" { + type = string +} + variable "private_archive_api_gateway" { type = bool default = true @@ -196,39 +200,3 @@ variable "tags" { type = map(string) default = {} } - -# ORCA Variables -#variable "db_admin_password" { -# -#} - -#variable "db_user_password" { -# -#} -variable "dlq_subscription_email" { - default = "pic8690@gmail.com" -} - -# TODO -# https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket/ -variable "orca_default_bucket" { - default = "csda-cumulus-cba-uat-orca-archive" # TODO - Go to Disaster Recovery Account -} -variable "orca_reports_bucket_name" { - default = "csda-cumulus-cba-uat-orca-reports" -} - -# TODO - Remove these from here all together during the PR -# These have been moved to ssm_parameters.tf -# Leaving these here while this task is still a Work in Progress -# -#variable "s3_access_key" { -# default = "Axxx" -#} -#variable "s3_secret_key" { -# default = "Axxx" -#} - - - - diff --git a/app/stacks/rds-cluster/outputs.tf b/app/stacks/rds-cluster/outputs.tf index de9ebe3..4509213 100644 --- a/app/stacks/rds-cluster/outputs.tf +++ b/app/stacks/rds-cluster/outputs.tf @@ -17,27 +17,3 @@ output "security_group_id" { output "user_credentials_secret_arn" { value = module.rds_cluster.user_credentials_secret_arn } - -# How do we output the password so that the Cumulus module can read it via -# -# # jsondecode("<%= json_output('rds-cluster.db_admin_password') %>") -# # jsondecode("<%= json_output('rds-cluster.rds_user_password') %>") -# -# This does not work... why.. -# -# [2023-09-27T17:36:47 #29 terraspace up rds-cluster]: Error: Unsupported attribute -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: on outputs.tf line 27, in output "rds_cluster_db_admin_password": -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: 27: value = module.rds_cluster.db_admin_password -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: This object does not have an attribute named "db_admin_password". -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: -#[2023-09-27T17:36:47 #29 terraspace up rds-cluster]: -# -#output "rds_cluster_db_admin_password" { -# value = module.rds_cluster.db_admin_password -#} -## -#output "rds_cluster_rds_user_password" { -# value = module.rds_cluster.rds_user_password -#} diff --git a/bin/pre-commit-terraspace-fmt.sh b/bin/pre-commit-terraspace-fmt.sh new file mode 100755 index 0000000..4c84ce8 --- /dev/null +++ b/bin/pre-commit-terraspace-fmt.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# This is intended to be used as a pre-commit hook, and will thus fail if any +# `*.tf` files were reformatted so that pre-commit fails. +! make fmt | grep "\.tf\s*$" From 132f715af14406d273d3ec7cac3ad7e44301889e Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 5 Oct 2023 15:18:43 -0400 Subject: [PATCH 08/11] Remove 'expansion' from base.tfvars --- .pre-commit-config.yaml | 2 +- app/stacks/cumulus/tfvars/base.tfvars | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f24bb8a..30d1521 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,4 +6,4 @@ repos: entry: bin/pre-commit-terraspace-fmt.sh language: system pass_filenames: false - files: \.tf$ + files: \.tf(vars)?$ diff --git a/app/stacks/cumulus/tfvars/base.tfvars b/app/stacks/cumulus/tfvars/base.tfvars index 703b682..c19d876 100644 --- a/app/stacks/cumulus/tfvars/base.tfvars +++ b/app/stacks/cumulus/tfvars/base.tfvars @@ -22,12 +22,10 @@ system_bucket = "<%= bucket('internal') %>" buckets = { # https://nasa.github.io/cumulus-orca/docs/developer/deployment-guide/deployment-s3-bucket/ orca_reports = { - # name = "<%= expand('csda-cumulus-cba-:ENV-orca-reports') %>" name = "<%= %Q[csda-cumulus-cba-#{Terraspace.env == 'prod' ? 'prod' : 'uat'}-orca-reports] %>" type = "orca" } orca_default = { - # name = "<%= expand('csda-cumulus-cba-:ENV-orca-archive') %>" name = "<%= %Q[csda-cumulus-cba-#{Terraspace.env == 'prod' ? 'prod' : 'uat'}-orca-archive] %>" type = "orca" } From 11ced88077645d68719005a797b902ad1cd5b6ec Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Wed, 18 Oct 2023 14:08:25 -0400 Subject: [PATCH 09/11] Add troubleshooting for API Gateway creation --- docs/TROUBLESHOOTING.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5eafad2..c87e612 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1,6 +1,7 @@ # Troubleshooting - [Deployment](#deployment) + - [Error creating API Gateway Deployment: BadRequestException: Private REST API doesn't have a resource policy attached to it](#error-creating-api-gateway-deployment-badrequestexception-private-rest-api-doesnt-have-a-resource-policy-attached-to-it) - [Aws::STS::Errors::InvalidClientTokenId: The security token included in the request is invalid](#awsstserrorsinvalidclienttokenid-the-security-token-included-in-the-request-is-invalid) - [Error describing SSM parameter: ParameterNotFound](#error-describing-ssm-parameter-parameternotfound) - [Running "up" Command Stopped](#running-up-command-stopped) @@ -18,6 +19,25 @@ ## Deployment +### Error creating API Gateway Deployment: BadRequestException: Private REST API doesn't have a resource policy attached to it + +You might encounter an error similar to the following during deployment: + +```plain +Error: Error creating API Gateway Deployment: BadRequestException: Private REST API doesn't have a resource policy attached to it + + on .terraform/modules/orca/modules/api-gateway/main.tf line 498, in resource "aws_api_gateway_deployment" "orca_api_deployment": + 498: resource "aws_api_gateway_deployment" "orca_api_deployment" { +``` + +This is likely due to a race condition between resources, as Terraform often +creates several resources in parallel. + +The fix for this problem is simple: **Rerun your deployment command**, and by +the time Terraform again attempts to perform the previously failing operation, +it will succeed. If it fails again, rerun the deployment again, until you no +longer see the error. + ### Aws::STS::Errors::InvalidClientTokenId: The security token included in the request is invalid If you see output similar to the following when running an "up" or "plan" From f4c3ada25b0e194608dc97132bdff4bc977f1cd0 Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 19 Oct 2023 06:04:00 -0400 Subject: [PATCH 10/11] Set correct email address for ORCA DLQ subscription --- app/stacks/cumulus/ssm_parameters.tf | 41 +++++++++++++-------------- app/stacks/cumulus/tfvars/base.tfvars | 2 +- app/stacks/cumulus/tfvars/prod.tfvars | 2 -- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/app/stacks/cumulus/ssm_parameters.tf b/app/stacks/cumulus/ssm_parameters.tf index ce571d8..270c068 100644 --- a/app/stacks/cumulus/ssm_parameters.tf +++ b/app/stacks/cumulus/ssm_parameters.tf @@ -54,6 +54,25 @@ data "aws_ssm_parameter" "csdap_client_password" { name = "/shared/cumulus/csdap-client-password" } +# ORCA Bucket Access +# +# Currently, the buckets must be setup in the Disaster Recovery (DR) AWS +# accounts. There are only DR AWS accounts for CBA UAT and CBA PROD. +# +# Unfortunately, this parameter must be refreshed every time these keys expire. +# To refresh, do the following: +# +# 1. Make new long-term access keys +# 2. For each environment, run the following +# +# DOTENV=<.env file for UAT or Prod> make bash +# aws ssm put-parameter --name ACCESS_NAME --overwrite --value NEW_ACCESS_KEY +# aws ssm put-parameter --name SECRET_NAME --overwrite --value NEW_SECRET_KEY +# +# where ACCESS_NAME and SECRET_NAME are the `name` values in the respective +# SSM parameters below, and NEW_ACCESS_KEY and NEW_SECRET_KEY are the new +# values, respectively. + data "aws_ssm_parameter" "orca_s3_access_key" { name = "/shared/cumulus/orca/dr/s3-access-key" } @@ -92,28 +111,6 @@ data "aws_ssm_parameter" "metrics_aws_account_id" { name = "/shared/cumulus/metrics-aws-account-id" } -# ORCA Bucket Access - Note: As of now, the Buckets must be setup on the DR AWS accounts -# There are only DR AWS accounts for CBA UAT and CBA PROD -# -# Unfortunately, This parameter must be refreshed everytime these keys expire. -# To refresh, do the following -# (1) Make new keys -# (2) For each environment, run the following commands -# (2a) make bash -# (2b) aws ssm put-parameter --type SecureString --name NAME --overwrite --value VALUE -# -# Note, for setting the FIRST time, the command is slightly different (no --overwrite) -# # aws ssm put-parameter --type SecureString --name NAME --value VALUE -# TODO - add some of the above stuff to the proper documentation - -#data "aws_ssm_parameter" "orca_s3_access_key" { -# name = "/shared/cumulus/orca/dr/s3-access-key" -#} - -#data "aws_ssm_parameter" "orca_s3_secret_key" { -# name = "/shared/cumulus/orca/dr/s3-secret-key" -#} - # <% end %> #------------------------------------------------------------------------------- diff --git a/app/stacks/cumulus/tfvars/base.tfvars b/app/stacks/cumulus/tfvars/base.tfvars index c19d876..54d2f07 100644 --- a/app/stacks/cumulus/tfvars/base.tfvars +++ b/app/stacks/cumulus/tfvars/base.tfvars @@ -15,7 +15,7 @@ #<% depends_on("rds-cluster") %> cmr_environment = "UAT" -orca_dlq_subscription_email = "pic8690@gmail.com" +orca_dlq_subscription_email = "csdap@uah.edu" system_bucket = "<%= bucket('internal') %>" diff --git a/app/stacks/cumulus/tfvars/prod.tfvars b/app/stacks/cumulus/tfvars/prod.tfvars index fd07562..0352275 100644 --- a/app/stacks/cumulus/tfvars/prod.tfvars +++ b/app/stacks/cumulus/tfvars/prod.tfvars @@ -17,8 +17,6 @@ cumulus_distribution_url = "https://data.csdap.earthdata.nasa.gov/" cumulus_distribution_url = "https://data.csda.earthdata.nasa.gov/" # <% end %> -#orca_dlq_subscription_email = "" - metrics_es_host = "https://d23fzndssjmbvi.cloudfront.net/" # <% if in_cba? then %> From 4a64bbdcf214dbdba59740503d71f7dbfcf6a8de Mon Sep 17 00:00:00 2001 From: Chuck Daniels Date: Thu, 19 Oct 2023 07:37:08 -0400 Subject: [PATCH 11/11] Remove non-CBA UAT tfvar values --- app/stacks/cumulus/tfvars/uat.tfvars | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/app/stacks/cumulus/tfvars/uat.tfvars b/app/stacks/cumulus/tfvars/uat.tfvars index a4ab130..7c2193e 100644 --- a/app/stacks/cumulus/tfvars/uat.tfvars +++ b/app/stacks/cumulus/tfvars/uat.tfvars @@ -7,22 +7,10 @@ csdap_host_url = "https://auth.csdap.uat.earthdatacloud.nasa.gov/" -# <% if in_cba? then %> # Trailing slash is required cumulus_distribution_url = "https://data.csdap.uat.earthdata.nasa.gov/" -# <% else %> -# Trailing slash is required -cumulus_distribution_url = "https://data.csda.uat.earthdata.nasa.gov/" -# <% end %> - -#orca_dlq_subscription_email = "" metrics_es_host = "https://dmzza2al43z4f.cloudfront.net/" -# <% if in_cba? then %> s3_replicator_target_bucket = "cloud-metrics-inbound-uat-csdap-distribution" -# <% else %> -s3_replicator_target_bucket = "esdis-metrics-inbound-uat-csdap-distribution" -# <% end %> - s3_replicator_target_prefix = "input/s3_access/csdapuat"