Merge pull request #231 from NASA-IMPACT/feature/mwaa-2-sm2a-pr-1

Clean up SM2A deployment before MWAA to SM2A migration

.github/actions/terraform-deploy-sm2a/action.yml

@@ -39,6 +39,7 @@ runs:
         terraform_version: 1.3.3
 
     - name: Deploy
+      working-directory: ${{ inputs.dir }}
       shell: bash
       env:
         AWS_DEFAULT_REGION: ${{ inputs.aws-region }}

dags/generate_dags.py

@@ -11,16 +11,28 @@
 def generate_dags():
     import boto3
     import json
+    from botocore.exceptions import ClientError, NoCredentialsError
 
     from pathlib import Path
 
 
     mwaa_stac_conf = Variable.get("MWAA_STACK_CONF", deserialize_json=True)
     bucket = mwaa_stac_conf["EVENT_BUCKET"]
 
-    client = boto3.client("s3")
-    response = client.list_objects_v2(Bucket=bucket, Prefix="collections/")
-
+    try:
+        client = boto3.client("s3")
+        response = client.list_objects_v2(Bucket=bucket, Prefix="collections/")
+    except ClientError as e:
+        # Handle general AWS service errors (e.g., wrong bucket name)
+        print(f"ClientError: {e}")
+        return
+    except NoCredentialsError:
+        # Handle missing credentials
+        print("Credentials not found.")
+        return
+    except Exception as ex:
+        print(f"An unexpected error occurred: {ex}")
+        return
     for file_ in response.get("Contents", []):
         key = file_["Key"]
         if key.endswith("/"):

sm2a/Makefile

@@ -1,6 +1,6 @@
 SECRET_NAME=""
 ENV_FILE=".env"
-SHELL=/bin/bash
+GITHUB_ACTIONS_ENV := $(shell test -n "$$GITHUB_ACTIONS" && echo "true" || echo "false")
 
 important_message = \
 	@echo "\033[0;31m$(1) \033[0m"
@@ -49,13 +49,17 @@ sm2a-local-build:
 	rm -rf dags
 
 sm2a-deploy:
+ifeq ($(GITHUB_ACTIONS_ENV),true)
 	@echo "Installing the deployment dependency"
 	pip install -r ../deploy_requirements.txt
 	@echo "Deploying SM2A"
 	cp -r ../dags .
 	python scripts/generate_env_file.py --secret-id ${SECRET_NAME} --env-file ${ENV_FILE}
-	./scripts/deploy.sh ${ENV_FILE} <<< init
-	./scripts/deploy.sh ${ENV_FILE} <<< deploy
+	@bash -c './scripts/deploy.sh ${ENV_FILE} <<< init'
+	@bash -c './scripts/deploy.sh ${ENV_FILE} <<< deploy'
+else
+	$(call important_message, "Wait a minute you are not github 😡")
+endif
 
 clean: sm2a-local-stop
 	@echo "Cleaning local env"

sm2a/infrastructure/main.tf

@@ -19,7 +19,8 @@ resource "random_password" "password" {
 
 
 module "sma-base" {
-  source                         =  "https://github.com/NASA-IMPACT/self-managed-apache-airflow/releases/download/v1.1.3/self-managed-apache-airflow.zip"
+  source                         = "https://github.com/NASA-IMPACT/self-managed-apache-airflow/releases/download/v1.1.4/self-managed-apache-airflow.zip"
+  project                        = var.project_name
   airflow_db                     = var.airflow_db
   fernet_key                     = var.fernet_key
   prefix                         = var.prefix
@@ -43,7 +44,7 @@ module "sma-base" {
   rds_allocated_storage          = var.rds_configuration[var.stage].rds_allocated_storage
   rds_max_allocated_storage      = var.rds_configuration[var.stage].rds_max_allocated_storage
   workers_logs_retention_days    = var.workers_configuration[var.stage].workers_logs_retention_days
-  airflow_custom_variables = var.airflow_custom_variables
+  airflow_custom_variables       = var.airflow_custom_variables
 
   extra_airflow_task_common_environment = [
     {

sm2a/infrastructure/variables.tf

@@ -181,9 +181,9 @@ variable "custom_worker_policy_statement" {
     {
       Effect = "Allow"
       Action = [
-				"sts:AssumeRole",
-				"iam:PassRole",
-				"logs:GetLogEvents"
+        "sts:AssumeRole",
+        "iam:PassRole",
+        "logs:GetLogEvents"
       ]
       "Resource" : [
         "*"
@@ -197,6 +197,10 @@ variable "custom_worker_policy_statement" {
 variable "airflow_custom_variables" {
   description = "Airflow custom variables"
   type        = map(string)
-  default = {}
+  default     = {}
+}
+variable "project_name" {
+  type    = string
+  default = "SM2A"
 }