parameterized mwaa reqs.txt

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - enforced TLS version 1.2, node-node encryption and encryption at rest on OS module
 - added `emr-serverless` module with unit-tests
 - added workflow entries to all IDF modules
+- made `requirements.txt` file of MWAA configurable via a user defined entry from module manifest file
 
 ### **Changed**
 

manifests/local/orchestration-modules.yaml

@@ -1,5 +1,8 @@
 name: mwaa
 path: modules/orchestration/mwaa/
+# Use the below if you want to customize a reqs.txt and provide the accurate path as a param
+dataFiles:
+  - filePath: data/mwaa/requirements/requirements-emr-serverless.txt
 parameters:
   - name: vpc-id
     valueFrom:
@@ -23,3 +26,5 @@ parameters:
     value: dags
   - name: airflow-version
     value: "2.5.1"
+  - name: custom-requirements-path
+    value: data/mwaa/requirements/requirements-emr-serverless.txt

modules/orchestration/mwaa/deployspec.yaml

@@ -10,11 +10,16 @@ deploy:
       - pushd requirements/ && pwd && popd
       - aws iam create-service-linked-role --aws-service-name airflow.amazonaws.com || true
       - aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com || true
-      - export REQUIREMENTS_FILE="requirements/${SEEDFARMER_PARAMETER_MWAA_REQUIREMENTS_FILE:=requirements.txt}"
-      - > 
-        if [[ ! -f ${REQUIREMENTS_FILE} ]]; then
-            export REQUIREMENTS_FILE="requirements/requirements.txt";
-        fi;
+      - >
+        if [ ! -z $SEEDFARMER_PARAMETER_CUSTOM_REQUIREMENTS_PATH ]; then 
+            if [ -f $SEEDFARMER_PARAMETER_CUSTOM_REQUIREMENTS_PATH ]; then 
+                export REQUIREMENTS_FILE=$(echo $SEEDFARMER_PARAMETER_CUSTOM_REQUIREMENTS_PATH | cut -d / -f 3-)
+                echo "Moving ${SEEDFARMER_PARAMETER_CUSTOM_REQUIREMENTS_PATH} to ${REQUIREMENTS_FILE}"
+                mv $SEEDFARMER_PARAMETER_CUSTOM_REQUIREMENTS_PATH $REQUIREMENTS_FILE
+            fi
+        else
+            export REQUIREMENTS_FILE="requirements/requirements.txt"
+        fi
       - export UNIQUE_REQUIREMENTS_FILE="${REQUIREMENTS_FILE}-$(date +%s)"
       - echo "Moving ${REQUIREMENTS_FILE} to ${UNIQUE_REQUIREMENTS_FILE}"
       - mv ${REQUIREMENTS_FILE} ${UNIQUE_REQUIREMENTS_FILE}