From 55e35850ce864fe648eb5452ed25ad760060040c Mon Sep 17 00:00:00 2001 From: James Noss Date: Thu, 14 Mar 2024 09:49:32 -0400 Subject: [PATCH 1/2] Add utility funcs to retrieve AWS secrets, parsed from apprunner.yaml Signed-off-by: James Noss --- README.md | 6 ++++++ apprunner.yaml | 1 + biospecdb/util.py | 25 +++++++++++++++++++++++++ pyproject.toml | 2 ++ requirements/prd.txt | 2 ++ scripts/prd/ec2_init.sh | 9 +++++++++ scripts/prd/export_secrets.sh | 35 +++++++++++++++++++++++++++++++++++ 7 files changed, 80 insertions(+) create mode 100755 scripts/prd/ec2_init.sh create mode 100755 scripts/prd/export_secrets.sh diff --git a/README.md b/README.md index 8ecc2a5a..f63d76c8 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,12 @@ The DB can be dumped to a file using the following: _NOTE: These commands must be run from the ``/app/`` directory on the server. +### AWS: + +The above management commands (and others) can be run in production from an EC2 instance correctly configured. To +aid in shell setup, the following script can be executed: + * ``source repo/biospecdb/scripts/prd/ec2_init.sh`` + # Usage ### URL Paths: diff --git a/apprunner.yaml b/apprunner.yaml index b219afd5..3fc28e35 100644 --- a/apprunner.yaml +++ b/apprunner.yaml @@ -25,6 +25,7 @@ run: - name: DJANGO_LOG_LEVEL value: "INFO" secrets: + # When adding/editing secrets, remember to also do so in scripts/prd/export_secrets.sh - name: DB_BSR_HOST value-from: "arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_HOST-Hq7aa9" - name: DB_ADMIN_HOST diff --git a/biospecdb/util.py b/biospecdb/util.py index 20d66053..9e036196 100644 --- a/biospecdb/util.py +++ b/biospecdb/util.py @@ -3,7 +3,9 @@ import os from pathlib import Path from uuid import UUID +import yaml +import boto3 from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ import numpy as np @@ -117,3 +119,26 @@ def get_object_or_raise_validation(obj, **kwargs): def get_field_value(series, obj, field, default=None): return series.get(getattr(obj, field).field.verbose_name.lower(), default=default) + + +def get_aws_secret(arn, region): + session = boto3.session.Session() + client = session.client(service_name='secretsmanager', region_name=region) + return client.get_secret_value(SecretId=arn)['SecretString'] + + +def parse_secure_secrets_from_apprunner(apprunner_yaml_file=find_repo_location() / "apprunner.yaml"): + with open(apprunner_yaml_file) as fp: + config = yaml.safe_load(fp) + return {list(d.values())[0]: list(d.values())[1] for d in config["run"]["secrets"]} + + +def get_aws_secrets(apprunner_yaml_file=find_repo_location() / "apprunner.yaml", region="eu-west-2", export=False): + secure_secrets = parse_secure_secrets_from_apprunner(apprunner_yaml_file) + unsecure_secrets = {k: get_aws_secret(v, region=region) for k, v in secure_secrets.items()} + + if export: + for k, v in unsecure_secrets: + os.environ[k] = v + + return unsecure_secrets diff --git a/pyproject.toml b/pyproject.toml index 0937f83e..fc06a16c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ license = {file = "LICENSE"} requires-python = ">=3.11" dependencies = [ + "boto3", "django", "django-crontab", "django-decorator-include", @@ -23,6 +24,7 @@ dependencies = [ "pandas", "plotly", "psycopg[binary,pool]", + "pyyaml", "whitenoise", "xlsxwriter" ] diff --git a/requirements/prd.txt b/requirements/prd.txt index 749035f9..d26332e5 100644 --- a/requirements/prd.txt +++ b/requirements/prd.txt @@ -1,3 +1,4 @@ +boto3==1.34.62 django==4.2.11 django-crontab==0.7.1 django-decorator-include==3.0 @@ -11,5 +12,6 @@ openpyxl==3.1.2 pandas==2.2.1 plotly==5.19.0 psycopg[binary,pool]==3.1.18 +pyyaml==6.0.1 xlsxwriter==3.2.0 whitenoise==6.6.0 diff --git a/scripts/prd/ec2_init.sh b/scripts/prd/ec2_init.sh new file mode 100755 index 00000000..07eb067c --- /dev/null +++ b/scripts/prd/ec2_init.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +conda activate biospecdb + +cd repo/biospecdb +git fetch +git checkout origin/main -f + +source ./scripts/prd/export_secrets.sh diff --git a/scripts/prd/export_secrets.sh b/scripts/prd/export_secrets.sh new file mode 100755 index 00000000..bd8a0fce --- /dev/null +++ b/scripts/prd/export_secrets.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# When adding/editing secrets, remember to also do so in apprunner.yaml + +get_secret() { + secret=$(aws secretsmanager get-secret-value --secret-id --query 'SecretString' $1) + secret="${secret%\"}" + secret="${secret#\"}" + echo "$secret" +} + +export DB_VENDOR=postgresql +export DJANGO_SETTINGS_MODULE=biospecdb.settings.aws + +export DB_BSR_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_HOST-Hq7aa9) +export DB_ADMIN_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_HOST-NTlDxu) +export DB_BSR_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_USER-EjV5uT) +export DB_BSR_PORT=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PORT-mADIze) +export DB_ADMIN_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_USER-n4WHdp) +export DB_ADMIN_PORT=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_PORT-M0eK1j) +export DB_BSR_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PASSWORD-xEoYOO) +export DB_ADMIN_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_PASSWORD-NdWuLS) +export SECRET_KEY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:SECRET_KEY-V3ABiP) +export DJANGO_SUPERUSER_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DJANGO_SUPERUSER_PASSWORD-baCj9f) +export N_WORKERS=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:N_WORKERS-HWBWTb) +export AWS_STORAGE_BUCKET_NAME=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:AWS_STORAGE_BUCKET_NAME-s1Fcyv) +export AWS_S3_REGION_NAME=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:AWS_S3_REGION_NAME-o1dcFN) +export SESSION_COOKIE_AGE=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:SESSION_COOKIE_AGE-mCp6jQ) +export HOST_DOMAIN=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:HOST_DOMAIN-VzvGf8) +export DB_BSR_PASSWORD_READONLY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PASSWORD_READONLY-qFuKvJ) +export DB_BSR_USER_READONLY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_USER_READONLY-wtGBhZ) +export EMAIL_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST-x2J4ik) +export EMAIL_HOST_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST_USER-s7GAZD) +export EMAIL_HOST_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST_PASSWORD-kPI0AW) +export EMAIL_SUBJECT_PREFIX=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_SUBJECT_PREFIX-9oWLpA) From 99eeb61773892e08b18d4bd037416d9d4b6259a0 Mon Sep 17 00:00:00 2001 From: James Noss Date: Fri, 15 Mar 2024 14:48:12 -0400 Subject: [PATCH 2/2] Post review changes Signed-off-by: James Noss --- README.md | 2 ++ apprunner.yaml | 1 - biospecdb/util.py | 5 +++++ scripts/prd/ec2_init.sh | 5 ++++- scripts/prd/export_secrets.sh | 35 ----------------------------------- 5 files changed, 11 insertions(+), 37 deletions(-) delete mode 100755 scripts/prd/export_secrets.sh diff --git a/README.md b/README.md index f63d76c8..47036371 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,8 @@ The above management commands (and others) can be run in production from an EC2 aid in shell setup, the following script can be executed: * ``source repo/biospecdb/scripts/prd/ec2_init.sh`` +_NOTE: ``scripts/prd/ec2_init.sh`` will export all AWS secrets as shell environment variables. + # Usage ### URL Paths: diff --git a/apprunner.yaml b/apprunner.yaml index 3fc28e35..b219afd5 100644 --- a/apprunner.yaml +++ b/apprunner.yaml @@ -25,7 +25,6 @@ run: - name: DJANGO_LOG_LEVEL value: "INFO" secrets: - # When adding/editing secrets, remember to also do so in scripts/prd/export_secrets.sh - name: DB_BSR_HOST value-from: "arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_HOST-Hq7aa9" - name: DB_ADMIN_HOST diff --git a/biospecdb/util.py b/biospecdb/util.py index 9e036196..425d36ce 100644 --- a/biospecdb/util.py +++ b/biospecdb/util.py @@ -142,3 +142,8 @@ def get_aws_secrets(apprunner_yaml_file=find_repo_location() / "apprunner.yaml", os.environ[k] = v return unsecure_secrets + + +def print_aws_secrets(apprunner_yaml_file=find_repo_location() / "apprunner.yaml", region="eu-west-2"): + for k, v in get_aws_secrets(apprunner_yaml_file, region).items(): + print(f"{k}={v}") diff --git a/scripts/prd/ec2_init.sh b/scripts/prd/ec2_init.sh index 07eb067c..7223a19d 100755 --- a/scripts/prd/ec2_init.sh +++ b/scripts/prd/ec2_init.sh @@ -6,4 +6,7 @@ cd repo/biospecdb git fetch git checkout origin/main -f -source ./scripts/prd/export_secrets.sh +export DB_VENDOR=postgresql +export DJANGO_SETTINGS_MODULE=biospecdb.settings.aws + +export $(python -c "from biospecdb.util import print_aws_secrets;print_aws_secrets()") diff --git a/scripts/prd/export_secrets.sh b/scripts/prd/export_secrets.sh deleted file mode 100755 index bd8a0fce..00000000 --- a/scripts/prd/export_secrets.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# When adding/editing secrets, remember to also do so in apprunner.yaml - -get_secret() { - secret=$(aws secretsmanager get-secret-value --secret-id --query 'SecretString' $1) - secret="${secret%\"}" - secret="${secret#\"}" - echo "$secret" -} - -export DB_VENDOR=postgresql -export DJANGO_SETTINGS_MODULE=biospecdb.settings.aws - -export DB_BSR_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_HOST-Hq7aa9) -export DB_ADMIN_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_HOST-NTlDxu) -export DB_BSR_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_USER-EjV5uT) -export DB_BSR_PORT=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PORT-mADIze) -export DB_ADMIN_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_USER-n4WHdp) -export DB_ADMIN_PORT=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_PORT-M0eK1j) -export DB_BSR_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PASSWORD-xEoYOO) -export DB_ADMIN_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_ADMIN_PASSWORD-NdWuLS) -export SECRET_KEY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:SECRET_KEY-V3ABiP) -export DJANGO_SUPERUSER_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DJANGO_SUPERUSER_PASSWORD-baCj9f) -export N_WORKERS=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:N_WORKERS-HWBWTb) -export AWS_STORAGE_BUCKET_NAME=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:AWS_STORAGE_BUCKET_NAME-s1Fcyv) -export AWS_S3_REGION_NAME=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:AWS_S3_REGION_NAME-o1dcFN) -export SESSION_COOKIE_AGE=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:SESSION_COOKIE_AGE-mCp6jQ) -export HOST_DOMAIN=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:HOST_DOMAIN-VzvGf8) -export DB_BSR_PASSWORD_READONLY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_PASSWORD_READONLY-qFuKvJ) -export DB_BSR_USER_READONLY=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:DB_BSR_USER_READONLY-wtGBhZ) -export EMAIL_HOST=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST-x2J4ik) -export EMAIL_HOST_USER=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST_USER-s7GAZD) -export EMAIL_HOST_PASSWORD=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_HOST_PASSWORD-kPI0AW) -export EMAIL_SUBJECT_PREFIX=$(get_secret arn:aws:secretsmanager:eu-west-2:339712857727:secret:EMAIL_SUBJECT_PREFIX-9oWLpA)