diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6a6d7a40..d947a091 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -240,4 +240,25 @@ jobs: - name: Static checks and linting (mypy, flake8, black, isort) run: scripts/validate.sh --language python --path modules/ml/sagemaker-studio/ - name: Pytest - run: cd modules/ml/sagemaker-studio/ && pytest \ No newline at end of file + run: cd modules/ml/sagemaker-studio/ && pytest + + modules-service-catalog-app-registry: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.9] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install Requirements + run: | + python -m pip install --upgrade pip + pip install -r requirements-dev.txt + pip install -r modules/service-catalog/app-registry/requirements.txt + - name: Static checks and linting (mypy, flake8, black, isort) + run: scripts/validate.sh --language python --path modules/service-catalog/app-registry/ + - name: Pytest + run: cd modules/service-catalog/app-registry/ && pytest \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4963d767..5c7f94e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - added `emr-serverless` module with unit-tests - added workflow entries to all IDF modules - made `requirements.txt` file of MWAA configurable via a user defined entry from module manifest file +- added `app-registry` module for being able to scrape app-specific CloudFormation stacks for AWS Solutions ### **Changed** diff --git a/manifests/local/catalog-modules.yaml b/manifests/local/catalog-modules.yaml new file mode 100644 index 00000000..e3b03cc0 --- /dev/null +++ b/manifests/local/catalog-modules.yaml @@ -0,0 +1,9 @@ +name: app-reg +path: modules/service-catalog/app-registry +parameters: + - name: solution-id + value: id + - name: solution-name + value: name + - name: solution-version + value: version \ No newline at end of file diff --git a/modules/service-catalog/app-registry/README.md b/modules/service-catalog/app-registry/README.md new file mode 100644 index 00000000..b7e08aea --- /dev/null +++ b/modules/service-catalog/app-registry/README.md @@ -0,0 +1,45 @@ +# AWS Service Catalog - App Regitsry resources + +## Description + +You can consider deploying this module, if you are working on creating an AWS Solution. One of the requirements for creating an AWS solution is being able to track the CloudFormation stacks using AWS Service catalog - AppRegistry resource. + +This module: + +- Creates an AppRegistry application resource +- It also joins the CloudFormation stacks created externally into the AppRegistry application using boto3 + +## Inputs/Outputs + +### Input Parameters + +#### Required + +- `solution-id`: The solution ID for the AWS Solution +- `solution-name`: The solution Name for the AWS Solution +- `solution-version`: The solution Version for the AWS Solution + +### Sample declaration of AWS Batch Compute Configuration + +```yaml +parameters: + - name: solution-id + value: id + - name: solution-name + value: name + - name: solution-version + value: version +``` + +#### Optional + +### Module Metadata Outputs + +- `AppRegistryName`: Service Catalog - AppRegistry name +- `AttributeGroupName`: Service Catalog - Attribute group name + +#### Output Example + +```json +{"AppRegistryName":"addf-aws-solutions-wip-catalog-app-reg-AppRegistryApp","AttributeGroupName":"addf-aws-solutions-wip-catalog-app-reg-AppAttributeGroup"} +``` diff --git a/modules/service-catalog/app-registry/app.py b/modules/service-catalog/app-registry/app.py new file mode 100755 index 00000000..74ecd319 --- /dev/null +++ b/modules/service-catalog/app-registry/app.py @@ -0,0 +1,68 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + +from aws_cdk import App, CfnOutput, Environment + +from stack import AppRegistry + +# Project specific +project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "") +deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "") +module_name = os.getenv("SEEDFARMER_MODULE_NAME", "") + + +if len(f"{project_name}-{deployment_name}") > 36: + raise ValueError("This module cannot support a project+deployment name character length greater than 35") + + +def _param(name: str) -> str: + return f"SEEDFARMER_PARAMETER_{name}" + + +# App specific +solution_id = os.getenv(_param("SOLUTION_ID")) # required +solution_name = os.getenv(_param("SOLUTION_NAME")) # required +solution_version = os.getenv(_param("SOLUTION_VERSION")) # required + + +if not solution_id: + raise ValueError("Missing input parameter solution-id") + +if not solution_name: + raise ValueError("Missing input parameter solution-name") + +if not solution_version: + raise ValueError("Missing input parameter solution-version") + + +app = App() + +stack = AppRegistry( + scope=app, + id=f"{project_name}-{deployment_name}-{module_name}", + project_name=project_name, + deployment_name=deployment_name, + module_name=module_name, + solution_id=solution_id, + solution_name=solution_name, + solution_version=solution_version, + env=Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), +) + +CfnOutput( + scope=stack, + id="metadata", + value=stack.to_json_string( + { + "AppRegistryName": stack.app_registry.name, + "AttributeGroupName": stack.attribute_group.name, + } + ), +) + +app.synth(force=True) diff --git a/modules/service-catalog/app-registry/coverage.ini b/modules/service-catalog/app-registry/coverage.ini new file mode 100644 index 00000000..fe58f5a0 --- /dev/null +++ b/modules/service-catalog/app-registry/coverage.ini @@ -0,0 +1,4 @@ +[run] +omit = + tests/* + register-stacks.py \ No newline at end of file diff --git a/modules/service-catalog/app-registry/deployspec.yaml b/modules/service-catalog/app-registry/deployspec.yaml new file mode 100644 index 00000000..95d0c32e --- /dev/null +++ b/modules/service-catalog/app-registry/deployspec.yaml @@ -0,0 +1,25 @@ +publishGenericEnvVariables: true +deploy: + phases: + install: + commands: + - npm install -g aws-cdk@2.70.0 + - pip install -r requirements.txt + build: + commands: + - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json + # Export metadata + - export SEEDFARMER_MODULE_METADATA=$(python -c "import json; file=open('cdk-exports.json'); print(json.load(file)['${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}']['metadata'])") + # Associate Solution CFN stacks to AppRegistry + - python register-stacks.py associate +destroy: + phases: + install: + commands: + - npm install -g aws-cdk@2.70.0 + - pip install -r requirements.txt + build: + commands: + # Dissociate Solution CFN stacks to AppRegistry + - python register-stacks.py dissociate + - cdk destroy --force --app "python app.py" \ No newline at end of file diff --git a/modules/service-catalog/app-registry/modulestack.yaml b/modules/service-catalog/app-registry/modulestack.yaml new file mode 100644 index 00000000..60c9f2d3 --- /dev/null +++ b/modules/service-catalog/app-registry/modulestack.yaml @@ -0,0 +1,37 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: This template deploys a Module specific IAM permissions + +Parameters: + # ProjectName: + # Type: String + # Description: The name of the project + # DeploymentName: + # Type: String + # Description: The name of the deployment + # ModuleName: + # Type: String + # Description: The name of the Module + RoleName: + Type: String + Description: The name of the IAM Role + +Resources: + Policy: + Type: "AWS::IAM::Policy" + Properties: + PolicyDocument: + Statement: + - Action: + - "cloudformation:List*" + - "servicecatalog:Get*" + Effect: Allow + Resource: "*" + - Action: + - "servicecatalog:AssociateResource" + - "servicecatalog:DisassociateResource" + Effect: Allow + Resource: + - !Sub "arn:aws:servicecatalog:${AWS::Region}:${AWS::AccountId}:/applications/*" + Version: 2012-10-17 + PolicyName: idf-modulespecific-policy + Roles: [!Ref RoleName] \ No newline at end of file diff --git a/modules/service-catalog/app-registry/pyproject.toml b/modules/service-catalog/app-registry/pyproject.toml new file mode 100644 index 00000000..361877d5 --- /dev/null +++ b/modules/service-catalog/app-registry/pyproject.toml @@ -0,0 +1,35 @@ +[tool.black] +line-length = 120 +target-version = ["py36", "py37", "py38", "py39"] +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | \.env + | _build + | buck-out + | build + | dist + | codeseeder.out +)/ +''' + +[tool.isort] +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +py_version = 38 +skip_gitignore = false + +[tool.pytest.ini_options] +addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80" +pythonpath = [ + "." +] \ No newline at end of file diff --git a/modules/service-catalog/app-registry/register-stacks.py b/modules/service-catalog/app-registry/register-stacks.py new file mode 100644 index 00000000..71fc875e --- /dev/null +++ b/modules/service-catalog/app-registry/register-stacks.py @@ -0,0 +1,79 @@ +#!/usr/env/bin python + +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Registers CloudFormation stacks into AppRegistry""" + +import json +import os +import sys +from typing import List + +import boto3 +from botocore.exceptions import ClientError + +APPREG_CLIENT = boto3.client("servicecatalog-appregistry") +CFN_CLIENT = boto3.client("cloudformation") +PROJECT_NAME = os.getenv("SEEDFARMER_PROJECT_NAME", "addf") +DEP_NAME = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "aws-solutions-wip") +APP_REG_NAME = json.loads(os.getenv("SEEDFARMER_MODULE_METADATA"))["AppRegistryName"] # type: ignore +ACTION = sys.argv[1] + + +def main() -> None: + """Driver function""" + stacks_tobe_registsred = _list_stacks(prefix=f"{PROJECT_NAME}-{DEP_NAME}") + + if ACTION == "associate": + _asaociate_stacks(stacks=stacks_tobe_registsred) + else: + _dissaociate_stacks(stacks=stacks_tobe_registsred) + + +def _asaociate_stacks(stacks: List[str]) -> None: + """Associate CloudFormation Stacks to App Registry""" + for stack in stacks: + try: + APPREG_CLIENT.associate_resource(application=APP_REG_NAME, resourceType="CFN_STACK", resource=stack) + print(f"Associated the stack: {stack} successfully") + except ClientError as ex: + if ex.response["Error"]["Code"] == "ResourceNotFoundException": + print(f"Could not find the stack: {stack} associated") + continue + else: + raise ex + + +def _dissaociate_stacks(stacks: List[str]) -> None: + """Dissociate CloudFormation Stacks from App Registry""" + + for stack in stacks: + try: + APPREG_CLIENT.disassociate_resource(application=APP_REG_NAME, resourceType="CFN_STACK", resource=stack) + print(f"Disassociated the stack: {stack} successfully") + except ClientError as ex: + if ex.response["Error"]["Code"] == "ResourceNotFoundException": + print(f"Could not find the stack: {stack} associated") + continue + else: + raise ex + + +def _list_stacks(prefix: str) -> List[str]: + """List CloudFormation Stacks by the desired prefix""" + + stacks_tobe_registsred = [] + + response = CFN_CLIENT.list_stacks(StackStatusFilter=["CREATE_COMPLETE"]) + + for stack in response["StackSummaries"]: + if stack["StackName"].startswith(prefix): + stacks_tobe_registsred.append(stack["StackName"]) + + print("The list of stacks: {}".format(stacks_tobe_registsred)) + return stacks_tobe_registsred + + +if __name__ == "__main__": + main() diff --git a/modules/service-catalog/app-registry/requirements.in b/modules/service-catalog/app-registry/requirements.in new file mode 100644 index 00000000..dd2e0790 --- /dev/null +++ b/modules/service-catalog/app-registry/requirements.in @@ -0,0 +1,4 @@ +aws-cdk-lib==2.70.0 +cdk-nag==2.12.29 +constructs==10.0.91 +boto3~=1.21.0 diff --git a/modules/service-catalog/app-registry/requirements.txt b/modules/service-catalog/app-registry/requirements.txt new file mode 100644 index 00000000..30304bbf --- /dev/null +++ b/modules/service-catalog/app-registry/requirements.txt @@ -0,0 +1,83 @@ +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile requirements.in +# +attrs==23.1.0 + # via + # cattrs + # jsii +aws-cdk-asset-awscli-v1==2.2.200 + # via aws-cdk-lib +aws-cdk-asset-kubectl-v20==2.1.2 + # via aws-cdk-lib +aws-cdk-asset-node-proxy-agent-v5==2.0.166 + # via aws-cdk-lib +aws-cdk-lib==2.70.0 + # via + # -r requirements.in + # cdk-nag +boto3==1.21.46 + # via -r requirements.in +botocore==1.24.46 + # via + # boto3 + # s3transfer +cattrs==23.1.2 + # via jsii +cdk-nag==2.12.29 + # via -r requirements.in +constructs==10.0.91 + # via + # -r requirements.in + # aws-cdk-lib + # cdk-nag +exceptiongroup==1.1.3 + # via cattrs +importlib-resources==6.0.1 + # via jsii +jmespath==1.0.1 + # via + # boto3 + # botocore +jsii==1.88.0 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v5 + # aws-cdk-lib + # cdk-nag + # constructs +publication==0.0.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v5 + # aws-cdk-lib + # cdk-nag + # constructs + # jsii +python-dateutil==2.8.2 + # via + # botocore + # jsii +s3transfer==0.5.2 + # via boto3 +six==1.16.0 + # via python-dateutil +typeguard==2.13.3 + # via + # aws-cdk-asset-awscli-v1 + # aws-cdk-asset-kubectl-v20 + # aws-cdk-asset-node-proxy-agent-v5 + # aws-cdk-lib + # jsii +typing-extensions==4.7.1 + # via + # cattrs + # jsii +urllib3==1.26.16 + # via botocore +zipp==3.16.2 + # via importlib-resources diff --git a/modules/service-catalog/app-registry/setup.cfg b/modules/service-catalog/app-registry/setup.cfg new file mode 100644 index 00000000..c9c7a8df --- /dev/null +++ b/modules/service-catalog/app-registry/setup.cfg @@ -0,0 +1,29 @@ +[metadata] +license_files = + LICENSE + NOTICE + VERSION + +[flake8] +max-line-length = 120 +extend-ignore = E203, W503 +exclude = + .git, + __pycache__, + docs/source/conf.py, + old, + build, + dist, + .venv, + codeseeder.out, + bundle + tests + +[mypy] +python_version = 3.7 +strict = True +ignore_missing_imports = True +allow_untyped_decorators = True +exclude = + codeseeder.out/|example/|tests/ +warn_unused_ignores = False diff --git a/modules/service-catalog/app-registry/stack.py b/modules/service-catalog/app-registry/stack.py new file mode 100755 index 00000000..cc8bc7c1 --- /dev/null +++ b/modules/service-catalog/app-registry/stack.py @@ -0,0 +1,117 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import logging +from typing import Any, cast + +import cdk_nag +from aws_cdk import Aspects, Aws, Stack, Tags +from aws_cdk import aws_servicecatalogappregistry as appregistry +from cdk_nag import NagPackSuppression, NagSuppressions +from constructs import Construct, IConstruct + +_logger: logging.Logger = logging.getLogger(__name__) + + +class AppRegistry(Stack): + def __init__( + self, + scope: Construct, + id: str, + *, + project_name: str, + deployment_name: str, + module_name: str, + solution_id: str, + solution_name: str, + solution_version: str, + **kwargs: Any, + ) -> None: + super().__init__( + scope, + id, + description="Deploy AWS AppRegistry to visualize resources related to an AWS Solution", + **kwargs, + ) + + dep_mod = f"{project_name}-{deployment_name}-{module_name}" + + # used to tag AWS resources. Tag Value length cant exceed 256 characters + full_dep_mod = dep_mod[:256] if len(dep_mod) > 256 else dep_mod + """ + dep_mod is used to name OpenSearch domain and the max length cant exceed 28 characters + https://docs.aws.amazon.com/opensearch-service/latest/developerguide/createupdatedomains.html + """ + + Tags.of(scope=cast(IConstruct, self)).add(key="Deployment", value=full_dep_mod) + + self.solution_name = solution_name + self.solution_id = solution_id + self.solution_version = solution_version + + # App registry + self.app_registry = appregistry.CfnApplication( + self, + f"{full_dep_mod}-AppRegistryApp", + name=f"{full_dep_mod}-AppRegistryApp", + description=f"Service Catalog app to visualize resources for the solution {self.solution_name}", + tags={ + "Solutions:SolutionID": self.solution_id, + "Solutions:SolutionName": self.solution_name, + "Solutions:SolutionVersion": self.solution_version, + }, + ) + + # Attributes group + self.attribute_group = appregistry.CfnAttributeGroup( + self, + f"{full_dep_mod}-AppAttributeGroup", + name=f"{full_dep_mod}-AppAttributeGroup", + description="Attributes for Solutions Metadata", + attributes={ + "version": self.solution_version, + "solutionID": self.solution_id, + "solutionName": self.solution_name, + }, + ) + + # Attribute association + attribute_group_association = appregistry.CfnAttributeGroupAssociation( + self, + f"{full_dep_mod}-AttributeGroupAssociation", + application=self.app_registry.name, + attribute_group=self.attribute_group.name, + ) + attribute_group_association.node.add_dependency(self.app_registry) + attribute_group_association.node.add_dependency(self.attribute_group) + + # Stack association + cfn_resource_association = appregistry.CfnResourceAssociation( + self, + f"{full_dep_mod}-AppResourceAssociation", + application=self.app_registry.name, + resource=Aws.STACK_NAME, + resource_type="CFN_STACK", + ) + cfn_resource_association.node.add_dependency(self.app_registry) + + Aspects.of(self).add(cdk_nag.AwsSolutionsChecks()) + + NagSuppressions.add_stack_suppressions( + self, + apply_to_nested_stacks=True, + suppressions=[ + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM4", + "reason": "Managed Policies are for service account roles only", + } + ), + NagPackSuppression( + **{ + "id": "AwsSolutions-IAM5", + "reason": "Resource access restriced to IDF resources", + } + ), + ], + ) diff --git a/modules/service-catalog/app-registry/tests/__init__.py b/modules/service-catalog/app-registry/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/modules/service-catalog/app-registry/tests/test_app.py b/modules/service-catalog/app-registry/tests/test_app.py new file mode 100644 index 00000000..89e84fe5 --- /dev/null +++ b/modules/service-catalog/app-registry/tests/test_app.py @@ -0,0 +1,62 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import pytest + + +@pytest.fixture(scope="function") +def stack_defaults(): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project" + os.environ["SEEDFARMER_DEPLOYMENT_NAME"] = "test-deployment" + os.environ["SEEDFARMER_MODULE_NAME"] = "test-module" + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + os.environ["SEEDFARMER_PARAMETER_SOLUTION_ID"] = "sid" + os.environ["SEEDFARMER_PARAMETER_SOLUTION_NAME"] = "sname" + os.environ["SEEDFARMER_PARAMETER_SOLUTION_VERSION"] = "sversion" + + # Unload the app import so that subsequent tests don't reuse + if "app" in sys.modules: + del sys.modules["app"] + + +def test_app(stack_defaults): + import app # noqa: F401 + + +def test_project_deployment_name_length(stack_defaults): + os.environ["SEEDFARMER_PROJECT_NAME"] = "test-project-incredibly" + + with pytest.raises(Exception) as e: + import app # noqa: F401 + assert "module cannot support a project+deployment name character length greater than" in str(e) + + +def test_solution_id(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_SOLUTION_ID"] + + with pytest.raises(Exception): + import app # noqa: F401 + + assert os.environ["SEEDFARMER_PARAMETER_SOLUTION_ID"] == "sid" + + +def test_solution_name(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_SOLUTION_NAME"] + + with pytest.raises(Exception): + import app # noqa: F401 + + assert os.environ["SEEDFARMER_PARAMETER_SOLUTION_NAME"] == "sname" + + +def test_solution_version(stack_defaults): + del os.environ["SEEDFARMER_PARAMETER_SOLUTION_VERSION"] + + with pytest.raises(Exception): + import app # noqa: F401 + + assert os.environ["SEEDFARMER_PARAMETER_SOLUTION_VERSION"] == "sversion" diff --git a/modules/service-catalog/app-registry/tests/test_stack.py b/modules/service-catalog/app-registry/tests/test_stack.py new file mode 100644 index 00000000..d65bb7e3 --- /dev/null +++ b/modules/service-catalog/app-registry/tests/test_stack.py @@ -0,0 +1,50 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys + +import aws_cdk as cdk +import pytest +from aws_cdk.assertions import Template + + +@pytest.fixture(scope="function") +def stack_defaults(): + os.environ["CDK_DEFAULT_ACCOUNT"] = "111111111111" + os.environ["CDK_DEFAULT_REGION"] = "us-east-1" + + # Unload the app import so that subsequent tests don't reuse + + if "stack" in sys.modules: + del sys.modules["stack"] + + +def test_synthesize_stack(stack_defaults): + import stack + + app = cdk.App() + project_name = "test-project" + dep_name = "test-deployment" + mod_name = "test-module" + + app_reg_stack = stack.AppRegistry( + scope=app, + id=f"{project_name}-{dep_name}-{mod_name}", + project_name=project_name, + deployment_name=dep_name, + module_name=mod_name, + solution_id="sid", + solution_name="sname", + solution_version="sversion", + env=cdk.Environment( + account=os.environ["CDK_DEFAULT_ACCOUNT"], + region=os.environ["CDK_DEFAULT_REGION"], + ), + ) + + template = Template.from_stack(app_reg_stack) + template.resource_count_is("AWS::ServiceCatalogAppRegistry::Application", 1) + template.resource_count_is("AWS::ServiceCatalogAppRegistry::AttributeGroup", 1) + template.resource_count_is("AWS::ServiceCatalogAppRegistry::AttributeGroupAssociation", 1) + template.resource_count_is("AWS::ServiceCatalogAppRegistry::ResourceAssociation", 1)