Merge pull request #55 from srinivasreddych/feat/app-registry

added app-registry module

.github/workflows/main.yml

@@ -240,4 +240,25 @@ jobs:
       - name: Static checks and linting (mypy, flake8, black, isort)
         run: scripts/validate.sh --language python --path modules/ml/sagemaker-studio/
       - name: Pytest
-        run: cd modules/ml/sagemaker-studio/ && pytest
+        run: cd modules/ml/sagemaker-studio/ && pytest
+
+  modules-service-catalog-app-registry:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+          pip install -r modules/service-catalog/app-registry/requirements.txt
+      - name: Static checks and linting (mypy, flake8, black, isort)
+        run: scripts/validate.sh --language python --path modules/service-catalog/app-registry/
+      - name: Pytest
+        run: cd modules/service-catalog/app-registry/ && pytest

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - added `emr-serverless` module with unit-tests
 - added workflow entries to all IDF modules
 - made `requirements.txt` file of MWAA configurable via a user defined entry from module manifest file
+- added `app-registry` module for being able to scrape app-specific CloudFormation stacks for AWS Solutions
 
 ### **Changed**
 

manifests/local/catalog-modules.yaml

@@ -0,0 +1,9 @@
+name: app-reg
+path: modules/service-catalog/app-registry
+parameters:
+  - name: solution-id
+    value: id
+  - name: solution-name
+    value: name
+  - name: solution-version
+    value: version

modules/service-catalog/app-registry/README.md

@@ -0,0 +1,45 @@
+# AWS Service Catalog - App Regitsry resources
+
+## Description
+
+You can consider deploying this module, if you are working on creating an AWS Solution. One of the requirements for creating an AWS solution is being able to track the CloudFormation stacks using AWS Service catalog - AppRegistry resource.
+
+This module:
+
+- Creates an AppRegistry application resource
+- It also joins the CloudFormation stacks created externally into the AppRegistry application using boto3
+
+## Inputs/Outputs
+
+### Input Parameters
+
+#### Required
+
+- `solution-id`: The solution ID for the AWS Solution
+- `solution-name`: The solution Name for the AWS Solution
+- `solution-version`: The solution Version for the AWS Solution
+
+### Sample declaration of AWS Batch Compute Configuration
+
+```yaml
+parameters:
+  - name: solution-id
+    value: id
+  - name: solution-name
+    value: name
+  - name: solution-version
+    value: version
+```
+
+#### Optional
+
+### Module Metadata Outputs
+
+- `AppRegistryName`: Service Catalog - AppRegistry name
+- `AttributeGroupName`: Service Catalog - Attribute group name
+
+#### Output Example
+
+```json
+{"AppRegistryName":"addf-aws-solutions-wip-catalog-app-reg-AppRegistryApp","AttributeGroupName":"addf-aws-solutions-wip-catalog-app-reg-AppAttributeGroup"}
+```

modules/service-catalog/app-registry/app.py

@@ -0,0 +1,68 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+from aws_cdk import App, CfnOutput, Environment
+
+from stack import AppRegistry
+
+# Project specific
+project_name = os.getenv("SEEDFARMER_PROJECT_NAME", "")
+deployment_name = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "")
+module_name = os.getenv("SEEDFARMER_MODULE_NAME", "")
+
+
+if len(f"{project_name}-{deployment_name}") > 36:
+    raise ValueError("This module cannot support a project+deployment name character length greater than 35")
+
+
+def _param(name: str) -> str:
+    return f"SEEDFARMER_PARAMETER_{name}"
+
+
+# App specific
+solution_id = os.getenv(_param("SOLUTION_ID"))  # required
+solution_name = os.getenv(_param("SOLUTION_NAME"))  # required
+solution_version = os.getenv(_param("SOLUTION_VERSION"))  # required
+
+
+if not solution_id:
+    raise ValueError("Missing input parameter solution-id")
+
+if not solution_name:
+    raise ValueError("Missing input parameter solution-name")
+
+if not solution_version:
+    raise ValueError("Missing input parameter solution-version")
+
+
+app = App()
+
+stack = AppRegistry(
+    scope=app,
+    id=f"{project_name}-{deployment_name}-{module_name}",
+    project_name=project_name,
+    deployment_name=deployment_name,
+    module_name=module_name,
+    solution_id=solution_id,
+    solution_name=solution_name,
+    solution_version=solution_version,
+    env=Environment(
+        account=os.environ["CDK_DEFAULT_ACCOUNT"],
+        region=os.environ["CDK_DEFAULT_REGION"],
+    ),
+)
+
+CfnOutput(
+    scope=stack,
+    id="metadata",
+    value=stack.to_json_string(
+        {
+            "AppRegistryName": stack.app_registry.name,
+            "AttributeGroupName": stack.attribute_group.name,
+        }
+    ),
+)
+
+app.synth(force=True)

modules/service-catalog/app-registry/coverage.ini

@@ -0,0 +1,4 @@
+[run]
+omit =
+    tests/*
+    register-stacks.py

modules/service-catalog/app-registry/deployspec.yaml

@@ -0,0 +1,25 @@
+publishGenericEnvVariables: true
+deploy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    build:
+      commands:
+      - cdk deploy --require-approval never --progress events --app "python app.py" --outputs-file ./cdk-exports.json
+      # Export metadata
+      - export SEEDFARMER_MODULE_METADATA=$(python -c "import json; file=open('cdk-exports.json'); print(json.load(file)['${SEEDFARMER_PROJECT_NAME}-${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}']['metadata'])")
+      # Associate Solution CFN stacks to AppRegistry
+      - python register-stacks.py associate
+destroy:
+  phases:
+    install:
+      commands:
+      - npm install -g [email protected]
+      - pip install -r requirements.txt
+    build:
+      commands:
+      # Dissociate Solution CFN stacks to AppRegistry
+      - python register-stacks.py dissociate
+      - cdk destroy --force --app "python app.py"

modules/service-catalog/app-registry/modulestack.yaml

@@ -0,0 +1,37 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: This template deploys a Module specific IAM permissions
+
+Parameters:
+  # ProjectName:
+  #   Type: String
+  #   Description: The name of the project
+  # DeploymentName:
+  #   Type: String
+  #   Description: The name of the deployment
+  # ModuleName:
+  #   Type: String
+  #   Description: The name of the Module
+  RoleName:
+    Type: String
+    Description: The name of the IAM Role
+
+Resources:
+  Policy:
+    Type: "AWS::IAM::Policy"
+    Properties:
+      PolicyDocument:
+        Statement:
+          - Action:
+              - "cloudformation:List*"
+              - "servicecatalog:Get*"
+            Effect: Allow
+            Resource: "*"
+          - Action:
+              - "servicecatalog:AssociateResource"
+              - "servicecatalog:DisassociateResource"
+            Effect: Allow
+            Resource:
+               - !Sub "arn:aws:servicecatalog:${AWS::Region}:${AWS::AccountId}:/applications/*"
+        Version: 2012-10-17
+      PolicyName: idf-modulespecific-policy
+      Roles: [!Ref RoleName]

modules/service-catalog/app-registry/pyproject.toml

@@ -0,0 +1,35 @@
+[tool.black]
+line-length = 120
+target-version = ["py36", "py37", "py38", "py39"]
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | \.env
+  | _build
+  | buck-out
+  | build
+  | dist
+  | codeseeder.out
+)/
+'''
+
+[tool.isort]
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+line_length = 120
+py_version = 38
+skip_gitignore = false
+
+[tool.pytest.ini_options]
+addopts = "-v --cov=. --cov-report term --cov-config=coverage.ini --cov-fail-under=80"
+pythonpath = [
+  "."
+]

modules/service-catalog/app-registry/register-stacks.py

@@ -0,0 +1,79 @@
+#!/usr/env/bin python
+
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Registers CloudFormation stacks into AppRegistry"""
+
+import json
+import os
+import sys
+from typing import List
+
+import boto3
+from botocore.exceptions import ClientError
+
+APPREG_CLIENT = boto3.client("servicecatalog-appregistry")
+CFN_CLIENT = boto3.client("cloudformation")
+PROJECT_NAME = os.getenv("SEEDFARMER_PROJECT_NAME", "addf")
+DEP_NAME = os.getenv("SEEDFARMER_DEPLOYMENT_NAME", "aws-solutions-wip")
+APP_REG_NAME = json.loads(os.getenv("SEEDFARMER_MODULE_METADATA"))["AppRegistryName"]  # type: ignore
+ACTION = sys.argv[1]
+
+
+def main() -> None:
+    """Driver function"""
+    stacks_tobe_registsred = _list_stacks(prefix=f"{PROJECT_NAME}-{DEP_NAME}")
+
+    if ACTION == "associate":
+        _asaociate_stacks(stacks=stacks_tobe_registsred)
+    else:
+        _dissaociate_stacks(stacks=stacks_tobe_registsred)
+
+
+def _asaociate_stacks(stacks: List[str]) -> None:
+    """Associate CloudFormation Stacks to App Registry"""
+    for stack in stacks:
+        try:
+            APPREG_CLIENT.associate_resource(application=APP_REG_NAME, resourceType="CFN_STACK", resource=stack)
+            print(f"Associated the stack: {stack} successfully")
+        except ClientError as ex:
+            if ex.response["Error"]["Code"] == "ResourceNotFoundException":
+                print(f"Could not find the stack: {stack} associated")
+                continue
+            else:
+                raise ex
+
+
+def _dissaociate_stacks(stacks: List[str]) -> None:
+    """Dissociate CloudFormation Stacks from App Registry"""
+
+    for stack in stacks:
+        try:
+            APPREG_CLIENT.disassociate_resource(application=APP_REG_NAME, resourceType="CFN_STACK", resource=stack)
+            print(f"Disassociated the stack: {stack} successfully")
+        except ClientError as ex:
+            if ex.response["Error"]["Code"] == "ResourceNotFoundException":
+                print(f"Could not find the stack: {stack} associated")
+                continue
+            else:
+                raise ex
+
+
+def _list_stacks(prefix: str) -> List[str]:
+    """List CloudFormation Stacks by the desired prefix"""
+
+    stacks_tobe_registsred = []
+
+    response = CFN_CLIENT.list_stacks(StackStatusFilter=["CREATE_COMPLETE"])
+
+    for stack in response["StackSummaries"]:
+        if stack["StackName"].startswith(prefix):
+            stacks_tobe_registsred.append(stack["StackName"])
+
+    print("The list of stacks: {}".format(stacks_tobe_registsred))
+    return stacks_tobe_registsred
+
+
+if __name__ == "__main__":
+    main()

modules/service-catalog/app-registry/requirements.in

@@ -0,0 +1,4 @@
+aws-cdk-lib==2.70.0
+cdk-nag==2.12.29
+constructs==10.0.91
+boto3~=1.21.0