From d9bf290032ceda7e6dd8244342d9017f107f66f7 Mon Sep 17 00:00:00 2001 From: srinivasreddych Date: Mon, 26 Jun 2023 15:29:32 -0500 Subject: [PATCH] fixed the images replication error which were doing string sub --- manifests/local-isolated/compute-modules.yaml | 138 +++++++++++++++++- manifests/local-isolated/deployment.yaml | 12 +- manifests/local/compute-modules.yaml | 19 +-- manifests/local/deployment.yaml | 4 + manifests/local/replicator-modules.yaml | 10 ++ modules/compute/eks/app.py | 8 +- .../eks-isolated/eks-isolated-deployment.yaml | 8 +- .../eks/docs/eks-isolated/eks-isolated.md | 8 +- .../monitoring-config/cloudwatch-agent.yaml | 4 +- ...secrets-store-csi-driver-provider-aws.yaml | 2 +- .../dockerimage-replication/README.md | 2 +- .../dockerimage-replication/coverage.ini | 2 +- .../{cleanup.py => delete-repos.py} | 0 .../dockerimage-replication/deployspec.yaml | 13 +- .../dockerimage-replication/modulestack.yaml | 7 +- .../dockerimage-replication/replication.sh | 3 +- .../tests/test_utils.py | 2 +- requirements.txt | 2 +- scripts/setup-secrets-dockerhub.sh | 28 ++++ 19 files changed, 229 insertions(+), 43 deletions(-) create mode 100644 manifests/local/replicator-modules.yaml rename modules/replication/dockerimage-replication/{cleanup.py => delete-repos.py} (100%) create mode 100755 scripts/setup-secrets-dockerhub.sh diff --git a/manifests/local-isolated/compute-modules.yaml b/manifests/local-isolated/compute-modules.yaml index 8d3da659..90359827 100644 --- a/manifests/local-isolated/compute-modules.yaml +++ b/manifests/local-isolated/compute-modules.yaml @@ -34,4 +34,140 @@ parameters: max_vcpus: 4800 desired_vcpus: 0 compute_type: FARGATE - order: 1 \ No newline at end of file + order: 1 +--- +name: eks +path: modules/compute/eks/ +dataFiles: + - filePath: data/eks_dockerimage-replication/versions/1.25.yaml + - filePath: data/eks_dockerimage-replication/versions/default.yaml +parameters: + - name: replicated-ecr-images-metadata-s3-path + valueFrom: + moduleMetadata: + group: replication + name: replication + key: s3_full_path + - name: vpc-id + valueFrom: + moduleMetadata: + group: networking + name: basic-networking + key: VpcId + - name: controlplane-subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: basic-networking + key: PrivateSubnetIds + - name: dataplane-subnet-ids + valueFrom: + moduleMetadata: + group: networking + name: basic-networking + key: PrivateSubnetIds + # - name: custom-subnet-ids + # value: ["subnet-0132ecb9b627b1c57", "subnet-0f2c81d12cba7ca69"] + - name: eks-admin-role-name + value: Admin + - name: eks-poweruser-role-name + value: PowerUser + - name: eks-read-only-role-name + value: ReadOnly + - name: eks-version + # value: 1.25 + valueFrom: + envVariable: GLOBAL_EKS_VERSION + - name: eks-compute + value: + eks_nodegroup_config: + - eks_ng_name: ng1 + eks_node_quantity: 2 + eks_node_max_quantity: 5 + eks_node_min_quantity: 1 + eks_node_disk_size: 20 + eks_node_instance_type: "m5.large" + - eks_ng_name: ng2 + eks_node_quantity: 2 + eks_node_max_quantity: 5 + eks_node_min_quantity: 1 + eks_node_disk_size: 20 + eks_node_instance_type: "m5.xlarge" + eks_node_spot: False + eks_api_endpoint_private: False + eks_secrets_envelope_encryption: True + - name: eks-addons + value: + # Load balancing + deploy_aws_lb_controller: True + deploy_nginx_controller: + value: False + nginx_additional_annotations: + nginx.ingress.kubernetes.io/whitelist-source-range: "100.64.0.0/10,10.0.0.0/8" + # DNS Component + deploy_external_dns: True + # Storage drivers + deploy_aws_ebs_csi: True + deploy_aws_efs_csi: True + # Autoscaling + deploy_cluster_autoscaler: True + deploy_metrics_server: True + # Secrets management + deploy_secretsmanager_csi: True + deploy_external_secrets: False + # Monitoring/Logging/Tracing + deploy_adot: True + deploy_cloudwatch_container_insights_metrics: True + deploy_cloudwatch_container_insights_logs: True + cloudwatch_container_insights_logs_retention_days: 7 + deploy_amp: True + deploy_grafana_for_amp: True + # Reboot agent + deploy_kured: True + # Network level segmentation + deploy_calico: False + # Security policies + deploy_kyverno: + value: False + kyverno_policies: + validate: + - block-ephemeral-containers + - block-stale-images + - block-updates-deletes + - check-deprecated-apis + - disallow-cri-sock-mount + - disallow-custom-snippets + - disallow-empty-ingress-host + - disallow-helm-tiller + - disallow-latest-tag + - disallow-localhost-services + - disallow-secrets-from-env-vars + - ensure-probes-different + - ingress-host-match-tls + - limit-hostpath-vols + - prevent-naked-pods + - require-drop-cap-net-raw + - require-emptydir-requests-limits + - require-labels + - require-pod-requests-limits + - require-probes + - restrict-annotations + - restrict-automount-sa-token + - restrict-binding-clusteradmin + - restrict-clusterrole-nodesproxy + - restrict-escalation-verbs-roles + - restrict-ingress-classes + - restrict-ingress-defaultbackend + - restrict-node-selection + - restrict-path + - restrict-service-external-ips + - restrict-wildcard-resources + - restrict-wildcard-verbs + - unique-ingress-host-and-path + # mutate: + # - add-networkpolicy-dns + # - add-pod-priorityclassname + # - add-ttl-jobs + # - always-pull-images + # - mitigate-log4shell + \ No newline at end of file diff --git a/manifests/local-isolated/deployment.yaml b/manifests/local-isolated/deployment.yaml index 459d36f9..af7ee82d 100644 --- a/manifests/local-isolated/deployment.yaml +++ b/manifests/local-isolated/deployment.yaml @@ -6,15 +6,13 @@ nameGenerator: toolchainRegion: us-east-1 groups: - name: networking - path: manifests/local/networking-modules.yaml + path: manifests/local-isolated/networking-modules.yaml - name: storage - path: manifests/local/storage-modules.yaml + path: manifests/local-isolated/storage-modules.yaml + - name: replication + path: manifests/local-isolated/replicator-modules.yaml - name: compute - path: manifests/local/compute-modules.yaml - - name: database - path: manifests/local/database-modules.yaml - - name: orchestration - path: manifests/local/orchestration-modules.yaml + path: manifests/local-isolated/compute-modules.yaml targetAccountMappings: - alias: primary accountId: diff --git a/manifests/local/compute-modules.yaml b/manifests/local/compute-modules.yaml index 54e92fb4..99e80f90 100644 --- a/manifests/local/compute-modules.yaml +++ b/manifests/local/compute-modules.yaml @@ -42,12 +42,13 @@ dataFiles: - filePath: data/eks_dockerimage-replication/versions/1.25.yaml - filePath: data/eks_dockerimage-replication/versions/default.yaml parameters: - # - name: replicated-ecr-images-metadata-s3-path - # valueFrom: - # moduleMetadata: - # group: replication - # name: replication - # key: s3_full_path + # Use the below parameter if you want to use replicated container images from account's specific ECR repo(s) + - name: replicated-ecr-images-metadata-s3-path + valueFrom: + moduleMetadata: + group: replication + name: replication + key: s3_full_path - name: vpc-id valueFrom: moduleMetadata: @@ -101,7 +102,7 @@ parameters: # Load balancing deploy_aws_lb_controller: True deploy_nginx_controller: - value: False + value: True nginx_additional_annotations: nginx.ingress.kubernetes.io/whitelist-source-range: "100.64.0.0/10,10.0.0.0/8" # DNS Component @@ -125,10 +126,10 @@ parameters: # Reboot agent deploy_kured: True # Network level segmentation - deploy_calico: False + deploy_calico: True # Security policies deploy_kyverno: - value: False + value: True kyverno_policies: validate: - block-ephemeral-containers diff --git a/manifests/local/deployment.yaml b/manifests/local/deployment.yaml index 459d36f9..fe703111 100644 --- a/manifests/local/deployment.yaml +++ b/manifests/local/deployment.yaml @@ -9,6 +9,8 @@ groups: path: manifests/local/networking-modules.yaml - name: storage path: manifests/local/storage-modules.yaml + - name: replication + path: manifests/local/replicator-modules.yaml - name: compute path: manifests/local/compute-modules.yaml - name: database @@ -17,6 +19,8 @@ groups: path: manifests/local/orchestration-modules.yaml targetAccountMappings: - alias: primary + parametersGlobal: + dockerCredentialsSecret: aws-idf-docker-credentials accountId: valueFrom: envVariable: PRIMARY_ACCOUNT diff --git a/manifests/local/replicator-modules.yaml b/manifests/local/replicator-modules.yaml new file mode 100644 index 00000000..00450560 --- /dev/null +++ b/manifests/local/replicator-modules.yaml @@ -0,0 +1,10 @@ +name: replication +path: modules/replication/dockerimage-replication/ +dataFiles: + - filePath: data/eks_dockerimage-replication/versions/1.25.yaml + - filePath: data/eks_dockerimage-replication/versions/default.yaml +parameters: + - name: eks-version + # value: "1.25" + valueFrom: + envVariable: GLOBAL_EKS_VERSION \ No newline at end of file diff --git a/modules/compute/eks/app.py b/modules/compute/eks/app.py index de664a6a..ec8d3352 100644 --- a/modules/compute/eks/app.py +++ b/modules/compute/eks/app.py @@ -60,9 +60,11 @@ def _param(name: str) -> str: eks_compute_config=eks_compute_config, eks_addons_config=eks_addons_config, custom_subnet_ids=custom_subnet_ids, - codebuild_sg_id=codebuild_sg_id if os.getenv("SEEDFARMER_PARAMETER_CODEBUILD_SG_ID") else None, - replicated_ecr_images_metadata=replicated_ecr_images_metadata - if os.getenv("SEEDFARMER_PARAMETER_REPLICATED_ECR_IMAGES_METADATA") + codebuild_sg_id=json.loads(os.getenv(_param("CODEBUILD_SG_ID")))[0] + if os.getenv(_param("CODEBUILD_SG_ID")) + else None, + replicated_ecr_images_metadata=json.loads(os.getenv(_param("REPLICATED_ECR_IMAGES_METADATA"))) + if os.getenv(_param("REPLICATED_ECR_IMAGES_METADATA")) else {}, env=aws_cdk.Environment( account=os.environ["CDK_DEFAULT_ACCOUNT"], diff --git a/modules/compute/eks/docs/eks-isolated/eks-isolated-deployment.yaml b/modules/compute/eks/docs/eks-isolated/eks-isolated-deployment.yaml index 6c5525a5..89c42901 100644 --- a/modules/compute/eks/docs/eks-isolated/eks-isolated-deployment.yaml +++ b/modules/compute/eks/docs/eks-isolated/eks-isolated-deployment.yaml @@ -14,7 +14,7 @@ targetAccountMappings: - region: eu-west-2 default: true parametersRegional: - dockerCredentialsSecret: aws-addf-docker-credentials + dockerCredentialsSecret: aws-idf-docker-credentials # replace the below networking details with customer specific values vpcId: vpc-XXXXXXXX publicSubnetIds: @@ -35,16 +35,16 @@ targetAccountMappings: parameterValue: vpcId # Alternatively you can grab the networking values from SSM parameter store # valueFrom: - # parameterStore: /addf/vpc-id + # parameterStore: /idf/vpc-id privateSubnetIds: valueFrom: parameterValue: privateSubnetIds # Alternatively you can grab the networking values from SSM parameter store # valueFrom: - # parameterStore: /addf/private-ids + # parameterStore: /idf/private-ids securityGroupIds: valueFrom: parameterValue: securityGroupIds # Alternatively you can grab the codebuild security group from SSM parameter store # valueFrom: - # parameterStore: /addf/sg-ids \ No newline at end of file + # parameterStore: /idf/sg-ids \ No newline at end of file diff --git a/modules/compute/eks/docs/eks-isolated/eks-isolated.md b/modules/compute/eks/docs/eks-isolated/eks-isolated.md index dd7ef663..36ac9713 100644 --- a/modules/compute/eks/docs/eks-isolated/eks-isolated.md +++ b/modules/compute/eks/docs/eks-isolated/eks-isolated.md @@ -27,7 +27,7 @@ targetAccountMappings: - region: eu-west-2 default: true parametersRegional: - dockerCredentialsSecret: aws-addf-docker-credentials + dockerCredentialsSecret: aws-idf-docker-credentials # replace the below networking details with customer specific values vpcId: vpc-XXXXXXXX publicSubnetIds: @@ -48,19 +48,19 @@ targetAccountMappings: parameterValue: vpcId # Alternatively you can grab the networking values from SSM parameter store # valueFrom: - # parameterStore: /addf/vpc-id + # parameterStore: /idf/vpc-id privateSubnetIds: valueFrom: parameterValue: privateSubnetIds # Alternatively you can grab the networking values from SSM parameter store # valueFrom: - # parameterStore: /addf/private-ids + # parameterStore: /idf/private-ids securityGroupIds: valueFrom: parameterValue: securityGroupIds # Alternatively you can grab the codebuild security group from SSM parameter store # valueFrom: - # parameterStore: /addf/sg-ids + # parameterStore: /idf/sg-ids ``` ```observation diff --git a/modules/compute/eks/monitoring-config/cloudwatch-agent.yaml b/modules/compute/eks/monitoring-config/cloudwatch-agent.yaml index ae81dc96..09b0ca3c 100644 --- a/modules/compute/eks/monitoring-config/cloudwatch-agent.yaml +++ b/modules/compute/eks/monitoring-config/cloudwatch-agent.yaml @@ -53,14 +53,14 @@ spec: spec: containers: - name: cloudwatch-agent - image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247358.0b252413 + image: $image ports: - containerPort: 8125 hostPort: 8125 protocol: UDP resources: limits: - cpu: 200m + cpu: 200m memory: 200Mi requests: cpu: 200m diff --git a/modules/compute/eks/secrets-config/secrets-store-csi-driver-provider-aws.yaml b/modules/compute/eks/secrets-config/secrets-store-csi-driver-provider-aws.yaml index f05a6fd7..a20baeef 100644 --- a/modules/compute/eks/secrets-config/secrets-store-csi-driver-provider-aws.yaml +++ b/modules/compute/eks/secrets-config/secrets-store-csi-driver-provider-aws.yaml @@ -52,7 +52,7 @@ spec: hostNetwork: true containers: - name: provider-aws-installer - image: public.ecr.aws/aws-secrets-manager/secrets-store-csi-driver-provider-aws:1.0.r2-2021.08.13.20.34-linux-amd64 + image: $image imagePullPolicy: Always args: - --provider-volume=/etc/kubernetes/secrets-store-csi-providers diff --git a/modules/replication/dockerimage-replication/README.md b/modules/replication/dockerimage-replication/README.md index f089050b..b6218c53 100644 --- a/modules/replication/dockerimage-replication/README.md +++ b/modules/replication/dockerimage-replication/README.md @@ -35,6 +35,6 @@ path: modules/replication/dockerimage-replication/ ```json { - "aws-efs-csi-driver": "1234567890.dkr.ecr.eu-central-1.amazonaws.com/addf-amazon/aws-efs-csi-driver:v1.3.6" + "aws-efs-csi-driver": "1234567890.dkr.ecr.eu-central-1.amazonaws.com/idf-amazon/aws-efs-csi-driver:v1.3.6" } ``` diff --git a/modules/replication/dockerimage-replication/coverage.ini b/modules/replication/dockerimage-replication/coverage.ini index 0a2653b6..16a864a9 100644 --- a/modules/replication/dockerimage-replication/coverage.ini +++ b/modules/replication/dockerimage-replication/coverage.ini @@ -1,5 +1,5 @@ [run] omit = tests/* - cleanup.py + delete-repos.py get-list-of-eks-images.py diff --git a/modules/replication/dockerimage-replication/cleanup.py b/modules/replication/dockerimage-replication/delete-repos.py similarity index 100% rename from modules/replication/dockerimage-replication/cleanup.py rename to modules/replication/dockerimage-replication/delete-repos.py diff --git a/modules/replication/dockerimage-replication/deployspec.yaml b/modules/replication/dockerimage-replication/deployspec.yaml index 346e0500..4d3b6bd1 100644 --- a/modules/replication/dockerimage-replication/deployspec.yaml +++ b/modules/replication/dockerimage-replication/deployspec.yaml @@ -4,16 +4,18 @@ deploy: install: commands: - pip install -r requirements.txt - - wget https://get.helm.sh/helm-v3.11.3-linux-amd64.tar.gz - - tar zxf helm-v3.11.3-linux-amd64.tar.gz - - mv linux-amd64/helm /usr/local/bin/helm + - wget https://get.helm.sh/helm-v3.11.3-linux-amd64.tar.gz && tar zxf helm-v3.11.3-linux-amd64.tar.gz && mv linux-amd64/helm /usr/local/bin/helm build: commands: - S3_BUCKET_NAME="${SEEDFARMER_PROJECT_NAME}-dockerimages-replications-metadata-${AWS_DEFAULT_REGION}-${AWS_ACCOUNT_ID}" - S3_OBJECT_NAME="${SEEDFARMER_DEPLOYMENT_NAME}-${SEEDFARMER_MODULE_NAME}-metadata.json" - | if ! aws s3api head-bucket --bucket "${S3_BUCKET_NAME}"; then - aws s3api create-bucket --bucket "${S3_BUCKET_NAME}" --region "${AWS_DEFAULT_REGION}" --create-bucket-configuration LocationConstraint=${AWS_DEFAULT_REGION} + if [ ${AWS_DEFAULT_REGION} == "us-east-1" ]; then + aws s3api create-bucket --bucket "${S3_BUCKET_NAME}" --region "${AWS_DEFAULT_REGION}" + else + aws s3api create-bucket --bucket "${S3_BUCKET_NAME}" --region "${AWS_DEFAULT_REGION}" --create-bucket-configuration LocationConstraint=${AWS_DEFAULT_REGION} + fi fi - python3 get-list-of-eks-images.py --eks-version ${SEEDFARMER_PARAMETER_EKS_VERSION} --versions-directory data/eks_dockerimage-replication/versions --update-helm-repos --registry-prefix "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/${AWS_CODESEEDER_NAME}-" - chmod +x replication.sh @@ -33,5 +35,6 @@ destroy: build: commands: - chmod +x replication.sh - - bash replication.sh destroy + # The below command removes the ECR repos with `idf` as prefix which has the replicated images. PLEASE USE IT AT YOUR OWN RISK + # - bash replication.sh destroy build_type: BUILD_GENERAL1_LARGE diff --git a/modules/replication/dockerimage-replication/modulestack.yaml b/modules/replication/dockerimage-replication/modulestack.yaml index 1899ad62..d42bf040 100644 --- a/modules/replication/dockerimage-replication/modulestack.yaml +++ b/modules/replication/dockerimage-replication/modulestack.yaml @@ -8,6 +8,9 @@ Parameters: # ModuleName: # Type: String # Description: The name of the Module + ProjectName: + Type: String + Description: The name of the project RoleName: Type: String Description: The name of the IAM Role @@ -33,7 +36,7 @@ Resources: - "ecr:Put*" Effect: Allow Resource: - - !Sub "arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/addf-*" + - !Sub "arn:aws:ecr:${AWS::Region}:${AWS::AccountId}:repository/${ProjectName}-*" - Action: - "s3:CreateBucket" Effect: Allow @@ -50,5 +53,5 @@ Resources: Effect: Allow Resource: "arn:aws:s3:::*-dockerimages-replications-*/*" Version: 2012-10-17 - PolicyName: "addf-modulespecific-policy" + PolicyName: "idf-modulespecific-policy" Roles: [!Ref RoleName] \ No newline at end of file diff --git a/modules/replication/dockerimage-replication/replication.sh b/modules/replication/dockerimage-replication/replication.sh index 08ed183c..876aa041 100755 --- a/modules/replication/dockerimage-replication/replication.sh +++ b/modules/replication/dockerimage-replication/replication.sh @@ -26,7 +26,8 @@ create() { } destroy() { - echo "Sorry... not working" + echo "WARNING: The destroy workflow removes the ECR repositories which we were created during replication" + python delete-repos.py } $1 diff --git a/modules/replication/dockerimage-replication/tests/test_utils.py b/modules/replication/dockerimage-replication/tests/test_utils.py index 70100720..39a19620 100644 --- a/modules/replication/dockerimage-replication/tests/test_utils.py +++ b/modules/replication/dockerimage-replication/tests/test_utils.py @@ -14,8 +14,8 @@ import unittest +from helmparser.logging import boto3_logger, logger from helmparser.utils.utils import deep_merge -from helmparser.logging import logger, boto3_logger class TestMain(unittest.TestCase): diff --git a/requirements.txt b/requirements.txt index 350a136e..43640aa9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -seed-farmer~=2.8.0 +seed-farmer~=2.9.0 diff --git a/scripts/setup-secrets-dockerhub.sh b/scripts/setup-secrets-dockerhub.sh new file mode 100755 index 00000000..9bcc25e0 --- /dev/null +++ b/scripts/setup-secrets-dockerhub.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +SECRET_NAME="aws-idf-docker-credentials" + +read -p "DockerHub Username: " DOCKERHUB_USER +read -sp "DockerHub Password: " DOCKERHUB_PASS + +echo -e "\nCreating Secret" + +SECRET_VALUE="{\"docker.io\": { \"username\": \"$DOCKERHUB_USER\", \"password\": \"$DOCKERHUB_PASS\" }}" + +EXISTS=$(aws secretsmanager describe-secret --secret-id $SECRET_NAME 2>/dev/null) +if [ $? -eq 0 ]; then + echo "Secret ($SECRET_NAME) exists. Updating" + aws secretsmanager put-secret-value \ + --secret-id $SECRET_NAME \ + --secret-string "$SECRET_VALUE" + + echo "$SECRET_NAME updated" +else + echo "Secret ($SECRET_NAME) doesn't exist. Creating" + aws secretsmanager create-secret \ + --name $SECRET_NAME \ + --description "Credentials for DockerHub" \ + --secret-string "$SECRET_VALUE" + + echo "$SECRET_NAME created" +fi