refactored docker images replication to idf

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - refactored Neptune module, to be agnostic of a project with unit-tests
 - added module output example for `dummy/blank` module
 - added EKS module, refactored it to be agnostic of a project with unit-tests
+- added Docker images replication module, refactored it to be agnostic of a project with unit-tests
 
 ### **Changed**
 

manifests/local-isolated/compute-modules.yaml

@@ -34,4 +34,140 @@ parameters:
           max_vcpus: 4800
           desired_vcpus: 0
           compute_type: FARGATE
-          order: 1
+          order: 1
+---
+name: eks
+path: modules/compute/eks/
+dataFiles:
+  - filePath: data/eks_dockerimage-replication/versions/1.25.yaml
+  - filePath: data/eks_dockerimage-replication/versions/default.yaml
+parameters:
+  - name: replicated-ecr-images-metadata-s3-path
+    valueFrom:
+      moduleMetadata:
+        group: replication
+        name: replication
+        key: s3_full_path
+  - name: vpc-id
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: basic-networking
+        key: VpcId
+  - name: controlplane-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: basic-networking
+        key: PrivateSubnetIds
+  - name: dataplane-subnet-ids
+    valueFrom:
+      moduleMetadata:
+        group: networking
+        name: basic-networking
+        key: PrivateSubnetIds
+  # - name: custom-subnet-ids
+  #   value: ["subnet-0132ecb9b627b1c57", "subnet-0f2c81d12cba7ca69"]
+  - name: eks-admin-role-name
+    value: Admin
+  - name: eks-poweruser-role-name
+    value: PowerUser
+  - name: eks-read-only-role-name
+    value: ReadOnly
+  - name: eks-version
+    # value: 1.25
+    valueFrom:
+      envVariable: GLOBAL_EKS_VERSION
+  - name: eks-compute
+    value:
+      eks_nodegroup_config:
+        - eks_ng_name: ng1
+          eks_node_quantity: 2
+          eks_node_max_quantity: 5
+          eks_node_min_quantity: 1
+          eks_node_disk_size: 20
+          eks_node_instance_type: "m5.large"
+        - eks_ng_name: ng2
+          eks_node_quantity: 2
+          eks_node_max_quantity: 5
+          eks_node_min_quantity: 1
+          eks_node_disk_size: 20
+          eks_node_instance_type: "m5.xlarge"
+      eks_node_spot: False
+      eks_api_endpoint_private: False
+      eks_secrets_envelope_encryption: True
+  - name: eks-addons
+    value:
+      # Load balancing
+      deploy_aws_lb_controller: True 
+      deploy_nginx_controller:
+        value: False
+        nginx_additional_annotations:
+          nginx.ingress.kubernetes.io/whitelist-source-range: "100.64.0.0/10,10.0.0.0/8"
+      # DNS Component
+      deploy_external_dns: True 
+      # Storage drivers
+      deploy_aws_ebs_csi: True 
+      deploy_aws_efs_csi: True 
+      # Autoscaling
+      deploy_cluster_autoscaler: True 
+      deploy_metrics_server: True 
+      # Secrets management
+      deploy_secretsmanager_csi: True 
+      deploy_external_secrets: False
+      # Monitoring/Logging/Tracing
+      deploy_adot: True
+      deploy_cloudwatch_container_insights_metrics: True 
+      deploy_cloudwatch_container_insights_logs: True
+      cloudwatch_container_insights_logs_retention_days: 7
+      deploy_amp: True
+      deploy_grafana_for_amp: True
+      # Reboot agent
+      deploy_kured: True
+      # Network level segmentation
+      deploy_calico: False
+      # Security policies
+      deploy_kyverno:
+        value: False
+        kyverno_policies:
+          validate:
+            - block-ephemeral-containers
+            - block-stale-images
+            - block-updates-deletes
+            - check-deprecated-apis
+            - disallow-cri-sock-mount
+            - disallow-custom-snippets
+            - disallow-empty-ingress-host
+            - disallow-helm-tiller
+            - disallow-latest-tag
+            - disallow-localhost-services
+            - disallow-secrets-from-env-vars
+            - ensure-probes-different
+            - ingress-host-match-tls
+            - limit-hostpath-vols
+            - prevent-naked-pods
+            - require-drop-cap-net-raw
+            - require-emptydir-requests-limits
+            - require-labels
+            - require-pod-requests-limits
+            - require-probes
+            - restrict-annotations
+            - restrict-automount-sa-token
+            - restrict-binding-clusteradmin
+            - restrict-clusterrole-nodesproxy
+            - restrict-escalation-verbs-roles
+            - restrict-ingress-classes
+            - restrict-ingress-defaultbackend
+            - restrict-node-selection
+            - restrict-path
+            - restrict-service-external-ips
+            - restrict-wildcard-resources
+            - restrict-wildcard-verbs
+            - unique-ingress-host-and-path
+          # mutate:
+          #   - add-networkpolicy-dns
+          #   - add-pod-priorityclassname
+          #   - add-ttl-jobs
+          #   - always-pull-images
+          #   - mitigate-log4shell
+

manifests/local-isolated/deployment.yaml

@@ -6,15 +6,13 @@ nameGenerator:
 toolchainRegion: us-east-1
 groups:
   - name: networking
-    path: manifests/local/networking-modules.yaml
+    path: manifests/local-isolated/networking-modules.yaml
   - name: storage
-    path: manifests/local/storage-modules.yaml
+    path: manifests/local-isolated/storage-modules.yaml
+  - name: replication
+    path: manifests/local-isolated/replicator-modules.yaml
   - name: compute
-    path: manifests/local/compute-modules.yaml
-  - name: database
-    path: manifests/local/database-modules.yaml
-  - name: orchestration
-    path: manifests/local/orchestration-modules.yaml
+    path: manifests/local-isolated/compute-modules.yaml
 targetAccountMappings:
   - alias: primary
     accountId:

manifests/local-isolated/replicator-modules.yaml

@@ -0,0 +1,10 @@
+name: replication
+path: modules/replication/dockerimage-replication/
+dataFiles:
+  - filePath: data/eks_dockerimage-replication/versions/1.25.yaml
+  - filePath: data/eks_dockerimage-replication/versions/default.yaml
+parameters:
+  - name: eks-version
+    # value: "1.25"
+    valueFrom:
+      envVariable: GLOBAL_EKS_VERSION

manifests/local/compute-modules.yaml

@@ -42,6 +42,7 @@ dataFiles:
   - filePath: data/eks_dockerimage-replication/versions/1.25.yaml
   - filePath: data/eks_dockerimage-replication/versions/default.yaml
 parameters:
+  # Use the below parameter if you want to use replicated container images from account's specific ECR repo(s)
   # - name: replicated-ecr-images-metadata-s3-path
   #   valueFrom:
   #     moduleMetadata:
@@ -101,7 +102,7 @@ parameters:
       # Load balancing
       deploy_aws_lb_controller: True 
       deploy_nginx_controller:
-        value: False
+        value: True
         nginx_additional_annotations:
           nginx.ingress.kubernetes.io/whitelist-source-range: "100.64.0.0/10,10.0.0.0/8"
       # DNS Component
@@ -125,10 +126,10 @@ parameters:
       # Reboot agent
       deploy_kured: True
       # Network level segmentation
-      deploy_calico: False
+      deploy_calico: True
       # Security policies
       deploy_kyverno:
-        value: False
+        value: True
         kyverno_policies:
           validate:
             - block-ephemeral-containers

manifests/local/deployment.yaml

@@ -9,6 +9,8 @@ groups:
     path: manifests/local/networking-modules.yaml
   - name: storage
     path: manifests/local/storage-modules.yaml
+  # - name: replication
+  #   path: manifests/local/replicator-modules.yaml
   - name: compute
     path: manifests/local/compute-modules.yaml
   - name: database
@@ -17,6 +19,8 @@ groups:
     path: manifests/local/orchestration-modules.yaml
 targetAccountMappings:
   - alias: primary
+    parametersGlobal:
+      dockerCredentialsSecret: aws-idf-docker-credentials
     accountId:
       valueFrom:
         envVariable: PRIMARY_ACCOUNT

manifests/local/replicator-modules.yaml

@@ -0,0 +1,10 @@
+name: replication
+path: modules/replication/dockerimage-replication/
+dataFiles:
+  - filePath: data/eks_dockerimage-replication/versions/1.25.yaml
+  - filePath: data/eks_dockerimage-replication/versions/default.yaml
+parameters:
+  - name: eks-version
+    # value: "1.25"
+    valueFrom:
+      envVariable: GLOBAL_EKS_VERSION

modules/compute/eks/app.py

@@ -60,9 +60,11 @@ def _param(name: str) -> str:
     eks_compute_config=eks_compute_config,
     eks_addons_config=eks_addons_config,
     custom_subnet_ids=custom_subnet_ids,
-    codebuild_sg_id=codebuild_sg_id if os.getenv("SEEDFARMER_PARAMETER_CODEBUILD_SG_ID") else None,
-    replicated_ecr_images_metadata=replicated_ecr_images_metadata
-    if os.getenv("SEEDFARMER_PARAMETER_REPLICATED_ECR_IMAGES_METADATA")
+    codebuild_sg_id=json.loads(os.getenv(_param("CODEBUILD_SG_ID")))[0]
+    if os.getenv(_param("CODEBUILD_SG_ID"))
+    else None,
+    replicated_ecr_images_metadata=json.loads(os.getenv(_param("REPLICATED_ECR_IMAGES_METADATA")))
+    if os.getenv(_param("REPLICATED_ECR_IMAGES_METADATA"))
     else {},
     env=aws_cdk.Environment(
         account=os.environ["CDK_DEFAULT_ACCOUNT"],

modules/compute/eks/docs/eks-isolated/eks-isolated-deployment.yaml

@@ -14,7 +14,7 @@ targetAccountMappings:
       - region: eu-west-2
         default: true
         parametersRegional:
-          dockerCredentialsSecret: aws-addf-docker-credentials
+          dockerCredentialsSecret: aws-idf-docker-credentials
           # replace the below networking details with customer specific values
           vpcId: vpc-XXXXXXXX
           publicSubnetIds:
@@ -35,16 +35,16 @@ targetAccountMappings:
               parameterValue: vpcId
             # Alternatively you can grab the networking values from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/vpc-id
+            #   parameterStore: /idf/vpc-id
           privateSubnetIds:
             valueFrom:
               parameterValue: privateSubnetIds
             # Alternatively you can grab the networking values from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/private-ids
+            #   parameterStore: /idf/private-ids
           securityGroupIds:
             valueFrom:
               parameterValue: securityGroupIds
             # Alternatively you can grab the codebuild security group from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/sg-ids
+            #   parameterStore: /idf/sg-ids

modules/compute/eks/docs/eks-isolated/eks-isolated.md

@@ -27,7 +27,7 @@ targetAccountMappings:
       - region: eu-west-2
         default: true
         parametersRegional:
-          dockerCredentialsSecret: aws-addf-docker-credentials
+          dockerCredentialsSecret: aws-idf-docker-credentials
           # replace the below networking details with customer specific values
           vpcId: vpc-XXXXXXXX
           publicSubnetIds:
@@ -48,19 +48,19 @@ targetAccountMappings:
               parameterValue: vpcId
             # Alternatively you can grab the networking values from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/vpc-id
+            #   parameterStore: /idf/vpc-id
           privateSubnetIds:
             valueFrom:
               parameterValue: privateSubnetIds
             # Alternatively you can grab the networking values from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/private-ids
+            #   parameterStore: /idf/private-ids
           securityGroupIds:
             valueFrom:
               parameterValue: securityGroupIds
             # Alternatively you can grab the codebuild security group from SSM parameter store
             # valueFrom:
-            #   parameterStore: /addf/sg-ids
+            #   parameterStore: /idf/sg-ids
 ```
 
 ```observation

modules/compute/eks/monitoring-config/cloudwatch-agent.yaml

@@ -53,14 +53,14 @@ spec:
     spec:
       containers:
         - name: cloudwatch-agent
-          image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:1.247358.0b252413
+          image: $image
           ports:
             - containerPort: 8125
               hostPort: 8125
               protocol: UDP
           resources:
             limits:
-              cpu:  200m
+              cpu: 200m
               memory: 200Mi
             requests:
               cpu: 200m

modules/compute/eks/secrets-config/secrets-store-csi-driver-provider-aws.yaml

@@ -52,7 +52,7 @@ spec:
       hostNetwork: true
       containers:
         - name: provider-aws-installer
-          image: public.ecr.aws/aws-secrets-manager/secrets-store-csi-driver-provider-aws:1.0.r2-2021.08.13.20.34-linux-amd64
+          image: $image
           imagePullPolicy: Always
           args:
             - --provider-volume=/etc/kubernetes/secrets-store-csi-providers

modules/replication/dockerimage-replication/.coveragerc

@@ -0,0 +1,10 @@
+# .coveragerc to control coverage.py
+[run]
+omit =
+    */site-packages/*
+    */distutils/*
+    */tests/*
+    ./get-list-of-eks-images.py
+
+[report]
+show_missing=true

modules/replication/dockerimage-replication/.gitignore

@@ -0,0 +1,3 @@
+images.txt
+replication-result.json
+s3_metadata.yaml

modules/replication/dockerimage-replication/CODE_OF_CONDUCT.md

@@ -0,0 +1,4 @@
+## Code of Conduct
+This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
+For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
+[email protected] with any additional questions or comments.