Docker compose fixes and improvements for local dev env (#1068)

Co-authored-by: Sandro Tajè <[email protected]>
pagopa · Oct 8, 2024 · f0043fa · f0043fa
1 parent ade5c20
commit f0043fa
Show file tree

Hide file tree

Showing 9 changed files with 94 additions and 85 deletions.
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -1,6 +1,10 @@
+# Docker Compose for local development environment
+
 version: "3"
 name: pagopa-interop
 services:
+
+  # Event Store Postgres DB
   event-store:
     volumes:
       - ./event-store-init.sql:/docker-entrypoint-initdb.d/01-init.sql
@@ -13,9 +17,11 @@ services:
       POSTGRES_PASSWORD: root
     command: -c 'max_connections=512' -c 'wal_level=logical'
 
+  # Zookeeper for Kafka
   zookeeper:
     image: quay.io/debezium/zookeeper:2.3
 
+  # Kafka server
   kafka:
     image: quay.io/debezium/kafka:2.3
     ports:
@@ -32,6 +38,7 @@ services:
       KAFKA_NUM_PARTITIONS: 3
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
 
+  # Debezium to stream event store changes to kafka
   connect:
     image: quay.io/debezium/connect:2.3
     ports:
@@ -46,6 +53,7 @@ services:
     depends_on:
       - kafka
 
+  # Mongo DB readmodel
   readmodel:
     image: mongo:4.0
     ports:
@@ -54,57 +62,44 @@ services:
       MONGO_INITDB_ROOT_USERNAME: root
       MONGO_INITDB_ROOT_PASSWORD: example
 
-  token-generation-readmodel:
+  # Dynamo DB readmodel for token generation and privacy notices
+  dynamodb-local:
     command: "-jar DynamoDBLocal.jar -inMemory -sharedDb"
-    image: "amazon/dynamodb-local:latest"
+    image: amazon/dynamodb-local:2.5.2
     ports:
       - 8085:8000
-
-  platform-states-table-init:
-    depends_on:
-      - token-generation-readmodel
-    restart: on-failure
-    image: amazon/aws-cli
-    working_dir: /home/tables
-    volumes:
-      - ./dynamo-db/schema:/home/tables
-    environment:
-      AWS_ACCESS_KEY_ID: keyid
-      AWS_SECRET_ACCESS_KEY: key
-    command: dynamodb create-table
-      --cli-input-json file://./platform-states.json
-      --endpoint-url http://token-generation-readmodel:8000
-      --region eu-south-1
-
-  token-generation-states-table-init:
-    depends_on:
-      - token-generation-readmodel
-    restart: on-failure
-    image: amazon/aws-cli
-    working_dir: /home/tables
     volumes:
-      - ./dynamoDB-tables:/home/tables
-    environment:
-      AWS_ACCESS_KEY_ID: keyid
-      AWS_SECRET_ACCESS_KEY: key
-    command: dynamodb create-table
-      --cli-input-json file://./token-generation-states.json
-      --endpoint-url http://token-generation-readmodel:8000
-      --region eu-south-1
+      - ./dynamo-db-data:/data
+    working_dir: /home/dynamodblocal
 
+  # Web GUI for dynamodb
   dynamodb-admin:
-    image: "aaronshaf/dynamodb-admin"
-    container_name: dynamodb-admin_token-generation-readmodel
+    image: aaronshaf/dynamodb-admin:4.6.1
     depends_on:
-      - token-generation-readmodel
+      - dynamodb-local
     restart: always
     ports:
       - "8001:8001"
     environment:
-      - DYNAMO_ENDPOINT=http://token-generation-readmodel:8000
-      - AWS_REGION=eu-west-1
+      - DYNAMO_ENDPOINT=http://dynamodb-local:8000
+      - AWS_REGION=eu-south-1
+
+  # dynamodb migrations and data seed
+  dynamodb-migrations:
+    image: amazon/aws-cli:2.17.44
+    working_dir: /home/dynamodblocal
+    volumes:
+      - ./dynamo-db:/home/dynamodblocal
+    depends_on:
+      - dynamodb-local
+    environment:
+      - AWS_ACCESS_KEY_ID=local
+      - AWS_SECRET_ACCESS_KEY=local
+      - AWS_REGION=eu-south-1
+      - TABLES=privacy-notice,privacy-notice-acceptances,platform-states,token-generation-states
+    entrypoint: ./dynamo-migrations.sh
 
-  # Mongo Express is a web-based MongoDB admin interface, included for convenience
+  # Web GUI for mongo
   mongo-express:
     image: mongo-express:1.0.2-20
     ports:
@@ -119,7 +114,7 @@ services:
       - readmodel
     restart: always
 
-  # PGAdmin is a web-based PostgreSQL admin interface, included for convenience
+  # Web GUI for PostgreSQL
   pg-admin:
     image: dpage/pgadmin4:2024-04-01-1
     ports:
@@ -142,6 +137,8 @@ services:
       chmod 600 /var/lib/pgadmin/storage/root_example.com/pgpass;
       /entrypoint.sh
       "
+
+  # Web GUI for kafka
   kafka-ui:
     container_name: kafka-ui
     image: provectuslabs/kafka-ui:53a6553765a806eda9905c43bfcfe09da6812035
@@ -171,8 +168,9 @@ services:
       - ./minio-data:/data
     restart: always
 
+  # Seed data for MinIO
   minio-seed:
-    image: minio/mc
+    image: minio/mc:RELEASE.2024-02-09T22-18-24Z
     depends_on:
       - minio
     entrypoint: >
@@ -222,50 +220,13 @@ services:
     volumes:
       - ./local-kms-seed/jwks.json:/usr/share/nginx/html/jwks.json
 
+  # Mock for AWS SES
   ses-mock:
     image: node:20
     ports:
       - 8005:8005
     command: "/bin/bash -c 'npm install -g aws-ses-v2-local; aws-ses-v2-local --port=8005 --host=0.0.0.0'"
 
-  # dynamodb-local is a local DynamoDB-compatible database, to replace AWS DynamoDB in local development
-  dynamodb-local:
-    image: amazon/dynamodb-local:2.5.2
-    ports:
-      - 7001:8000
-    volumes:
-      - ./dynamo-db-data:/data
-    working_dir: /home/dynamodblocal
-    command: ["-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb"]
-
-  # migration for dynamodb with privacy-notice table
-  migration-dynamodb:
-    image: amazon/aws-cli:2.17.44
-    container_name: migration-dynamodb
-    working_dir: /home/dynamodblocal
-    volumes:
-      - ./dynamo-db:/home/dynamodblocal
-    depends_on:
-      - dynamodb-local
-    environment:
-      - AWS_ACCESS_KEY_ID=local
-      - AWS_SECRET_ACCESS_KEY=local
-      - AWS_REGION=eu-west-1
-      - TABLES=privacy-notice,privacy-notice-acceptances
-    entrypoint: ./dynamo-migrations.sh
-
-  # GUI for DynamoDB Local
-  dynamodb-console:
-    image: aaronshaf/dynamodb-admin:4.6.1
-    container_name: dynamodb-admin
-    depends_on:
-      - dynamodb-local
-    restart: always
-    ports:
-      - "8001:8001"
-    environment:
-      - DYNAMO_ENDPOINT=http://dynamodb-local:8000
-      - AWS_REGION=eu-west-1
   # Redis local cache database used by the RateLimiter
   redis:
     image: "redis:7.2.5-alpine3.20"

diff --git a/docker/dynamo-db/dynamo-migrations.sh b/docker/dynamo-db/dynamo-migrations.sh
@@ -7,6 +7,10 @@ for TABLE_NAME in "${TABLES[@]}"; do
   if [ -z "$TABLE_EXISTS" ]; then
     echo "Table $TABLE_NAME does not exist. Creating table..."
     aws dynamodb create-table --cli-input-json file://./schema/$TABLE_NAME-dynamo-db.json --billing-mode PAY_PER_REQUEST --endpoint-url http://dynamodb-local:8000
-    aws dynamodb batch-write-item --request-items file://./seed/$TABLE_NAME-seed.json --endpoint-url http://dynamodb-local:8000
+
+    if [ -f "./seed/$TABLE_NAME-seed.json" ]; then
+      echo "Seeding table $TABLE_NAME..."
+      aws dynamodb batch-write-item --request-items file://./seed/$TABLE_NAME-seed.json --endpoint-url http://dynamodb-local:8000
+    fi
   fi
-done
+done
diff --git a/docker/dynamo-db/schema/platform-states.json → ...-db/schema/platform-states-dynamo-db.json b/docker/dynamo-db/schema/platform-states.json → ...-db/schema/platform-states-dynamo-db.json
@@ -11,7 +11,7 @@
       "IndexName": "Agreement",
       "KeySchema": [
         { "AttributeName": "GSIPK_consumerId_eserviceId", "KeyType": "HASH" },
-        { "AttributeName": "GSISK_agreementTimestamp", "KeyType": "HASH" }
+        { "AttributeName": "GSISK_agreementTimestamp", "KeyType": "RANGE" }
       ],
       "Projection": { "ProjectionType": "ALL" }
     }

diff --git a/...mo-db/schema/token-generation-states.json → ...ma/token-generation-states-dynamo-db.json b/...mo-db/schema/token-generation-states.json → ...ma/token-generation-states-dynamo-db.json
diff --git a/docker/dynamo-db/seed/privacy-notice-seed.json b/docker/dynamo-db/seed/privacy-notice-seed.json
@@ -39,6 +39,46 @@
           }
         }
       }
+    },
+    {
+      "PutRequest": {
+        "Item": {
+          "privacyNoticeId": {
+            "S": "0df21ff6-3e8f-4320-af8f-23dea9135d57"
+          },
+          "createdDate": {
+            "S": "2023-03-10T15:14:39.847Z"
+          },
+          "lastPublishedDate": {
+            "S": "2024-07-05T10:33:26.423Z"
+          },
+          "organizationId": {
+            "S": "5d6e166b-531a-4014-96b6-1cae7db0f64e"
+          },
+          "persistedAt": {
+            "S": "2024-09-04T15:01:24.467Z"
+          },
+          "privacyNoticeVersion": {
+            "M": {
+              "name": {
+                "S": "Interoperabilità - Backoffice - PP"
+              },
+              "publishedDate": {
+                "S": "2024-07-05T10:33:26.423Z"
+              },
+              "status": {
+                "S": "ACTIVE"
+              },
+              "version": {
+                "N": "11"
+              },
+              "versionId": {
+                "S": "65ae113e-e62e-4f68-b79d-5b5659385264"
+              }
+            }
+          }
+        }
+      }
     }
   ]
 }
diff --git a/...inio-seed/privacy-notices-path/0805a4a4-5c7e-4092-96bd-b6dfeda4adfa/it/privacy-notice.txt b/...inio-seed/privacy-notices-path/0805a4a4-5c7e-4092-96bd-b6dfeda4adfa/it/privacy-notice.txt
@@ -0,0 +1,2 @@
+Test privacy notice ToS file!
+Do you accept?
diff --git a/...inio-seed/privacy-notices-path/65ae113e-e62e-4f68-b79d-5b5659385264/it/privacy-notice.txt b/...inio-seed/privacy-notices-path/65ae113e-e62e-4f68-b79d-5b5659385264/it/privacy-notice.txt
@@ -0,0 +1,2 @@
+Test privacy notice PP file!
+Do you accept?
diff --git a/packages/backend-for-frontend/.env b/packages/backend-for-frontend/.env
@@ -45,7 +45,7 @@ RISK_ANALYSIS_DOCUMENTS_CONTAINER="interop-local-bucket"
 
 PRIVACY_NOTICES_CONTAINER="interop-local-bucket"
 PRIVACY_NOTICES_PATH="privacy-notices-path"
-PRIVACY_NOTICES_FILE_NAME="privacy-notice"
+PRIVACY_NOTICES_FILE_NAME="privacy-notice.txt"
 
 PRIVACY_NOTICES_TOS_UUID="6bf8412a-41a7-41a0-82dc-26286ce61b1a"
 PRIVACY_NOTICES_PP_UUID="0df21ff6-3e8f-4320-af8f-23dea9135d57"

diff --git a/packages/backend-for-frontend/aws.config.local b/packages/backend-for-frontend/aws.config.local
@@ -8,5 +8,5 @@ services=local
 kms=
   endpoint_url=http://localhost:4566
 
-dynamodb= 
-  endpoint_url=http://localhost:7001
+dynamodb=
+  endpoint_url=http://localhost:8085