Merge pull request #6 from pagopa/PRDP-130-add-performance-tests

[PRDP-130] feat: Add performance tests & pipeline
pagopa · Oct 4, 2023 · 5fa00a7 · 5fa00a7
2 parents 9f6631d + d2fde85
commit 5fa00a7
Show file tree

Hide file tree

Showing 18 changed files with 819 additions and 0 deletions.
diff --git a/.devops/performance-test-pipelines.yml b/.devops/performance-test-pipelines.yml
@@ -0,0 +1,83 @@
+# azure-pipelines.yml
+trigger: none
+
+parameters:
+  - name: "ENVIRONMENT"
+    displayName: "Environment"
+    type: string
+    values:
+      - "dev"
+      - "uat"
+    default: "uat"
+  - name: "TEST_TYPE"
+    displayName: "Test type"
+    type: string
+    values:
+      - "constant"
+      - "load"
+      - "spike"
+      - "stress"
+    default: "constant"
+  - name: "SCRIPT"
+    displayName: "Script name"
+    type: string
+    values:
+      - receipt_generator
+  - name: "DB_NAME"
+    displayName: "DB name"
+    type: string
+    values:
+      - pagopa_receipt_pdf_generatork6
+  - name: "PROCESS_TIME"
+    displayName: "Process time"
+    type: number
+    default: 5
+
+variables:
+  ${{ if eq(parameters['ENVIRONMENT'], 'dev') }}:
+    receiptCosmosSubscriptionKey: "$(DEV_RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY)"
+    receiptQueueAccountName: "pagopadweureceiptsfnsa"
+    receiptQueueName: "pagopa-d-weu-receipts-queue-receipt-waiting-4-gen"
+    azureSubscription: "$(TF_DEV_AZURE_SERVICE_CONNECTION)"
+    poolImage: "pagopa-dev-loadtest-linux"
+  ${{ if eq(parameters['ENVIRONMENT'], 'uat') }}:
+    receiptCosmosSubscriptionKey: "$(UAT_RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY)"
+    receiptQueueAccountName: "pagopauweureceiptsfnsa"
+    receiptQueueName: "pagopa-u-weu-receipts-queue-receipt-waiting-4-gen"
+    azureSubscription: "$(TF_UAT_AZURE_SERVICE_CONNECTION)"
+    poolImage: "pagopa-uat-loadtest-linux"
+
+pool:
+  name: $(poolImage)
+
+steps:
+  - script: |
+      cd ./performance-test/src
+      docker pull grafana/k6
+    displayName: Pull k6 image
+  - script: |
+      apt-get update
+      apt-get install azure-cli
+      az version
+    displayName: Install Azure CLI
+  - task: AzureCLI@2
+    displayName: Login Azure
+    inputs:
+      azureSubscription: "$(azureSubscription)"
+      scriptType: bash
+      scriptLocation: inlineScript
+      inlineScript: |
+        end="$(date -d '+1 day' +'%Y-%m-%d')"
+        export SAS_TOKEN="$(az storage queue generate-sas \
+        -n ${{ variables.receiptQueueName }} \
+        --account-name ${{ variables.receiptQueueAccountName }} \
+        --permissions apru \
+        --expiry $end)"
+        echo "##vso[task.setvariable variable=SAS_TOKEN]$SAS_TOKEN"
+  - script: |
+      cd ./performance-test
+      sh ./run_performance_test.sh ${{ parameters.ENVIRONMENT }} ${{ parameters.TEST_TYPE }} ${{ parameters.SCRIPT }} ${{ parameters.DB_NAME }} $(SAS_TOKEN) $RECEIPT_COSMOS_SUBSCRIPTION_KEY $PROCESS_TIME
+    displayName: Run k6 ${{ parameters.SCRIPT }} on ${{ parameters.ENVIRONMENT }}
+    env:
+      RECEIPT_COSMOS_SUBSCRIPTION_KEY: ${{ variables.receiptCosmosSubscriptionKey }}
+      PROCESS_TIME: ${{ parameters.PROCESS_TIME }}
diff --git a/performance-test/README.md b/performance-test/README.md
@@ -0,0 +1,14 @@
+# K6 tests
+
+This is a set of [k6](https://k6.io) tests.
+
+To invoke k6 tests use `run_performance_test.sh` script.
+
+
+## How to run 🚀
+
+Use this command to launch the tests:
+
+``` shell
+sh run_performance_test.sh <local|dev|uat> <load|stress|spike|...> <script-filename> <DB-name> <receipts_queue_key> <receipt-cosmos-DB-key>
+```
diff --git a/performance-test/docker-compose.yaml b/performance-test/docker-compose.yaml
@@ -0,0 +1,29 @@
+version: '3.3'
+services:
+  k6:
+    image: grafana/k6
+    container_name: k6
+    volumes:
+      - '${PWD}/src:/scripts'
+    environment:
+      - RECEIPT_QUEUE_SUBSCRIPTION_KEY=${receipts_queue_key}
+      - RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY=${receipts_cosmos_key}
+      - PROCESS_TIME=${process_time}
+      - VARS=${env}.environment.json
+      - TEST_TYPE=/scripts/test-types/${type}.json
+      - K6_OUT=influxdb=http://nginx:8086/${db_name}
+    command: run /scripts/${script}.js
+    depends_on:
+      - nginx
+
+  nginx:
+    image: nginx
+    container_name: nginx
+    volumes:
+      - '${PWD}/nginx/nginx.conf:/etc/nginx/nginx.conf'
+    environment:
+      - ENVIRONMENT=${env}
+    ports:
+      - "8086:8086"
+      - "80:80"
+
diff --git a/performance-test/nginx/nginx.conf b/performance-test/nginx/nginx.conf
@@ -0,0 +1,16 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    server {
+        listen 8086;
+        location / {
+          proxy_pass  https://api.uat.platform.pagopa.it/shared/influxdb/v1/;
+          proxy_http_version 1.1;
+          proxy_set_header Host api.uat.platform.pagopa.it;
+          proxy_pass_request_headers on;
+        }
+    }
+}
+
diff --git a/performance-test/run_performance_test.sh b/performance-test/run_performance_test.sh
@@ -0,0 +1,39 @@
+# sh run_performance_test.sh <local|dev|uat|prod> <load|stress|spike|soak|...> <script-name> <db-name> <biz-event-cosmos-subkey> <receipts-cosmos-subkey>
+
+ENVIRONMENT=$1
+TYPE=$2
+SCRIPT=$3
+DB_NAME=$4
+RECEIPT_QUEUE_SUBSCRIPTION_KEY=$5
+RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY=$6
+PROCESS_TIME=$7
+
+if [ -z "$ENVIRONMENT" ]
+then
+  echo "No env specified: sh run_performance_test.sh <local|dev|uat|prod> <load|stress|spike|soak|...> <script-name> <db-name> <receipts-queue-subkey> <receipts-cosmos-subkey> <process_time>"
+  exit 1
+fi
+
+if [ -z "$TYPE" ]
+then
+  echo "No test type specified: sh run_performance_test.sh <local|dev|uat|prod> <load|stress|spike|soak|...> <script-name> <db-name> <receipts-queue-subkey> <receipts-cosmos-subkey> <process_time>"
+  exit 1
+fi
+if [ -z "$SCRIPT" ]
+then
+  echo "No script name specified: sh run_performance_test.sh <local|dev|uat|prod> <load|stress|spike|soak|...> <script-name> <db-name> <receipts-queue-subkey> <receipts-cosmos-subkey> <process_time>"
+  exit 1
+fi
+
+export env=${ENVIRONMENT}
+export type=${TYPE}
+export script=${SCRIPT}
+export db_name=${DB_NAME}
+export receipts_queue_key=${RECEIPT_QUEUE_SUBSCRIPTION_KEY}
+export receipts_cosmos_key=${RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY}
+export process_time=${PROCESS_TIME}
+
+stack_name=$(cd .. && basename "$PWD")
+docker compose -p "${stack_name}-k6" up -d --remove-orphans --force-recreate --build
+docker logs -f k6
+docker stop nginx
diff --git a/performance-test/src/README.md b/performance-test/src/README.md
@@ -0,0 +1,49 @@
+# K6 tests for _ReceiptsToDatastore_ project
+
+[k6](https://k6.io/) is a load testing tool. 👀 See [here](https://k6.io/docs/get-started/installation/) to install it.
+
+- [01. Receipt datastore function](#01-receipt-datastore-function)
+
+This is a set of [k6](https://k6.io) tests related to the _Biz Events to Datastore_ initiative.
+
+To invoke k6 test passing parameter use -e (or --env) flag:
+
+```
+-e MY_VARIABLE=MY_VALUE
+```
+
+## 01. Receipt datastore function
+
+Test the receipt datastore function:
+
+```
+k6 run --env VARS=local.environment.json --env TEST_TYPE=./test-types/load.json --env RECEIPT_QUEUE_SUBSCRIPTION_KEY=<your-secret> --env RECEIPT_COSMOS_DB_SUBSCRIPTION_KEY=<your-secret> receipt_generator.js
+```
+
+where the mean of the environment variables is:
+
+```json
+  "environment": [
+    {
+      "env": "local",
+      "receiptCosmosDBURI": "",
+      "receiptDatabaseID":"",
+      "receiptContainerID":"",
+      "receiptQueueAccountName": "",
+      "receiptQueueName": "",
+      "processTime":""
+    }
+  ]
+```
+
+`receiptCosmosDBURI`: CosmosDB url to access Receipts CosmosDB REST API
+
+`receiptDatabaseID`: database name to access Receipts Cosmos DB REST API
+
+`receiptContainerID`: collection name to access Receipts Cosmos DB REST API
+
+`receiptQueueAccountName`: storage account name to access Receipts Queue
+
+`receiptQueueName`: queue name for the Receipts,
+
+`processTime`: boundary time taken by azure function to fetch the payment event and save it in the datastore
diff --git a/performance-test/src/dev.environment.json b/performance-test/src/dev.environment.json
@@ -0,0 +1,12 @@
+{
+  "environment": [
+    {
+      "env": "dev",
+      "receiptCosmosDBURI": "https://pagopa-d-weu-receipts-ds-cosmos-account.documents.azure.com:443/",
+      "receiptDatabaseID":"db",
+      "receiptContainerID":"receipts",
+      "receiptQueueAccountName": "pagopadweureceiptsfnsa",
+      "receiptQueueName": "pagopa-d-weu-receipts-queue-receipt-waiting-4-gen"
+    }
+  ]
+}
diff --git a/performance-test/src/local.environment.json b/performance-test/src/local.environment.json
@@ -0,0 +1,12 @@
+{
+  "environment": [
+    {
+      "env": "local",
+      "receiptCosmosDBURI": "https://pagopa-d-weu-receipts-ds-cosmos-account.documents.azure.com:443/",
+      "receiptDatabaseID":"db",
+      "receiptContainerID":"receipts",
+      "receiptQueueAccountName": "pagopadweureceiptsfnsa",
+      "receiptQueueName": "pagopa-d-weu-receipts-queue-receipt-waiting-4-gen"
+    }
+  ]
+}
diff --git a/performance-test/src/modules/common.js b/performance-test/src/modules/common.js
@@ -0,0 +1,114 @@
+
+export function randomString(length, charset) {
+    let res = '';
+    while (length--) res += charset[(Math.random() * charset.length) | 0];
+    return res;
+}
+
+export function createEvent(id) {
+    const idPA = randomString(11, "0123456789");
+    const idPSP = randomString(11, "0123456789");
+	let json_event = {
+		"id": id,
+		"version": "2",
+		"complete": "false",
+		"receiptId": "0095ff2bafec4bc0a719c9bf003aee4a",
+		"missingInfo": [
+			"idPaymentManager",
+			"psp.pspPartitaIVA",
+			"paymentInfo.primaryCiIncurredFee",
+			"paymentInfo.idBundle",
+			"paymentInfo.idCiBundle",
+			"paymentInfo.metadata"
+		],
+        "debtorPosition": {
+            "modelType": "2",
+            "noticeNumber": randomString(18, "0123456789"),
+            "iuv": randomString(17, "0123456789")
+        },
+        "creditor": {
+            "idPA": idPA,
+            "idBrokerPA": idPA,
+            "idStation": `${idPA}_08`,
+            "companyName": "PA test_company"
+        },
+        "psp": {
+            "idPsp": idPSP,
+            "idBrokerPsp": idPSP,
+            "idChannel": `${idPSP}_08`,
+            "psp": "test_PSP"
+        },
+		"debtor": {
+			"fullName": "John Doe",
+			"entityUniqueIdentifierType": "F",
+            "entityUniqueIdentifierValue": "GENERATOR_PERF_TEST",
+			"streetName": "street",
+			"civicNumber": "12",
+			"postalCode": "89020",
+			"city": "city",
+			"stateProvinceRegion": "MI",
+			"country": "IT",
+			"eMail": "[email protected]"
+		},
+		"payer": {
+			"fullName": "John Doe",
+			"entityUniqueIdentifierType": "F",
+			"entityUniqueIdentifierValue": "GENERATOR_PERF_TEST",
+			"streetName": "street",
+			"civicNumber": "12",
+			"postalCode": "89020",
+			"city": "city",
+			"stateProvinceRegion": "MI",
+			"country": "IT",
+			"eMail": "[email protected]"
+		},
+		"paymentInfo": {
+			"paymentDateTime": "2022-12-13T01:52:02.926587",
+			"applicationDate": "2021-10-01",
+			"transferDate": "2021-10-02",
+			"dueDate": "2021-07-31",
+			"paymentToken": "0095ff2bafec4bc0a719c9bf003aee4a",
+			"amount": "70.0",
+			"fee": "2.0",
+			"totalNotice": "1",
+			"paymentMethod": "creditCard",
+			"touchpoint": "app",
+			"remittanceInformation": "TARI 2021",
+			"description": "TARI 2021"
+		},
+		"transferList": [
+            {
+                "fiscalCodePA": idPA,
+                "companyName": "PA test_company",
+                "amount": "2.00",
+                "transferCategory": "paTest",
+                "remittanceInformation": "/RFB/00202200000217527/5.00/TXT/"
+            },
+            {
+                "fiscalCodePA": randomString(11, "0123456789"),
+                "companyName": "PA test_company",
+                "amount": "8.00",
+                "transferCategory": "paTest",
+                "remittanceInformation": "/RFB/00202200000217527/5.00/TXT/"
+            }
+		],
+		"eventStatus": "DONE",
+		"eventRetryEnrichmentCount": 0
+	}
+    return json_event;
+}
+
+export function createReceipt(id) {
+    let receipt = 
+	{
+		"eventId": id,
+		"eventData": {
+			"payerFiscalCode": "GENERATOR_PERF_TEST",
+			"debtorFiscalCode": "GENERATOR_PERF_TEST"
+		},
+		"status": "INSERTED",
+		"numRetry": 0,
+		"id": id
+	}
+    return receipt
+}