added k6 load testing into the template

.github/workflows/k6-load-tests.yml

@@ -0,0 +1,170 @@
+name: Load Tests
+
+on: [pull_request]
+
+jobs:
+  test-base:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.base.sha }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Build
+      run: npm run build
+
+    - name: Start containers
+      run: docker-compose -f "docker-compose.yml" up -d --build
+
+    - name: Start Node.js API
+      run: npm run start:devnet &
+      # TODO: export environment config variables when base config is implemented into the template
+      # env:
+      #  SELF_URL: 'http://localhost:3000/assets-cdn'
+      #  PUBLIC_API_PORT: 3000
+      #  PUBLIC_API_PREFIX: 'assets-cdn'
+      #  PRIVATE_API_PORT: 4000
+
+    - name: Install k6
+      run: |
+        sudo gpg -k
+        sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+        echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+        sudo apt-get update
+        sudo apt-get install k6
+
+    - name: Wait for API to be ready
+      run: |
+        until curl --output /dev/null --silent --fail http://localhost:4000/hello; do
+          echo 'Waiting for API...'
+          sleep 1
+        done
+
+    - name: Run k6 Load Test
+      run: k6 run ./k6/script.js
+
+    - name: Upload result file for base branch
+      uses: actions/upload-artifact@v2
+      with:
+        name: base-results
+        path: k6/output/summary.json
+
+  test-head:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Set up Node.js
+      uses: actions/setup-node@v2
+      with:
+        node-version: '20'
+
+    - name: Install dependencies
+      run: npm ci
+
+    - name: Start containers
+      run: docker-compose -f "docker-compose.yml" up -d --build
+
+    - name: Build
+      run: npm run build
+
+    - name: Start Node.js API
+      run: npm run start:devnet &
+      # TODO: export environment config variables when base config is implemented into the template
+      # env:
+      #  SELF_URL: 'http://localhost:3000/assets-cdn'
+      #  PUBLIC_API_PORT: 3000
+      #  PUBLIC_API_PREFIX: 'assets-cdn'
+      #  PRIVATE_API_PORT: 4000
+
+    - name: Install k6
+      run: |
+        sudo gpg -k
+        sudo gpg --no-default-keyring --keyring /usr/share/keyrings/k6-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
+        echo "deb [signed-by=/usr/share/keyrings/k6-archive-keyring.gpg] https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
+        sudo apt-get update
+        sudo apt-get install k6
+
+    - name: Wait for API to be ready
+      run: |
+        until curl --output /dev/null --silent --fail http://localhost:4000/hello; do
+          echo 'Waiting for API...'
+          sleep 1
+        done
+
+    - name: Run k6 Load Test
+      run: k6 run ./k6/script.js
+
+    - name: Upload result file for head branch
+      uses: actions/upload-artifact@v2
+      with:
+        name: head-results
+        path: k6/output/summary.json
+
+
+  compare-results:
+    runs-on: ubuntu-latest
+    needs: [test-base, test-head]
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download all artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '20'
+
+      - name: Compare test results
+        run: |
+          node ./k6/compare-results.js ${{ github.event.pull_request.base.sha }} artifacts/base-results/summary.json ${{ github.event.pull_request.head.sha }} artifacts/head-results/summary.json report.md
+
+      - name: Render the report from the template
+        id: template
+        uses: chuhlomin/render-template@v1
+        if: github.event_name == 'pull_request'
+        with:
+          template: report.md
+          vars: |
+            base: ${{ github.event.pull_request.base.sha }}
+            head: ${{ github.event.pull_request.head.sha }}
+
+      - name: Upload the report markdown
+        uses: actions/upload-artifact@v3
+        if: github.event_name == 'pull_request'
+        with:
+          name: report-markdown
+          path: report.md
+
+      - name: Find the comment containing the report
+        id: fc
+        uses: peter-evans/find-comment@v2
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: 'k6 load testing comparison'
+
+      - name: Create or update the report comment
+        uses: peter-evans/create-or-update-comment@v2
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: ${{ steps.template.outputs.result }}
+          edit-mode: replace

k6/.gitignore

@@ -0,0 +1,2 @@
+output/*
+!output/.gitkeep

k6/compare-results.js

@@ -0,0 +1,99 @@
+const fs = require('fs');
+
+function generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath) {
+  // Load JSON outputs from k6
+  const baseMetrics = JSON.parse(fs.readFileSync(baseMetricsPath, 'utf8'));
+  const targetMetrics = JSON.parse(fs.readFileSync(targetMetricsPath, 'utf8'));
+
+  const baseData = extractMetrics(baseMetrics);
+  const targetData = extractMetrics(targetMetrics);
+
+  const table = generateTable(baseCommitHash, baseData, targetCommitHash, targetData);
+
+  fs.writeFileSync(outputPath, table);
+}
+
+function extractMetrics(metrics) {
+  const extractedMetrics = {};
+  const metricKeys = Object.keys(metrics.metrics);
+
+  for (const key of metricKeys) {
+    if (key.endsWith('_http_req_duration')) {
+      const values = metrics.metrics[key].values;
+      const avgResponseTime = values.avg;
+      const maxResponseTime = values.max;
+      const p90 = values['p(90)'];
+      const p95 = values['p(95)'];
+
+      const name = key.split('_')[0].charAt(0).toUpperCase() + key.split('_')[0].slice(1);
+
+      if (!extractedMetrics[name]) {
+        extractedMetrics[name] = { avgResponseTime, maxResponseTime, p90, p95 };
+      } else {
+        extractedMetrics[name].avgResponseTime = avgResponseTime;
+        extractedMetrics[name].maxResponseTime = maxResponseTime;
+        extractedMetrics[name].p90 = p90;
+        extractedMetrics[name].p95 = p95;
+      }
+    }
+  }
+
+  extractedMetrics['Test Run Duration'] = metrics.state.testRunDurationMs;
+
+  return extractedMetrics;
+}
+
+function generateTable(baseCommitHash, baseData, targetCommitHash, targetData) {
+  let table = `k6 load testing comparison.\nBase Commit Hash: ${baseCommitHash}\nTarget Commit Hash: ${targetCommitHash}\n`;
+  table += `Test duration: ${baseData['Test Run Duration'].toFixed(2)} ms (base) | ${targetData['Test Run Duration'].toFixed(2)} ms (target) \n\n`;
+  table += '| Endpoint \ Metric | Average | Max | p(90) | p(95) |\n';
+  table += '| ----------------- | ---- | ------ | ----- | ----- |\n';
+
+  for (const key of Object.keys(baseData)) {
+    if (key === 'Test Run Duration') {
+      continue;
+    }
+    const baseAvg = baseData[key].avgResponseTime;
+    const targetAvg = targetData[key].avgResponseTime;
+    const baseMax = baseData[key].maxResponseTime;
+    const targetMax = targetData[key].maxResponseTime;
+    const baseP90 = baseData[key].p90;
+    const targetP90 = targetData[key].p90;
+    const baseP95 = baseData[key].p95;
+    const targetP95 = targetData[key].p95;
+
+    table += `| **${key}** | ${computeCell(baseAvg, targetAvg)} | ${computeCell(baseMax, targetMax)} | ${computeCell(baseP90, targetP90)} | ${computeCell(baseP95, targetP95)} | \n`;
+  }
+
+  return table;
+}
+
+function computeCell(baseDuration, targetDuration) {
+  return `${baseDuration.toFixed(2)} ms -> ${targetDuration.toFixed(2)} ms (${getDifferencePercentage(baseDuration, targetDuration)})`;
+}
+
+function getDifferencePercentage(baseValue, targetValue) {
+  const difference = ((targetValue - baseValue) / baseValue) * 100;
+  const sign = difference >= 0 ? '+' : '';
+  let output = `${sign}${difference.toFixed(2)}%`;
+  if (difference > 20.0) {
+    output += ' ⚠️';
+  } else if (difference < -20) {
+    output += ' 🟢';
+  }
+
+  return `${output}`;
+}
+
+if (process.argv.length !== 7) {
+  console.error('Usage: node compare-results.js baseCommitHash baseMetricsPath targetCommitHash targetMetricsPath outputFile');
+  process.exit(1);
+}
+
+const baseCommitHash = process.argv[2];
+const baseMetricsPath = process.argv[3];
+const targetCommitHash = process.argv[4];
+const targetMetricsPath = process.argv[5];
+const outputPath = process.argv[6];
+
+generateComparisonTable(baseCommitHash, baseMetricsPath, targetCommitHash, targetMetricsPath, outputPath);

k6/script.js

@@ -0,0 +1,56 @@
+import http from 'k6/http';
+import { Trend } from 'k6/metrics';
+
+const BASE_URL = 'http://localhost:3000';
+
+const usersApiCallTrend = new Trend('users_http_req_duration', true);
+const examplesApiCallTrend = new Trend('examples_http_req_duration', true);
+const tokensApiCallTrend = new Trend('tokens_http_req_duration', true);
+
+export const options = {
+  scenarios: {
+    users: {
+      executor: 'constant-vus',
+      vus: 10,
+      duration: '1m',
+      gracefulStop: '0s',
+      exec: 'users',
+    },
+    tokens: {
+      executor: 'constant-vus',
+      vus: 10,
+      duration: '1m',
+      gracefulStop: '0s',
+      exec: 'tokens',
+    },
+    examples: {
+      executor: 'constant-vus',
+      vus: 10,
+      duration: '1m',
+      gracefulStop: '0s',
+      exec: 'examples',
+    },
+  },
+  discardResponseBodies: true,
+};
+
+export function users() {
+  const response = http.get(`${BASE_URL}/users`);
+  usersApiCallTrend.add(response.timings.duration);
+}
+
+export function examples() {
+  const response = http.get(`${BASE_URL}/examples`);
+  examplesApiCallTrend.add(response.timings.duration);
+}
+
+export function tokens() {
+  const response = http.get(`${BASE_URL}/tokens`);
+  tokensApiCallTrend.add(response.timings.duration);
+}
+
+export function handleSummary(data) {
+  return {
+    'k6/output/summary.json': JSON.stringify(data),
+  };
+}

package.json

@@ -72,7 +72,8 @@
     "copy-devnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.devnet.yaml ./apps/cache-warmer/config/config.yaml",
     "copy-testnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.testnet.yaml ./apps/cache-warmer/config/config.yaml",
     "copy-mainnet-config-cache-warmer": "cp ./apps/cache-warmer/config/config.mainnet.yaml ./apps/cache-warmer/config/config.yaml",
-    "copy-custom-config-cache-warmer": "cp ./apps/cache-warmer/config/config.custom.yaml ./apps/cache-warmer/config/config.yaml"
+    "copy-custom-config-cache-warmer": "cp ./apps/cache-warmer/config/config.custom.yaml ./apps/cache-warmer/config/config.yaml",
+    "load-test": "k6 run ./k6/script.js"
   },
   "dependencies": {
     "@multiversx/sdk-core": "^12.15.0",