Backport PR jupyterlab#11128: Run comparative benchmark

.gitignore

@@ -43,6 +43,7 @@ tests/**/.cache-loader
 galata/benchmark-results
 galata/playwright-report
 galata/test-results
+galata/*benchmark-expected.json
 
 package_json.tar.gz
 

galata/README.md

@@ -386,7 +386,7 @@ test.describe.serial('Table of Contents', () => {
 
 ## Benchmark
 
-Benchmark of JupyterLab is done automatically using Playwright. The actions measured are:
+Benchmark of JupyterLab is done using Playwright. The actions measured are:
 
 - Opening a file
 - Switching from the file to a simple text file
@@ -395,6 +395,13 @@ Benchmark of JupyterLab is done automatically using Playwright. The actions meas
 
 Two files are tested: a notebook with many code cells and another with many markdown cells.
 
+The test is run on the CI by comparing the result in the commit at which a PR branch started and the PR branch head on
+the same CI job to ensure using the same hardware.  
+The benchmark job is triggered on:
+
+- Approved PR review
+- PR review that contains the sentence `please run benchmark`
+
 The tests are located in the subfolder [test/benchmark](./test/benchmark). And they can be
 executed with the following command:
 
@@ -464,13 +471,14 @@ By default, Galata will generate a text report in the form of `markdown` table a
       { outputFile: 'lab-benchmark.json',
         vegaLiteConfigFactory: (
           allData: Array<IReportRecord>, // All test records
-          comparison: 'snapshot' | 'project'// Logic of test comparisons:'snapshot' or 'project'.
+          comparison?: 'snapshot' | 'project'// Logic of test comparisons:'snapshot' or 'project' - default 'snapshot'.
         ) => {
           // Return a Vega-Lite graph configuration object
           return {};
         }
         textReportFactory: (
-          allData: Array<IReportRecord> // All test records
+          allData: Array<IReportRecord>, // All test records
+          comparison?: 'snapshot' | 'project'// Logic of test comparisons:'snapshot' or 'project' - default 'snapshot'.
         ) => {
           // Return a promise of with the tuple [report content, file extension]
           return Promise.resolve(['My report content', 'md']);

galata/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jupyterlab/galata",
-  "version": "4.0.3",
+  "version": "4.1.0",
   "description": "JupyterLab UI Testing Framework",
   "homepage": "https://github.com/jupyterlab/jupyterlab",
   "bugs": {
@@ -54,6 +54,7 @@
     "@lumino/algorithm": "^1.3.3",
     "@lumino/coreutils": "^1.5.3",
     "@playwright/test": "^1.16.2",
+    "@stdlib/stats": "^0.0.13",
     "canvas": "^2.6.1",
     "fs-extra": "^9.0.1",
     "http-server": "^13.0.0",

galata/playwright-benchmark.config.js

@@ -18,7 +18,11 @@ module.exports = {
       { outputFile: 'lab-benchmark.json' }
     ]
   ],
-  use: { ...baseConfig.use, video: 'off' },
+  use: {
+    ...baseConfig.use,
+    video: 'off',
+    baseURL: process.env.TARGET_URL ?? 'http://127.0.0.1:8888'
+  },
   preserveOutput: 'failures-only',
   workers: 1
 };