chore(repo): Introduce integration test for vite with sdk-node (#1921)

* feat(repo): Add integration test for express + vite

* feat(repo): Enable express tests in CICD

* fix(repo): Copy integration tests to OS temp path

This change will help us avoid accidentally using
top-level node_modules/ from the current monorepo
and will allow us to run tests isolated from the
monorepo related dependencies.
We had to use a folder outside the monorepo for
the integration tests to avoid having the npm
module resolution algorithm find unrelated dependencies.

* fix(repo): Exit integration tests on application error exit code

* fix(repo): Link local clerk packages when version is missing

* fix(repo): Fix `npm run nuke` yalc cleanup

.github/workflows/ci.yml

@@ -132,7 +132,7 @@ jobs:
 
     strategy:
       matrix:
-        test-name: [ 'generic', 'nextjs' ]
+        test-name: [ 'generic', 'nextjs', 'express' ]
 
     steps:
       - name: Checkout Repo

integration/constants.ts

@@ -1,9 +1,10 @@
 /* eslint-disable turbo/no-undeclared-env-vars */
+import * as os from 'node:os';
 import * as path from 'node:path';
 
 export const constants = {
-  TMP_DIR: path.join(process.cwd(), '.temp_integration'),
-  APPS_STATE_FILE: path.join(process.cwd(), '.temp_integration', 'state.json'),
+  TMP_DIR: path.join(os.tmpdir(), '.temp_integration'),
+  APPS_STATE_FILE: path.join(os.tmpdir(), '.temp_integration', 'state.json'),
   /**
    * A URL to a running app that will be used to run the tests against.
    * This is usually used when running the app has been started manually,

integration/models/application.ts

@@ -61,9 +61,8 @@ export const application = (config: ApplicationConfig, appDirPath: string, appDi
         stderr: opts.detached ? fs.openSync(stderrFilePath, 'a') : undefined,
         log: opts.detached ? undefined : log,
       });
-      // TODO @dimitris: Fail early if server exits
-      // const shouldRetry = () => proc.exitCode !== 0 && proc.exitCode !== null;
-      await waitForServer(serverUrl, { log, maxAttempts: Infinity });
+      const shouldExit = () => !!proc.exitCode && proc.exitCode !== 0;
+      await waitForServer(serverUrl, { log, maxAttempts: Infinity, shouldExit });
       log(`Server started at ${serverUrl}, pid: ${proc.pid}`);
       cleanupFns.push(() => awaitableTreekill(proc.pid, 'SIGKILL'));
       state.serverUrl = serverUrl;
@@ -85,7 +84,6 @@ export const application = (config: ApplicationConfig, appDirPath: string, appDi
     serve: async (opts: { port?: number; manualStart?: boolean } = {}) => {
       const port = opts.port || (await getPort());
       const serverUrl = `http://localhost:${port}`;
-      const log = logger.child({ prefix: 'serve' }).info;
       // If this is ever used as a background process, we need to make sure
       // it's not using the log function. See the dev() method above
       const proc = run(scripts.serve, { cwd: appDirPath, env: { PORT: port.toString() } });

integration/models/applicationConfig.ts

@@ -1,5 +1,6 @@
 import * as path from 'node:path';
 
+import { constants } from '../constants';
 import { createLogger, fs } from '../scripts';
 import { application } from './application.js';
 import type { EnvironmentConfig } from './environment';
@@ -62,10 +63,9 @@ export const applicationConfig = () => {
     commit: async (opts?: { stableHash?: string }) => {
       const { stableHash } = opts || {};
       logger.info(`Creating project "${name}"`);
-      const TMP_DIR = path.join(process.cwd(), '.temp_integration');
 
       const appDirName = stableHash || `${name}__${Date.now()}__${hash()}`;
-      const appDirPath = path.resolve(TMP_DIR, appDirName);
+      const appDirPath = path.resolve(constants.TMP_DIR, appDirName);
 
       // Copy template files
       for (const template of templates) {

integration/presets/express.ts

@@ -0,0 +1,18 @@
+import { constants } from '../constants';
+import { applicationConfig } from '../models/applicationConfig';
+import { templates } from '../templates';
+
+const clerkNodeLocal = `file:${process.cwd()}/packages/sdk-node`;
+const vite = applicationConfig()
+  .setName('express-vite')
+  .useTemplate(templates['express-vite'])
+  .setEnvFormatter('public', key => `VITE_${key}`)
+  .addScript('setup', 'npm i --prefer-offline')
+  .addScript('dev', 'npm run dev')
+  .addScript('build', 'npm run build')
+  .addScript('serve', 'npm run start')
+  .addDependency('@clerk/clerk-sdk-node', constants.E2E_CLERK_VERSION || clerkNodeLocal);
+
+export const express = {
+  vite,
+} as const;

integration/presets/index.ts

@@ -1,11 +1,13 @@
 import { envs } from './envs';
+import { express } from './express';
 import { createLongRunningApps } from './longRunningApps';
 import { next } from './next';
 import { react } from './react';
 import { remix } from './remix';
 
 export const appConfigs = {
   envs,
+  express,
   longRunningApps: createLongRunningApps(),
   next,
   react,

integration/presets/longRunningApps.ts

@@ -1,6 +1,7 @@
 import type { LongRunningApplication } from '../models/longRunningApplication';
 import { longRunningApplication } from '../models/longRunningApplication';
 import { envs } from './envs';
+import { express } from './express';
 import { next } from './next';
 import { react } from './react';
 import { remix } from './remix';
@@ -12,6 +13,7 @@ import { remix } from './remix';
  */
 export const createLongRunningApps = () => {
   const configs = [
+    { id: 'express.vite.withEmailCodes', config: express.vite, env: envs.withEmailCodes },
     { id: 'react.vite.withEmailCodes', config: react.vite, env: envs.withEmailCodes },
     { id: 'react.vite.withEmailLinks', config: react.vite, env: envs.withEmailLinks },
     { id: 'remix.node.withEmailCodes', config: remix.remixNode, env: envs.withEmailCodes },

integration/presets/next.ts

@@ -2,6 +2,7 @@ import { constants } from '../constants';
 import { applicationConfig } from '../models/applicationConfig.js';
 import { templates } from '../templates/index.js';
 
+const clerkNextjsLocal = `file:${process.cwd()}/packages/nextjs`;
 const appRouter = applicationConfig()
   .setName('next-app-router')
   .useTemplate(templates['next-app-router'])
@@ -11,7 +12,7 @@ const appRouter = applicationConfig()
   .addScript('build', 'npm run build')
   .addScript('serve', 'npm run start')
   .addDependency('next', constants.E2E_NEXTJS_VERSION)
-  .addDependency('@clerk/nextjs', constants.E2E_CLERK_VERSION);
+  .addDependency('@clerk/nextjs', constants.E2E_CLERK_VERSION || clerkNextjsLocal);
 
 const appRouterTurbo = appRouter
   .clone()

integration/presets/react.ts

@@ -2,6 +2,9 @@ import { constants } from '../constants';
 import { applicationConfig } from '../models/applicationConfig';
 import { templates } from '../templates';
 
+const clerkReactLocal = `file:${process.cwd()}/packages/react`;
+const clerkThemesLocal = `file:${process.cwd()}/packages/themes`;
+
 const cra = applicationConfig()
   .setName('react-cra')
   .useTemplate(templates['react-cra'])
@@ -10,8 +13,8 @@ const cra = applicationConfig()
   .addScript('dev', 'npm run start')
   .addScript('build', 'npm run build')
   .addScript('serve', 'npm run start')
-  .addDependency('@clerk/clerk-react', constants.E2E_CLERK_VERSION)
-  .addDependency('@clerk/themes', constants.E2E_CLERK_VERSION);
+  .addDependency('@clerk/clerk-react', constants.E2E_CLERK_VERSION || clerkReactLocal)
+  .addDependency('@clerk/themes', constants.E2E_CLERK_VERSION || clerkThemesLocal);
 
 const vite = cra
   .clone()

integration/presets/remix.ts

@@ -1,14 +1,17 @@
+import { constants } from '../constants';
 import { applicationConfig } from '../models/applicationConfig.js';
 import { templates } from '../templates/index.js';
 
+const clerkRemixLocal = `file:${process.cwd()}/packages/remix`;
 const remixNode = applicationConfig()
   .setName('remix-node')
   .useTemplate(templates['remix-node'])
   .setEnvFormatter('public', key => `${key}`)
   .addScript('setup', 'npm i --prefer-offline')
   .addScript('dev', 'npm run dev')
-  .addScript('build', 'npm run build');
-// .addScript('serve', 'npm run start');
+  .addScript('build', 'npm run build')
+  .addScript('serve', 'npm run start')
+  .addDependency('@clerk/remix', constants.E2E_CLERK_VERSION || clerkRemixLocal);
 
 export const remix = {
   remixNode,

integration/scripts/clerkJsServer.ts

@@ -3,6 +3,7 @@
 import os from 'node:os';
 import path from 'node:path';
 
+import { constants } from '../constants';
 import { stateFile } from '../models/stateFile';
 import { awaitableTreekill, fs, waitForServer } from './index';
 import { run } from './run';
@@ -40,9 +41,8 @@ const serveFromTempDir = async () => {
   const port = 18211;
   const serverUrl = `http://localhost:${port}`;
   const now = Date.now();
-  const TMP_DIR = path.join(process.cwd(), '.temp_integration');
-  const stdoutFilePath = path.resolve(TMP_DIR, `clerkJsHttpServer.${now}.log`);
-  const stderrFilePath = path.resolve(TMP_DIR, `clerkJsHttpServer.${now}.err.log`);
+  const stdoutFilePath = path.resolve(constants.TMP_DIR, `clerkJsHttpServer.${now}.log`);
+  const stderrFilePath = path.resolve(constants.TMP_DIR, `clerkJsHttpServer.${now}.err.log`);
   const clerkJsTempDir = getClerkJsTempDir();
   const proc = run(`node_modules/.bin/http-server ${clerkJsTempDir} -d --gzip --cors -a localhost`, {
     cwd: process.cwd(),

integration/scripts/waitForServer.ts

@@ -1,8 +1,19 @@
+type WaitForServerArgsType = {
+  log;
+  delayInMs?: number;
+  maxAttempts?: number;
+  shouldExit?: () => boolean;
+};
+
 // Poll a url until it returns a 200 status code
-export const waitForServer = async (url: string, opts: { delayInMs?: number; maxAttempts?: number; log }) => {
-  const { delayInMs = 1000, maxAttempts = 20, log } = opts || {};
+export const waitForServer = async (url: string, opts: WaitForServerArgsType) => {
+  const { log, delayInMs = 1000, maxAttempts = 20, shouldExit = () => false } = opts;
   let attempts = 0;
   while (attempts < maxAttempts) {
+    if (shouldExit()) {
+      throw new Error(`Polling ${url} failed after ${maxAttempts} attempts (due to forced exit)`);
+    }
+
     try {
       log(`Polling ${url}...`);
       const res = await fetch(url);
@@ -15,5 +26,6 @@ export const waitForServer = async (url: string, opts: { delayInMs?: number; max
     attempts++;
     await new Promise(resolve => setTimeout(resolve, delayInMs));
   }
+
   throw new Error(`Polling ${url} failed after ${maxAttempts} attempts`);
 };

integration/templates/express-vite/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

integration/templates/express-vite/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "express-vite",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "build": "vite build",
+    "dev": "PORT=$PORT ts-node src/server/main.ts",
+    "lint": "eslint src --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
+    "preview": "vite preview --port $PORT --no-open",
+    "start": "PORT=$PORT ts-node src/server/main.ts"
+  },
+  "dependencies": {
+    "dotenv": "^16.3.1",
+    "ejs": "^3.1.6",
+    "express": "^4.18.2",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.3",
+    "vite-express": "^0.11.0"
+  },
+  "devDependencies": {
+    "@types/express": "^4.17.15",
+    "@types/node": "^18.11.18",
+    "nodemon": "^2.0.20",
+    "vite": "^4.0.4"
+  }
+}

integration/templates/express-vite/src/server/main.ts

@@ -0,0 +1,60 @@
+// Should be at the top of the file - used to load clerk secret key
+import * as dotenv from 'dotenv';
+dotenv.config();
+
+import { clerkClient } from '@clerk/clerk-sdk-node';
+import express from 'express';
+import ViteExpress from 'vite-express';
+
+const app = express();
+
+app.set('view engine', 'ejs');
+app.set('views', 'src/views');
+
+app.get('/api/protected', [clerkClient.expressRequireAuth() as any], (_req: any, res: any) => {
+  res.send('Protected API response').end();
+});
+
+app.get('/sign-in', (_req: any, res: any) => {
+  return res.render('sign-in.ejs', {
+    publishableKey: process.env.VITE_CLERK_PUBLISHABLE_KEY,
+    signInUrl: process.env.CLERK_SIGN_IN_URL,
+  });
+});
+
+app.get('/', (_req: any, res: any) => {
+  return res.render('index.ejs', {
+    publishableKey: process.env.VITE_CLERK_PUBLISHABLE_KEY,
+    signInUrl: process.env.CLERK_SIGN_IN_URL,
+  });
+});
+
+app.get('/sign-up', (_req: any, res: any) => {
+  return res.render('sign-up.ejs', {
+    publishableKey: process.env.VITE_CLERK_PUBLISHABLE_KEY,
+    signUpUrl: process.env.CLERK_SIGN_UP_URL,
+  });
+});
+
+app.get('/protected', (_req: any, res: any) => {
+  return res.render('protected.ejs', {
+    publishableKey: process.env.VITE_CLERK_PUBLISHABLE_KEY,
+    signInUrl: process.env.CLERK_SIGN_IN_URL,
+    signUpUrl: process.env.CLERK_SIGN_UP_URL,
+  });
+});
+
+// Handle authentication error, otherwise application will crash
+// @ts-ignore
+app.use((err, req, res, next) => {
+  if (err) {
+    console.error(err);
+    res.status(401).end();
+    return;
+  }
+
+  return next();
+});
+
+const port = parseInt(process.env.PORT as string) || 3002;
+ViteExpress.listen(app, port, () => console.log(`Server is listening on port ${port}...`));

integration/templates/express-vite/src/views/index.ejs

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <script
+      data-clerk-publishable-key="<%= publishableKey %>"
+      onLoad="startClerk()"
+      crossorigin="anonymous"
+      async=""
+      src="https://clerk.clerk.com/npm/@clerk/clerk-js@4/dist/clerk.browser.js"
+    ></script>
+  </head>
+  <body>
+    <div id="app"></div>
+    <div id="user-state"></div>
+
+    <script>
+      window.startClerk = async () => {
+        await Clerk.load({ signInUrl: '<%= signInUrl %>' });
+        const appEl = document.querySelector('#app');
+        const controlStateEl = document.querySelector('#user-state');
+
+        if (Clerk.user) {
+          Clerk.mountUserButton(appEl);
+          controlStateEl.innerHTML = 'SignedIn';
+        } else {
+          controlStateEl.innerHTML = 'SignedOut';
+        }
+      };
+    </script>
+  </body>
+</html>

integration/templates/express-vite/src/views/protected.ejs

@@ -0,0 +1,32 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <script
+      data-clerk-publishable-key="<%= publishableKey %>"
+      onLoad="startClerk()"
+      crossorigin="anonymous"
+      async=""
+      src="https://clerk.clerk.com/npm/@clerk/clerk-js@4/dist/clerk.browser.js"
+    ></script>
+  </head>
+  <body>
+    <div id="app"></div>
+    <div id="user-state"></div>
+
+    <script>
+      window.startClerk = async () => {
+        await Clerk.load({ signInUrl: '<%= signInUrl %>' });
+
+        if (Clerk.user) {
+          const apiResponse = await fetch('/api/protected').then(res => res.text());
+
+          const div = document.createElement('div');
+          div.setAttribute('data-test-id', 'protected-api-response');
+          div.innerText = apiResponse;
+          document.body.appendChild(div);
+        }
+      };
+    </script>
+  </body>
+</html>