From be151616be1b6440850243cac296837dcd9816fa Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 01:49:17 +0900 Subject: [PATCH 01/17] debug flaky test --- tests/specs/cli.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 31901a83..0429d4de 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -314,7 +314,7 @@ export default testSuite(({ describe }, node: NodeApis) => { expect(output).toMatch(/EXIT_CODE:\s+130/); }, 10_000); - test('Catchable', async () => { + test('Catchable', async ({ onTestFail }) => { const output = await ptyShell( [ // Windows doesn't support shebangs @@ -324,6 +324,12 @@ export default testSuite(({ describe }, node: NodeApis) => { ], ); + onTestFail(() => { + console.log({ + output, + }); + }); + expectMatchInOrder(output, [ 'READY\r\n', process.platform === 'win32' ? '' : '^C', From dab7f042d81129db887bcf426acc91d937ef5b52 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 02:04:24 +0900 Subject: [PATCH 02/17] debug flaky test --- src/cli.ts | 19 +++++++++++++++++++ tests/specs/cli.ts | 6 +++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index 290ae9b0..0e81d94e 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,6 +15,12 @@ import { import { isFeatureSupported, testRunnerGlob } from './utils/node-features.js'; import { createIpcServer } from './utils/ipc/server.js'; +const debug = (...messages: any[]) => { + if (process.env.DEBUG) { + console.log(...messages); + } +}; + const relaySignals = ( childProcess: ChildProcess, ipcSocket: Server, @@ -31,6 +37,12 @@ const relaySignals = ( } }); + /** + * Wait for signal from preflight bindHiddenSignalsHandler + * Ideally the timeout should be as low as possible + * since the child lets the parent know that it received + * the signal + */ const waitForSignalFromChild = () => { const p = new Promise((resolve) => { // Aribrary timeout based on flaky tests @@ -62,12 +74,19 @@ const relaySignals = ( */ const signalFromChild = await waitForSignalFromChild(); + debug({ + signalFromChild, + }); + /** * If child didn't receive a signal, it's either because it was * sent to the parent directly via kill PID or the child is * unresponsive (e.g. infinite loop). Relay signal to child. */ if (signalFromChild !== signal) { + debug('killing child', { + signal, + }); childProcess.kill(signal); /** diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 0429d4de..031989b2 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -199,7 +199,11 @@ export default testSuite(({ describe }, node: NodeApis) => { test(signal, async ({ onTestFail }) => { const tsxProcess = tsx([ path.join(fixture.path, 'catch-signals.js'), - ]); + ], { + env: { + DEBUG: '1', + }, + }); tsxProcess.stdout!.once('data', () => { tsxProcess.kill(signal, { From a16b13c690770d2523c818cfa950786c5d2b667a Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 02:10:11 +0900 Subject: [PATCH 03/17] debug flaky test --- tests/specs/cli.ts | 11 ++++++----- tests/utils/pty-shell/index.ts | 4 +++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 031989b2..916391b7 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -199,11 +199,7 @@ export default testSuite(({ describe }, node: NodeApis) => { test(signal, async ({ onTestFail }) => { const tsxProcess = tsx([ path.join(fixture.path, 'catch-signals.js'), - ], { - env: { - DEBUG: '1', - }, - }); + ]); tsxProcess.stdout!.once('data', () => { tsxProcess.kill(signal, { @@ -326,6 +322,11 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => stdout.includes('READY') && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], + { + env: { + DEBUG: '1', + }, + }, ); onTestFail(() => { diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 89da49c8..2828c460 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -1,5 +1,5 @@ import { fileURLToPath } from 'url'; -import { execaNode } from 'execa'; +import { execaNode, type NodeOptions } from 'execa'; import stripAnsi from 'strip-ansi'; export const isWindows = process.platform === 'win32'; @@ -22,11 +22,13 @@ const getStdin = ( export const ptyShell = ( stdins: StdInArray, + options?: NodeOptions<'utf8'>, ) => new Promise((resolve, reject) => { const childProcess = execaNode( fileURLToPath(new URL('node-pty.mjs', import.meta.url)), [shell], { + ...options, stdio: 'pipe', }, ); From 8a57e09ce9da13cef916c08a6b438c595b70cecb Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 02:36:49 +0900 Subject: [PATCH 04/17] debug flaky test --- tests/specs/cli.ts | 1 + tests/utils/pty-shell/index.ts | 63 +++++++++++++++++++++++++++------- 2 files changed, 52 insertions(+), 12 deletions(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 916391b7..b8c4226c 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -322,6 +322,7 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => stdout.includes('READY') && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], + 8000, { env: { DEBUG: '1', diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 2828c460..29b6308a 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -1,3 +1,4 @@ +import { setTimeout } from 'timers/promises'; import { fileURLToPath } from 'url'; import { execaNode, type NodeOptions } from 'execa'; import stripAnsi from 'strip-ansi'; @@ -20,10 +21,23 @@ const getStdin = ( ); }; -export const ptyShell = ( +const throwTimeout = ( + timeout: number, + abortController: AbortController, +) => ( + setTimeout(timeout, true, abortController).then( + () => { + throw new Error('Timeout'); + }, + () => {}, + ) +); + +export const ptyShell = async ( stdins: StdInArray, + timeout?: number, options?: NodeOptions<'utf8'>, -) => new Promise((resolve, reject) => { +) => { const childProcess = execaNode( fileURLToPath(new URL('node-pty.mjs', import.meta.url)), [shell], @@ -33,8 +47,6 @@ export const ptyShell = ( }, ); - childProcess.on('error', reject); - let currentStdin = getStdin(stdins); let buffer = Buffer.alloc(0); @@ -53,12 +65,39 @@ export const ptyShell = ( } }); - childProcess.stderr!.on('data', (data) => { - reject(new Error(stripAnsi(data.toString()))); - }); + const abortController = new AbortController(); - childProcess.on('exit', () => { - const outString = stripAnsi(buffer.toString()); - resolve(outString); - }); -}); + const promises = [ + new Promise((resolve, reject) => { + childProcess.on('error', reject); + childProcess.stderr!.on('data', (data) => { + reject(new Error(stripAnsi(data.toString()))); + }); + childProcess.on('exit', resolve); + }), + ]; + + if (typeof timeout === 'number') { + promises.push(throwTimeout(timeout, abortController)); + } + + try { + await Promise.race(promises); + } catch (error) { + if (error instanceof Error && error.message === 'Timeout') { + childProcess.kill(); + const outString = stripAnsi(buffer.toString()); + + console.log('Incomplete output', { + outString, + stdins, + }); + } + + throw error; + } finally { + abortController.abort(); + } + + return stripAnsi(buffer.toString()); +}; From d14bea2c81167591c328863126ea19a9f8d5b680 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 11:22:23 +0900 Subject: [PATCH 05/17] debug flaky test --- tests/specs/cli.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index b8c4226c..792de6f7 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -322,7 +322,7 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => stdout.includes('READY') && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], - 8000, + 8_000, { env: { DEBUG: '1', @@ -353,6 +353,12 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => /\d+\r\n/.test(stdout) && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], + 8_000, + { + env: { + DEBUG: '1', + }, + }, ); expect(output).toMatch(/EXIT_CODE:\s+130/); }, 10_000); From a3c04fddfe66b188b83c91f518f85784cce374d5 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 11:42:43 +0900 Subject: [PATCH 06/17] debug flaky test --- tests/specs/cli.ts | 4 ++-- tests/utils/pty-shell/index.ts | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 792de6f7..876d3fc7 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -322,7 +322,7 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => stdout.includes('READY') && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], - 8_000, + 8000, { env: { DEBUG: '1', @@ -353,7 +353,7 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => /\d+\r\n/.test(stdout) && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], - 8_000, + 8000, { env: { DEBUG: '1', diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 29b6308a..4d654b77 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -27,7 +27,7 @@ const throwTimeout = ( ) => ( setTimeout(timeout, true, abortController).then( () => { - throw new Error('Timeout'); + throw new Error(`Timeout: ${timeout}ms`); }, () => {}, ) @@ -57,6 +57,11 @@ export const ptyShell = async ( if (currentStdin) { const stdin = currentStdin(outString); if (stdin) { + console.log({ + outString, + sending: stdin, + stdins, + }); childProcess.send(stdin); currentStdin = getStdin(stdins); } @@ -84,7 +89,7 @@ export const ptyShell = async ( try { await Promise.race(promises); } catch (error) { - if (error instanceof Error && error.message === 'Timeout') { + if (error instanceof Error && error.message.startsWith('Timeout')) { childProcess.kill(); const outString = stripAnsi(buffer.toString()); From d67e784c551d00bf5898fff33e1ffb728604b3c3 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 12:16:27 +0900 Subject: [PATCH 07/17] debug flaky test --- tests/specs/cli.ts | 2 ++ tests/utils/pty-shell/index.ts | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 876d3fc7..108b8c27 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -324,6 +324,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ], 8000, { + name: 'catchable', env: { DEBUG: '1', }, @@ -355,6 +356,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ], 8000, { + name: 'infinite-loop', env: { DEBUG: '1', }, diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 4d654b77..c26f3374 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -36,7 +36,7 @@ const throwTimeout = ( export const ptyShell = async ( stdins: StdInArray, timeout?: number, - options?: NodeOptions<'utf8'>, + options?: NodeOptions<'utf8'> & { name?: string }, ) => { const childProcess = execaNode( fileURLToPath(new URL('node-pty.mjs', import.meta.url)), @@ -58,6 +58,7 @@ export const ptyShell = async ( const stdin = currentStdin(outString); if (stdin) { console.log({ + name: options?.name, outString, sending: stdin, stdins, @@ -94,6 +95,7 @@ export const ptyShell = async ( const outString = stripAnsi(buffer.toString()); console.log('Incomplete output', { + name: options?.name, outString, stdins, }); From df6113219e2505f9a26ac7c7347bb61c2c1f579d Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 12:22:56 +0900 Subject: [PATCH 08/17] debug flaky test --- src/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli.ts b/src/cli.ts index 0e81d94e..ea420757 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -46,7 +46,7 @@ const relaySignals = ( const waitForSignalFromChild = () => { const p = new Promise((resolve) => { // Aribrary timeout based on flaky tests - setTimeout(() => resolve(undefined), 30); + setTimeout(() => resolve(undefined), 50); waitForSignal = resolve; }); From 5637180f6b3768d604a1e4c92dc28cf627906347 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 12:31:26 +0900 Subject: [PATCH 09/17] debug flaky test --- tests/utils/pty-shell/index.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index c26f3374..41e05a7e 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -56,13 +56,15 @@ export const ptyShell = async ( if (currentStdin) { const stdin = currentStdin(outString); + + console.log({ + name: options?.name, + outString, + sending: stdin, + stdins, + }); + if (stdin) { - console.log({ - name: options?.name, - outString, - sending: stdin, - stdins, - }); childProcess.send(stdin); currentStdin = getStdin(stdins); } From a9b470beffa56a5647a52f38bf8e9bd84f8952da Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:21:59 +0900 Subject: [PATCH 10/17] debug flaky test --- package.json | 2 ++ pnpm-lock.yaml | 17 ++++++++++ tests/specs/cli.ts | 6 ++-- tests/utils/pty-shell/index.ts | 59 +++++++++++++++------------------- 4 files changed, 48 insertions(+), 36 deletions(-) diff --git a/package.json b/package.json index 868f6419..34b3d4bc 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@types/node": "^20.9.4", "@types/react": "^18.2.38", "@types/semver": "^7.5.6", + "@types/split2": "^4.2.3", "cachedir": "^2.4.0", "chokidar": "^3.5.3", "clean-pkg-json": "^1.2.0", @@ -85,6 +86,7 @@ "pkgroll": "^2.0.1", "semver": "^7.5.4", "simple-git-hooks": "^2.9.0", + "split2": "^4.2.0", "strip-ansi": "^7.1.0", "type-fest": "^4.8.2", "type-flag": "^3.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64a54553..4f9c7e1a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -36,6 +36,9 @@ devDependencies: '@types/semver': specifier: ^7.5.6 version: 7.5.6 + '@types/split2': + specifier: ^4.2.3 + version: 4.2.3 cachedir: specifier: ^2.4.0 version: 2.4.0 @@ -99,6 +102,9 @@ devDependencies: simple-git-hooks: specifier: ^2.9.0 version: 2.9.0 + split2: + specifier: ^4.2.0 + version: 4.2.0 strip-ansi: specifier: ^7.1.0 version: 7.1.0 @@ -1007,6 +1013,12 @@ packages: resolution: {integrity: sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==} dev: true + /@types/split2@4.2.3: + resolution: {integrity: sha512-59OXIlfUsi2k++H6CHgUQKEb2HKRokUA39HY1i1dS8/AIcqVjtAAFdf8u+HxTWK/4FUHMJQlKSZ4I6irCBJ1Zw==} + dependencies: + '@types/node': 20.9.4 + dev: true + /@types/stack-utils@2.0.3: resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} dev: true @@ -4261,6 +4273,11 @@ packages: resolution: {integrity: sha512-sh8PWc/ftMqAAdFiBu6Fy6JUOYjqDJBJvIhpfDMyHrr0Rbp5liZqd4TjtQ/RgfLjKFZb+LMx5hpml5qOWy0qvg==} dev: true + /split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + dev: true + /stack-utils@2.0.6: resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} engines: {node: '>=10'} diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 108b8c27..f79d99a5 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -324,7 +324,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ], 8000, { - name: 'catchable', + debug: 'catchable', env: { DEBUG: '1', }, @@ -351,12 +351,12 @@ export default testSuite(({ describe }, node: NodeApis) => { [ // Windows doesn't support shebangs `${node.path} ${tsxPath} ${path.join(fixture.path, 'infinite-loop.js')}\r`, - stdout => /\d+\r\n/.test(stdout) && CtrlC, + stdout => /^\d+$/.test(stdout) && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], 8000, { - name: 'infinite-loop', + debug: 'infinite-loop', env: { DEBUG: '1', }, diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 41e05a7e..acbd6e4e 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -2,6 +2,7 @@ import { setTimeout } from 'timers/promises'; import { fileURLToPath } from 'url'; import { execaNode, type NodeOptions } from 'execa'; import stripAnsi from 'strip-ansi'; +import split from 'split2'; export const isWindows = process.platform === 'win32'; const shell = isWindows ? 'powershell.exe' : 'bash'; @@ -10,17 +11,6 @@ const commandCaret = `${isWindows ? '>' : '$'} `; type ConditionalStdin = (outChunk: string) => string | false; type StdInArray = (string | ConditionalStdin)[]; -const getStdin = ( - stdins: StdInArray, -): ConditionalStdin | undefined => { - const stdin = stdins.shift(); - return ( - typeof stdin === 'string' - ? outChunk => outChunk.includes(commandCaret) && stdin - : stdin - ); -}; - const throwTimeout = ( timeout: number, abortController: AbortController, @@ -36,7 +26,7 @@ const throwTimeout = ( export const ptyShell = async ( stdins: StdInArray, timeout?: number, - options?: NodeOptions<'utf8'> & { name?: string }, + options?: NodeOptions<'utf8'> & { debug?: string }, ) => { const childProcess = execaNode( fileURLToPath(new URL('node-pty.mjs', import.meta.url)), @@ -47,29 +37,30 @@ export const ptyShell = async ( }, ); - let currentStdin = getStdin(stdins); + let currentStdin = stdins.shift(); let buffer = Buffer.alloc(0); childProcess.stdout!.on('data', (data) => { buffer = Buffer.concat([buffer, data]); - const outString = stripAnsi(data.toString()); - - if (currentStdin) { - const stdin = currentStdin(outString); + if (buffer.toString().endsWith(commandCaret)) { + if (!currentStdin) { + childProcess.kill(); + } else if (typeof currentStdin === 'string') { + childProcess.send(currentStdin); + currentStdin = stdins.shift(); + } + } + }); - console.log({ - name: options?.name, - outString, - sending: stdin, - stdins, - }); + childProcess.stdout!.pipe(split()).on('data', (line) => { + line = stripAnsi(line); - if (stdin) { - childProcess.send(stdin); - currentStdin = getStdin(stdins); + if (typeof currentStdin === 'function') { + const send = currentStdin(line); + if (send) { + childProcess.send(send); + currentStdin = stdins.shift(); } - } else if (outString.includes(commandCaret)) { - childProcess.kill(); } }); @@ -96,11 +87,13 @@ export const ptyShell = async ( childProcess.kill(); const outString = stripAnsi(buffer.toString()); - console.log('Incomplete output', { - name: options?.name, - outString, - stdins, - }); + if (options?.debug) { + console.log('Incomplete output', { + name: options.debug, + outString, + stdins, + }); + } } throw error; From 0f5720f57b2103527352ad24e0fd8957caabcdee Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:25:33 +0900 Subject: [PATCH 11/17] debug flaky test --- tests/specs/cli.ts | 2 +- tests/utils/pty-shell/index.ts | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index f79d99a5..64eb0d53 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -324,7 +324,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ], 8000, { - debug: 'catchable', + // debug: 'catchable', env: { DEBUG: '1', }, diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index acbd6e4e..2f20214a 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -55,6 +55,10 @@ export const ptyShell = async ( childProcess.stdout!.pipe(split()).on('data', (line) => { line = stripAnsi(line); + if (options?.debug) { + console.log({ line }); + } + if (typeof currentStdin === 'function') { const send = currentStdin(line); if (send) { From 5e52536bdc2ed161d6fdba34e8619be5fed93db6 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:28:54 +0900 Subject: [PATCH 12/17] debug flaky test --- tests/specs/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 64eb0d53..6037445c 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -351,7 +351,7 @@ export default testSuite(({ describe }, node: NodeApis) => { [ // Windows doesn't support shebangs `${node.path} ${tsxPath} ${path.join(fixture.path, 'infinite-loop.js')}\r`, - stdout => /^\d+$/.test(stdout) && CtrlC, + stdout => /^\r?\d+$/.test(stdout) && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], 8000, From d8d701f8c7b46902f903fb9519a6dd1b2fc7bce1 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:34:52 +0900 Subject: [PATCH 13/17] debug flaky test --- tests/specs/cli.ts | 2 ++ tests/utils/pty-shell/index.ts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 6037445c..40d5e9b2 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -362,6 +362,8 @@ export default testSuite(({ describe }, node: NodeApis) => { }, }, ); + + console.log({ output }); expect(output).toMatch(/EXIT_CODE:\s+130/); }, 10_000); }); diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 2f20214a..86c76738 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -46,6 +46,7 @@ export const ptyShell = async ( if (!currentStdin) { childProcess.kill(); } else if (typeof currentStdin === 'string') { + console.log({ send: currentStdin }); childProcess.send(currentStdin); currentStdin = stdins.shift(); } @@ -62,6 +63,7 @@ export const ptyShell = async ( if (typeof currentStdin === 'function') { const send = currentStdin(line); if (send) { + console.log({ send }); childProcess.send(send); currentStdin = stdins.shift(); } From 7db7f0df2c4999b43e9d71a526b7a073f05d41bb Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:39:46 +0900 Subject: [PATCH 14/17] debug flaky test --- tests/specs/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 40d5e9b2..6156625a 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -364,7 +364,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ); console.log({ output }); - expect(output).toMatch(/EXIT_CODE:\s+130/); + expect(output).toMatch(/EXIT_CODE:[\s\r\n]+130/); }, 10_000); }); }); From f1fec377a08993e23b44bf8f746c2c1be36b31c9 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 13:44:14 +0900 Subject: [PATCH 15/17] debug flaky test --- tests/specs/cli.ts | 17 ++++++++--------- tests/utils/pty-shell/index.ts | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 6156625a..27c7dc3d 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -324,7 +324,7 @@ export default testSuite(({ describe }, node: NodeApis) => { ], 8000, { - // debug: 'catchable', + debug: 'catchable', env: { DEBUG: '1', }, @@ -355,16 +355,15 @@ export default testSuite(({ describe }, node: NodeApis) => { `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], 8000, - { - debug: 'infinite-loop', - env: { - DEBUG: '1', - }, - }, + // { + // debug: 'infinite-loop', + // env: { + // DEBUG: '1', + // }, + // }, ); - console.log({ output }); - expect(output).toMatch(/EXIT_CODE:[\s\r\n]+130/); + expect(output).toMatch(/EXIT_CODE:\s+130/); }, 10_000); }); }); diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index 86c76738..dda412a7 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -46,7 +46,13 @@ export const ptyShell = async ( if (!currentStdin) { childProcess.kill(); } else if (typeof currentStdin === 'string') { - console.log({ send: currentStdin }); + if (options?.debug) { + console.log({ + name: options.debug, + send: currentStdin, + }); + } + childProcess.send(currentStdin); currentStdin = stdins.shift(); } @@ -63,7 +69,12 @@ export const ptyShell = async ( if (typeof currentStdin === 'function') { const send = currentStdin(line); if (send) { - console.log({ send }); + if (options?.debug) { + console.log({ + name: options.debug, + send, + }); + } childProcess.send(send); currentStdin = stdins.shift(); } From 10d62361570d8bdfbc68653990c01d36b5f88c1c Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 14:06:10 +0900 Subject: [PATCH 16/17] debug flaky test --- src/cli.ts | 24 ++++++++++++------------ tests/specs/cli.ts | 24 +++--------------------- 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/cli.ts b/src/cli.ts index ea420757..d58da4fe 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -15,11 +15,11 @@ import { import { isFeatureSupported, testRunnerGlob } from './utils/node-features.js'; import { createIpcServer } from './utils/ipc/server.js'; -const debug = (...messages: any[]) => { - if (process.env.DEBUG) { - console.log(...messages); - } -}; +// const debug = (...messages: any[]) => { +// if (process.env.DEBUG) { +// console.log(...messages); +// } +// }; const relaySignals = ( childProcess: ChildProcess, @@ -46,7 +46,7 @@ const relaySignals = ( const waitForSignalFromChild = () => { const p = new Promise((resolve) => { // Aribrary timeout based on flaky tests - setTimeout(() => resolve(undefined), 50); + setTimeout(() => resolve(undefined), 30); waitForSignal = resolve; }); @@ -74,9 +74,9 @@ const relaySignals = ( */ const signalFromChild = await waitForSignalFromChild(); - debug({ - signalFromChild, - }); + // debug({ + // signalFromChild, + // }); /** * If child didn't receive a signal, it's either because it was @@ -84,9 +84,9 @@ const relaySignals = ( * unresponsive (e.g. infinite loop). Relay signal to child. */ if (signalFromChild !== signal) { - debug('killing child', { - signal, - }); + // debug('killing child', { + // signal, + // }); childProcess.kill(signal); /** diff --git a/tests/specs/cli.ts b/tests/specs/cli.ts index 27c7dc3d..344e86c1 100644 --- a/tests/specs/cli.ts +++ b/tests/specs/cli.ts @@ -314,7 +314,7 @@ export default testSuite(({ describe }, node: NodeApis) => { expect(output).toMatch(/EXIT_CODE:\s+130/); }, 10_000); - test('Catchable', async ({ onTestFail }) => { + test('Catchable', async () => { const output = await ptyShell( [ // Windows doesn't support shebangs @@ -322,21 +322,9 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => stdout.includes('READY') && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], - 8000, - { - debug: 'catchable', - env: { - DEBUG: '1', - }, - }, + 9000, ); - onTestFail(() => { - console.log({ - output, - }); - }); - expectMatchInOrder(output, [ 'READY\r\n', process.platform === 'win32' ? '' : '^C', @@ -354,13 +342,7 @@ export default testSuite(({ describe }, node: NodeApis) => { stdout => /^\r?\d+$/.test(stdout) && CtrlC, `echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`, ], - 8000, - // { - // debug: 'infinite-loop', - // env: { - // DEBUG: '1', - // }, - // }, + 9000, ); expect(output).toMatch(/EXIT_CODE:\s+130/); From 45d502a72bf0bfb269453450700564997987a899 Mon Sep 17 00:00:00 2001 From: Hiroki Osame Date: Tue, 2 Apr 2024 14:14:08 +0900 Subject: [PATCH 17/17] debug flaky test --- tests/utils/pty-shell/index.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/utils/pty-shell/index.ts b/tests/utils/pty-shell/index.ts index dda412a7..8e9e0ecf 100644 --- a/tests/utils/pty-shell/index.ts +++ b/tests/utils/pty-shell/index.ts @@ -94,26 +94,26 @@ export const ptyShell = async ( ]; if (typeof timeout === 'number') { - promises.push(throwTimeout(timeout, abortController)); + promises.push( + throwTimeout(timeout, abortController).catch((error) => { + childProcess.kill(); + + if (options?.debug) { + const outString = stripAnsi(buffer.toString()); + console.log('Incomplete output', { + name: options.debug, + outString, + stdins, + }); + } + + throw error; + }), + ); } try { await Promise.race(promises); - } catch (error) { - if (error instanceof Error && error.message.startsWith('Timeout')) { - childProcess.kill(); - const outString = stripAnsi(buffer.toString()); - - if (options?.debug) { - console.log('Incomplete output', { - name: options.debug, - outString, - stdins, - }); - } - } - - throw error; } finally { abortController.abort(); }