From b9b2fa28186bc79b49a2fea2e0a39fb454b687f5 Mon Sep 17 00:00:00 2001 From: shaylevi Date: Wed, 17 Jul 2024 12:55:40 +0300 Subject: [PATCH 1/3] vp test: adding new test to handle ESM page + small refactoring --- test/e2e/specs/linksConsolErros.spec.ts | 36 ++----------- .../specs/linksConsoleErrorsEsmPage.spec.ts | 52 +++++++++++++++++++ test/e2e/src/helpers/validatePageErrors.ts | 25 +++++++++ .../helpers/waitForPageToLoadWithTimeout.ts | 9 ++++ test/e2e/testData/esmPageLinksData.ts | 39 ++++++++++++++ test/e2e/testData/pageLinksData.ts | 4 +- test/e2e/types/exampleLinkType.ts | 4 ++ 7 files changed, 134 insertions(+), 35 deletions(-) create mode 100644 test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts create mode 100644 test/e2e/src/helpers/validatePageErrors.ts create mode 100644 test/e2e/src/helpers/waitForPageToLoadWithTimeout.ts create mode 100644 test/e2e/testData/esmPageLinksData.ts create mode 100644 test/e2e/types/exampleLinkType.ts diff --git a/test/e2e/specs/linksConsolErros.spec.ts b/test/e2e/specs/linksConsolErros.spec.ts index 2af0ac83..c2594dcc 100644 --- a/test/e2e/specs/linksConsolErros.spec.ts +++ b/test/e2e/specs/linksConsolErros.spec.ts @@ -1,6 +1,8 @@ -import { ConsoleMessage, Page, expect } from '@playwright/test'; +import { ConsoleMessage, expect } from '@playwright/test'; import { vpTest } from '../fixtures/vpTest'; import { LINKS } from '../testData/pageLinksData'; +import { waitForPageToLoadWithTimeout } from '../src/helpers/waitForPageToLoadWithTimeout'; +import { validatePageErrors } from '../src/helpers/validatePageErrors'; /** * Console error test generated by LINKS object array data. @@ -45,35 +47,3 @@ function handleCommonBrowsersErrors(linkName: string, consoleErrors: ConsoleMess expect(consoleErrors, `The following unexpected console errors were found: ${JSON.stringify(consoleErrors)}`).toHaveLength(0); } } - -/** - * Wait until there are no network connections for at least 5000 ms or for given timeout to pass. - * Needed for console logs to appear. and in pages that loading video 'waitForLoadState('networkidle')' entering a long loop. - */ -async function waitForPageToLoadWithTimeout(page: Page, timeout: number): Promise { - return Promise.race([page.waitForLoadState('networkidle'), new Promise((r) => setTimeout(r, timeout))]); -} - -/** - * Validating that the expected error is part of the console errors - */ -function validatePageErrors(consoleErrors: ConsoleMessage[], expectedErrorMessages: string[], ignoreErrorMessages: string[]): void { - /** - * Filters console messages to exclude ignored messages - */ - const relevantMessages = consoleErrors.filter((consoleError) => !ignoreErrorMessages.some((ignoreError) => consoleError.text().includes(ignoreError))); - - /** - * Filters expected error messages that are not found in relevant console messages - */ - const missingExpectedErrors = expectedErrorMessages.filter((expectedErrorMessage) => !relevantMessages.some((relevantError) => relevantError.text().includes(expectedErrorMessage))); - - expect(missingExpectedErrors.length, `The following expected console errors were not found: ${JSON.stringify(missingExpectedErrors)}`).toBe(0); - - /** - * Filters relevant console messages that are not part of the expected error messages - */ - const unexpectedErrors = relevantMessages.filter((relevantError) => !expectedErrorMessages.some((expectedErrorMessage) => relevantError.text().includes(expectedErrorMessage))); - - expect(unexpectedErrors.length, `The following unexpected console errors were found: ${JSON.stringify(unexpectedErrors)}`).toBe(0); -} diff --git a/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts b/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts new file mode 100644 index 00000000..2a410375 --- /dev/null +++ b/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts @@ -0,0 +1,52 @@ +import { ConsoleMessage, expect } from '@playwright/test'; +import { vpTest } from '../fixtures/vpTest'; +import { ESM_LINKS } from '../testData/esmPageLinksData'; +import { waitForPageToLoadWithTimeout } from '../src/helpers/waitForPageToLoadWithTimeout'; +import { validatePageErrors } from '../src/helpers/validatePageErrors'; + +const ESM_URL = 'https://cld-vp-esm-pages.netlify.app/'; +/** + * Console error test generated by LINKS object array data. + */ +for (const link of ESM_LINKS) { + vpTest(`Test console errors on link ${link.name}`, async ({ page, consoleErrors, vpExamples }) => { + vpTest.skip(link.name === 'Profiles', 'Waiting for https://github.com/cloudinary/cloudinary-video-player/pull/683'); + /** + * Navigate to ESM Imports examples page + */ + await page.goto(ESM_URL); + await vpExamples.clickLinkByName(link.name); + await waitForPageToLoadWithTimeout(page, 5000); + expect(page.url()).toContain(link.endpoint); + handleCommonEsmBrowsersErrors(link.name, consoleErrors); + }); +} + +/** + * Testing number of links in page. + */ +vpTest('ESM page Link count test', async ({ page }) => { + await page.goto(ESM_URL); + const expectedNumberOfLinks = 32; + const numberOfLinks = await page.getByRole('link').count(); + expect(numberOfLinks).toBe(expectedNumberOfLinks); +}); +/** + * Helper function to handle common browser errors. + */ +function handleCommonEsmBrowsersErrors(linkName: string, consoleErrors: ConsoleMessage[]) { + switch (linkName) { + case 'Custom Errors': + validatePageErrors( + consoleErrors, + ['(CODE:999 undefined) My custom error message'], + ['No compatible source was found for this media', 'Video cannot be played Public ID snow_horses not found', 'the server responded with a status of 404', 'Cannot read properties of undefined'] + ); + break; + case 'VAST & VPAID Support': + validatePageErrors(consoleErrors, [], ["Blocked script execution in 'about:blank' because the document's frame is sandboxed and the 'allow-scripts' permission is not set"]); + break; + default: + expect(consoleErrors, `The following unexpected console errors were found: ${JSON.stringify(consoleErrors)}`).toHaveLength(0); + } +} diff --git a/test/e2e/src/helpers/validatePageErrors.ts b/test/e2e/src/helpers/validatePageErrors.ts new file mode 100644 index 00000000..4a0d4eaf --- /dev/null +++ b/test/e2e/src/helpers/validatePageErrors.ts @@ -0,0 +1,25 @@ +import { ConsoleMessage, expect } from '@playwright/test'; + +/** + * Validating that the expected error is part of the console errors + */ +export function validatePageErrors(consoleErrors: ConsoleMessage[], expectedErrorMessages: string[], ignoreErrorMessages: string[]): void { + /** + * Filters console messages to exclude ignored messages + */ + const relevantMessages = consoleErrors.filter((consoleError) => !ignoreErrorMessages.some((ignoreError) => consoleError.text().includes(ignoreError))); + + /** + * Filters expected error messages that are not found in relevant console messages + */ + const missingExpectedErrors = expectedErrorMessages.filter((expectedErrorMessage) => !relevantMessages.some((relevantError) => relevantError.text().includes(expectedErrorMessage))); + + expect(missingExpectedErrors.length, `The following expected console errors were not found: ${JSON.stringify(missingExpectedErrors)}`).toBe(0); + + /** + * Filters relevant console messages that are not part of the expected error messages + */ + const unexpectedErrors = relevantMessages.filter((relevantError) => !expectedErrorMessages.some((expectedErrorMessage) => relevantError.text().includes(expectedErrorMessage))); + + expect(unexpectedErrors.length, `The following unexpected console errors were found: ${JSON.stringify(unexpectedErrors)}`).toBe(0); +} diff --git a/test/e2e/src/helpers/waitForPageToLoadWithTimeout.ts b/test/e2e/src/helpers/waitForPageToLoadWithTimeout.ts new file mode 100644 index 00000000..f2df8e22 --- /dev/null +++ b/test/e2e/src/helpers/waitForPageToLoadWithTimeout.ts @@ -0,0 +1,9 @@ +import { Page } from '@playwright/test'; + +/** + * Wait until there are no network connections for at least 5000 ms or for given timeout to pass. + * Needed for console logs to appear. and in pages that loading video 'waitForLoadState('networkidle')' entering a long loop. + */ +export async function waitForPageToLoadWithTimeout(page: Page, timeout: number): Promise { + return Promise.race([page.waitForLoadState('networkidle'), new Promise((r) => setTimeout(r, timeout))]); +} diff --git a/test/e2e/testData/esmPageLinksData.ts b/test/e2e/testData/esmPageLinksData.ts new file mode 100644 index 00000000..6cb6fad6 --- /dev/null +++ b/test/e2e/testData/esmPageLinksData.ts @@ -0,0 +1,39 @@ +import { ExampleLinkType } from '../types/exampleLinkType'; + +/** + * Array of all the examples pages names and links on ESM import page. + */ +export const ESM_LINKS: ExampleLinkType[] = [ + { name: 'Adaptive Streaming', endpoint: 'adaptive-streaming' }, + { name: 'Analytics', endpoint: 'analytics' }, + { name: 'API and Events', endpoint: 'api' }, + { name: 'Audio Player', endpoint: 'audio' }, + { name: 'Autoplay on Scroll', endpoint: 'autoplay-on-scroll' }, + { name: 'Chapters', endpoint: 'chapters' }, + { name: 'Cloudinary Analytics', endpoint: 'cloudinary-analytics' }, + { name: 'Codecs and formats', endpoint: 'codec-formats' }, + { name: 'Colors API', endpoint: 'colors' }, + { name: 'Components', endpoint: 'components' }, + { name: 'Custom Errors', endpoint: 'custom-cld-errors' }, + { name: 'Display Configurations', endpoint: 'ui-config' }, + { name: 'Debug', endpoint: 'debug' }, + { name: 'Floating Player', endpoint: 'floating-player' }, + { name: 'Fluid Layouts', endpoint: 'fluid' }, + { name: 'Force HLS Subtitles', endpoint: 'force-hls-subtitles' }, + { name: 'Highlights Graph', endpoint: 'highlights-graph' }, + { name: 'Interaction Area', endpoint: 'interaction-area' }, + { name: 'Multiple Players', endpoint: 'multiple-players' }, + { name: 'Playlist', endpoint: 'playlist' }, + { name: 'Playlist by Tag', endpoint: 'playlist-by-tag' }, + { name: 'Poster Options', endpoint: 'poster' }, + { name: 'Profiles', endpoint: 'profiles' }, + { name: 'Raw URL', endpoint: 'raw-url' }, + { name: 'Recommendations', endpoint: 'recommendations' }, + { name: 'Seek Thumbnails', endpoint: 'seek-thumbs' }, + { name: 'Shoppable Videos', endpoint: 'shoppable' }, + { name: 'Subtitles & Captions', endpoint: 'subtitles-and-captions' }, + { name: 'Video Transformations', endpoint: 'transformations' }, + { name: 'UI Config', endpoint: 'ui-config' }, + { name: 'VAST & VPAID Support', endpoint: 'vast-vpaid' }, + { name: 'VR/360 Videos', endpoint: '360' }, +]; diff --git a/test/e2e/testData/pageLinksData.ts b/test/e2e/testData/pageLinksData.ts index 277f8c76..8fb5c37e 100644 --- a/test/e2e/testData/pageLinksData.ts +++ b/test/e2e/testData/pageLinksData.ts @@ -1,9 +1,9 @@ -type Link = { name: string; endpoint: string }; +import { ExampleLinkType } from '../types/exampleLinkType'; /** * Array of all the examples pages names and links. */ -export const LINKS: Link[] = [ +export const LINKS: ExampleLinkType[] = [ //{ name: 'Adaptive Streaming', endpoint: 'adaptive-streaming.html' }, { name: 'AI Highlights Graph', endpoint: 'highlights-graph.html' }, { name: 'Analytics', endpoint: 'analytics.html' }, diff --git a/test/e2e/types/exampleLinkType.ts b/test/e2e/types/exampleLinkType.ts new file mode 100644 index 00000000..0b649bbc --- /dev/null +++ b/test/e2e/types/exampleLinkType.ts @@ -0,0 +1,4 @@ +/** + * Example links type + */ +export type ExampleLinkType = { name: string; endpoint: string }; From 34316f127d7bde0c3b55dead13ab40d5c4793a05 Mon Sep 17 00:00:00 2001 From: shaylevi Date: Wed, 17 Jul 2024 13:05:43 +0300 Subject: [PATCH 2/3] vp test: remove flaky adaptive streaming --- test/e2e/testData/esmPageLinksData.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/testData/esmPageLinksData.ts b/test/e2e/testData/esmPageLinksData.ts index 6cb6fad6..cd361ccd 100644 --- a/test/e2e/testData/esmPageLinksData.ts +++ b/test/e2e/testData/esmPageLinksData.ts @@ -4,7 +4,7 @@ import { ExampleLinkType } from '../types/exampleLinkType'; * Array of all the examples pages names and links on ESM import page. */ export const ESM_LINKS: ExampleLinkType[] = [ - { name: 'Adaptive Streaming', endpoint: 'adaptive-streaming' }, + //{ name: 'Adaptive Streaming', endpoint: 'adaptive-streaming' }, { name: 'Analytics', endpoint: 'analytics' }, { name: 'API and Events', endpoint: 'api' }, { name: 'Audio Player', endpoint: 'audio' }, From 238d6236790c42715f02545b0f7f30411c5cc809 Mon Sep 17 00:00:00 2001 From: shaylevi Date: Wed, 17 Jul 2024 14:40:40 +0300 Subject: [PATCH 3/3] vp test: remove skip from ESM profile page and skip adaptive streaming --- test/e2e/specs/linksConsolErros.spec.ts | 1 + test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts | 2 +- test/e2e/testData/esmPageLinksData.ts | 2 +- test/e2e/testData/pageLinksData.ts | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/e2e/specs/linksConsolErros.spec.ts b/test/e2e/specs/linksConsolErros.spec.ts index c2594dcc..6be1cdde 100644 --- a/test/e2e/specs/linksConsolErros.spec.ts +++ b/test/e2e/specs/linksConsolErros.spec.ts @@ -9,6 +9,7 @@ import { validatePageErrors } from '../src/helpers/validatePageErrors'; */ for (const link of LINKS) { vpTest(`Test console errors on link ${link.name}`, async ({ page, consoleErrors, vpExamples }) => { + vpTest.skip(link.name === 'Adaptive Streaming', 'Flaky on CI'); await vpExamples.clickLinkByName(link.name); await waitForPageToLoadWithTimeout(page, 5000); expect(page.url()).toContain(link.endpoint); diff --git a/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts b/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts index 2a410375..d1c7a36c 100644 --- a/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts +++ b/test/e2e/specs/linksConsoleErrorsEsmPage.spec.ts @@ -10,7 +10,7 @@ const ESM_URL = 'https://cld-vp-esm-pages.netlify.app/'; */ for (const link of ESM_LINKS) { vpTest(`Test console errors on link ${link.name}`, async ({ page, consoleErrors, vpExamples }) => { - vpTest.skip(link.name === 'Profiles', 'Waiting for https://github.com/cloudinary/cloudinary-video-player/pull/683'); + vpTest.skip(link.name === 'Adaptive Streaming', 'Flaky on CI'); /** * Navigate to ESM Imports examples page */ diff --git a/test/e2e/testData/esmPageLinksData.ts b/test/e2e/testData/esmPageLinksData.ts index cd361ccd..6cb6fad6 100644 --- a/test/e2e/testData/esmPageLinksData.ts +++ b/test/e2e/testData/esmPageLinksData.ts @@ -4,7 +4,7 @@ import { ExampleLinkType } from '../types/exampleLinkType'; * Array of all the examples pages names and links on ESM import page. */ export const ESM_LINKS: ExampleLinkType[] = [ - //{ name: 'Adaptive Streaming', endpoint: 'adaptive-streaming' }, + { name: 'Adaptive Streaming', endpoint: 'adaptive-streaming' }, { name: 'Analytics', endpoint: 'analytics' }, { name: 'API and Events', endpoint: 'api' }, { name: 'Audio Player', endpoint: 'audio' }, diff --git a/test/e2e/testData/pageLinksData.ts b/test/e2e/testData/pageLinksData.ts index 8fb5c37e..82cf6265 100644 --- a/test/e2e/testData/pageLinksData.ts +++ b/test/e2e/testData/pageLinksData.ts @@ -4,7 +4,7 @@ import { ExampleLinkType } from '../types/exampleLinkType'; * Array of all the examples pages names and links. */ export const LINKS: ExampleLinkType[] = [ - //{ name: 'Adaptive Streaming', endpoint: 'adaptive-streaming.html' }, + { name: 'Adaptive Streaming', endpoint: 'adaptive-streaming.html' }, { name: 'AI Highlights Graph', endpoint: 'highlights-graph.html' }, { name: 'Analytics', endpoint: 'analytics.html' }, { name: 'API and Events', endpoint: 'api.html' },