Merge layout testing to master (#112)

* Layout testing support (#106)

* Run Unit tests for PRs to 'layout-testing'

* Run unit tests for PRs to 'layout-testing'

* Layout testing prototype

bump version

Prefix the package name

Prefix the package name

Fix package name

Fix package name

Publish a user-scoped package as public

Compatibility with TestCafe 1.20.0 + version bump

suppress warnings + allow quarantine

* refactor and add tests

* fix new unit tests

* maybe a fix

* bump version

* try store paths in strictly POSIX

* get rid of excessive regexp

* rename an Uploader method

* Support new scenarios: no baseline screenshot and size mismatch (#108)

* support new screenshot and size mismatch

bump reporter version

fix tests

* decouple noScreenshotUpload and layoutTestingEnabled + bump TypeScript to match TestCafe 2.0

* bump version one more time

* bump version to 1.0.0 (#110)

* get rid of fork-specific stuff

* fix audit

* remove outdated test

* Address Alexander's remarks

* avoid minimatch version override by updating mocha

* bump version

.github/workflows/npm-audit.yml

@@ -3,7 +3,7 @@ name: Audit
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ master, layout-testing ]
+    branches: [ master ]
 
 jobs:
   run:

.github/workflows/unit-tests.yml

@@ -3,7 +3,7 @@ name: Unit tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [ master, layout-testing ]
+    branches: [ master ]
 
 jobs:
   run:

package.json

@@ -1,6 +1,6 @@
 {
   "name": "testcafe-reporter-dashboard",
-  "version": "1.0.0-rc.4",
+  "version": "1.0.0-rc.5",
   "description": "Dashboard TestCafe reporter plugin.",
   "author": {
     "name": "Developer Express Inc.",
@@ -46,12 +46,12 @@
     "eslint-plugin-standard": "^4.0.1",
     "gulp": "^4.0.2",
     "gulp-typescript": "^5.0.1",
-    "mocha": "^7.0.1",
+    "mocha": "^10.1.0",
     "mock-require": "^3.0.3",
     "publish-please": "^5.5.2",
     "testcafe": "^1.20.0",
     "ts-node": "^8.7.0",
-    "typescript": "^3.8.3"
+    "typescript": "4.7.4"
   },
   "dependencies": {
     "fp-ts": "^2.12.1",

src/consts.ts

@@ -1,4 +1,3 @@
-export const MAX_BUILD_ID_LENGTH            = 100;
 export const CONCURRENT_ERROR_CODE          = 409;
 export const SERVICE_UNAVAILABLE_ERROR_CODE = 503;
 

src/env/index.ts

@@ -9,6 +9,11 @@ export const TESTCAFE_DASHBOARD_URL = env.TESTCAFE_DASHBOARD_URL || 'https://das
 export const NO_SCREENSHOT_UPLOAD = parseBooleanVariable(env.TESTCAFE_DASHBOARD_NO_SCREENSHOT_UPLOAD);
 export const NO_VIDEO_UPLOAD      = parseBooleanVariable(env.TESTCAFE_DASHBOARD_NO_VIDEO_UPLOAD);
 
+export const LAYOUT_TESTING_ENABLED = parseBooleanVariable(env.TESTCAFE_DASHBOARD_LAYOUT_TESTING_ENABLED);
+export const LT_SCREENSHOTS_DIR     = env.TESTCAFE_DASHBOARD_LT_SCREENSHOTS_DIR;
+export const LT_DESTINATION_DIR     = env.TESTCAFE_DASHBOARD_LT_DESTINATION_DIR;
+export const LT_COMPARER_BASE_DIR   = env.TESTCAFE_DASHBOARD_LT_COMPARER_BASE_DIR;
+
 export const TESTCAFE_DASHBOARD_AUTHENTICATION_TOKEN = env.TESTCAFE_DASHBOARD_TOKEN;
 export const TESTCAFE_DASHBOARD_BUILD_ID: string | undefined = env.TESTCAFE_DASHBOARD_BUILD_ID;
 

src/get-reporter-settings.ts

@@ -6,12 +6,17 @@ import {
     TESTCAFE_DASHBOARD_BUILD_ID,
     TESTCAFE_DASHBOARD_URL,
     CI_INFO,
+    LAYOUT_TESTING_ENABLED,
     RESPONSE_TIMEOUT,
-    REQUEST_RETRY_COUNT
+    REQUEST_RETRY_COUNT,
+    LT_SCREENSHOTS_DIR,
+    LT_DESTINATION_DIR,
+    LT_COMPARER_BASE_DIR
 } from './env';
-import { ReporterPluginOptions } from './types';
+import { ReporterPluginOptions, TaskProperties } from './types';
+import { LayoutTestingSettings } from './types/internal';
 
-export default function getReporterSettings (options: ReporterPluginOptions) {
+export function getReporterSettings (options: ReporterPluginOptions) {
     const {
         url,
         token,
@@ -35,3 +40,14 @@ export default function getReporterSettings (options: ReporterPluginOptions) {
         ciInfo:              CI_INFO
     };
 }
+
+export function getLayoutTestingSettings (taskProperties: TaskProperties): LayoutTestingSettings {
+    const comparerProperties = taskProperties.configuration['screenshots-comparer'] ?? {};
+
+    return {
+        layoutTestingEnabled: LAYOUT_TESTING_ENABLED,
+        outputRelativeDir:    LT_SCREENSHOTS_DIR || comparerProperties['screenshotsRelativePath'] || '/screenshots',
+        resultsRelativeDir:   LT_DESTINATION_DIR || comparerProperties['destinationRelativePath'] || '/artifacts/compared-screenshots',
+        comparerBaseDir:      LT_COMPARER_BASE_DIR || comparerProperties['path'] || './testing'
+    };
+}

src/index.ts

@@ -3,9 +3,9 @@ import fetch from 'isomorphic-fetch';
 import { promisify } from 'util';
 
 import { ReporterPluginObject } from './types/internal';
-import reporterObjectFactory from './reporter-object-factory';
+import { reporterObjectFactory } from './reporter-object-factory';
 import logger from './logger';
-import getReporterSettings from './get-reporter-settings';
+import { getReporterSettings } from './get-reporter-settings';
 import { ReporterPluginOptions } from './types';
 import path from 'path';
 
@@ -31,5 +31,5 @@ module.exports = function pluginFactory (options: ReporterPluginOptions = {}): R
     const settings  = getReporterSettings(options);
     const tcVersion = getTestCafeVersion();
 
-    return reporterObjectFactory(promisify(fs.readFile), fetch, settings, logger, tcVersion);
+    return reporterObjectFactory(promisify(fs.readFile), promisify(fs.exists), fetch, settings, logger, tcVersion);
 };

src/reporter-object-factory.ts

@@ -8,7 +8,9 @@ import {
     ReadFileMethod,
     DashboardSettings,
     Logger,
-    ReporterPluginObject
+    ReporterPluginObject,
+    LayoutTestingSettings,
+    FileExistsMethod
 } from './types/internal/';
 import {
     ActionInfo,
@@ -17,7 +19,9 @@ import {
     DashboardInfo,
     DashboardValidationResult,
     ReportedTestStructureItem,
+    ScreenshotMapItem,
     ShortId,
+    TaskProperties,
     TestDoneArgs,
     TestError,
     Warning,
@@ -31,16 +35,13 @@ import assignReporterMethods from './assign-reporter-methods';
 import { validateSettings } from './validate-settings';
 import createReportUrl from './create-report-url';
 import BLANK_REPORTER from './blank-reporter';
+import path from 'path';
+import { getLayoutTestingSettings } from './get-reporter-settings';
+import { addArrayValueByKey, getScreenshotComparerArtifactsPath, getShouldUploadLayoutTestingData, makePathRelativeStartingWith } from './utils';
 
-function addArrayValueByKey (collection: Record<string, any[]>, key: string, value: any) {
-    if (!collection[key])
-        collection[key] = [value];
-    else if (!collection[key].includes(value))
-        collection[key].push(value);
-};
-
-export default function reporterObjectFactory (
+export function reporterObjectFactory (
     readFile: ReadFileMethod,
+    fileExists: FileExistsMethod,
     fetch: FetchMethod,
     settings: DashboardSettings,
     logger: Logger,
@@ -65,7 +66,7 @@ export default function reporterObjectFactory (
     const id: string = runId || uuid();
 
     const transport      = new Transport(fetch, dashboardUrl, authenticationToken, isLogEnabled, logger, responseTimeout, requestRetryCount);
-    const uploader       = new Uploader(readFile, transport, logger);
+    const uploader       = new Uploader(readFile, fileExists, transport, logger);
     const reportCommands = reportCommandsFactory(id, transport);
 
     const testRunToWarningsMap: Record<string, Warning[]>         = {};
@@ -75,7 +76,9 @@ export default function reporterObjectFactory (
     const testRunIdToTestIdMap: Record<string, string>            = {};
     const errorsToTestIdMap: Record<string, string[]>             = {};
 
+
     let rejectReport = false;
+    let layoutTestingSettings: LayoutTestingSettings;
 
     function processDashboardWarnings (dashboardInfo: DashboardInfo) {
         if (dashboardInfo.type === DashboardValidationResult.warning) {
@@ -139,9 +142,11 @@ export default function reporterObjectFactory (
     }
 
     assignReporterMethods(reporterPluginObject, {
-        async reportTaskStart (startTime, userAgents, testCount, taskStructure: ReportedTestStructureItem[]): Promise<void> {
+        async reportTaskStart (startTime, userAgents, testCount, taskStructure: ReportedTestStructureItem[], taskProperties: TaskProperties): Promise<void> {
             if (rejectReport) return;
 
+            layoutTestingSettings = getLayoutTestingSettings(taskProperties);
+
             logger.log(createReportUrlMessage(buildId || id, authenticationToken, dashboardUrl));
 
             await reportCommands.sendTaskStartCommand({
@@ -227,21 +232,71 @@ export default function reporterObjectFactory (
         async reportTestDone (name, testRunInfo): Promise<void> {
             if (rejectReport) return;
 
-            const { screenshots, videos, errs, durationMs, testId, browsers, skipped, unstable } = testRunInfo;
+            const { screenshots, videos, errs, durationMs, testId, browsers, skipped, unstable, fixture } = testRunInfo;
+            const { layoutTestingEnabled, outputRelativeDir, resultsRelativeDir, comparerBaseDir }        = layoutTestingSettings;
+
+            const testRunToScreenshotsMap: Record<string, ScreenshotMapItem[]> = {};
+
+            const testRunToVideosMap: Record<string, string[]>  = {};
+            const testRunToErrorsMap: Record<string, TestError> = {};
 
-            const testRunToScreenshotsMap: Record<string, string[]> = {};
-            const testRunToVideosMap: Record<string, string[]>      = {};
-            const testRunToErrorsMap: Record<string, TestError>     = {};
+            const testBrowserRuns               = browserToRunsMap[testId];
+            const shouldUploadLayoutTestingData = getShouldUploadLayoutTestingData(layoutTestingEnabled, browsers);
 
-            if (!noScreenshotUpload) {
+            if (!noScreenshotUpload || shouldUploadLayoutTestingData) {
                 for (const screenshotInfo of screenshots) {
-                    const { screenshotPath, screenshotData, testRunId } = screenshotInfo;
+                    const { screenshotPath, screenshotData, testRunId, actionId } = screenshotInfo;
 
-                    const uploadId = await uploader.uploadFile(screenshotPath, screenshotData);
+                    const comparisonArtifactsPath = shouldUploadLayoutTestingData ? await getScreenshotComparerArtifactsPath(fileExists, screenshotPath, outputRelativeDir, resultsRelativeDir) : void 0;
+                    const comparisonFailed        = !!comparisonArtifactsPath;
 
-                    if (!uploadId) continue;
+                    if (noScreenshotUpload && !comparisonFailed)
+                        continue;
+
+                    const currentUploadId = await uploader.uploadFile(screenshotPath, screenshotData);
+
+                    if (!currentUploadId)
+                        continue;
+
+                    if (actionId) {
+                        const actions          = testRunToActionsMap[testRunId];
+                        const screenshotAction = actions?.find(action => action.command.actionId === actionId);
+
+                        if (screenshotAction)
+                            screenshotAction.screenshotPath = screenshotPath;
+                    }
 
-                    addArrayValueByKey(testRunToScreenshotsMap, testRunId, uploadId);
+                    const screenshotMapItem: ScreenshotMapItem = {
+                        path: screenshotPath,
+                        ids:  {
+                            current: currentUploadId
+                        }
+                    };
+
+                    if (comparisonFailed) {
+                        const testPath                       = fixture.path;
+                        const baselineScreenshotPath         = path.join(path.dirname(testPath), 'etalons', path.basename(screenshotPath));
+                        const baselineScreenshotRelativePath = makePathRelativeStartingWith(baselineScreenshotPath, path.normalize(comparerBaseDir));
+
+                        if (baselineScreenshotRelativePath) {
+                            const posixPath = baselineScreenshotRelativePath.split(path.sep).join(path.posix.sep);
+
+                            screenshotMapItem.baselineSourcePath = posixPath;
+                            screenshotMapItem.maskSourcePath     = posixPath.replace(/.png$/, '_mask.png');
+                        }
+
+                        screenshotMapItem.ids = {
+                            ...screenshotMapItem.ids,
+
+                            baseline: await uploader.uploadLayoutTestingArtifact(comparisonArtifactsPath, '_etalon'),
+                            diff:     await uploader.uploadLayoutTestingArtifact(comparisonArtifactsPath, '_diff'),
+                            mask:     await uploader.uploadLayoutTestingArtifact(comparisonArtifactsPath, '_mask')
+                        };
+                    }
+
+                    screenshotMapItem.comparisonFailed = comparisonFailed;
+
+                    addArrayValueByKey(testRunToScreenshotsMap, testRunId, screenshotMapItem);
                 }
             }
 
@@ -270,8 +325,6 @@ export default function reporterObjectFactory (
 
             delete errorsToTestIdMap[testId];
 
-            const testBrowserRuns = browserToRunsMap[testId];
-
             const browserRuns = browsers.reduce((runs, browser) => {
                 const { alias, testRunId } = browser;
                 const runIds               = testBrowserRuns && testBrowserRuns[alias] || null;
@@ -286,14 +339,17 @@ export default function reporterObjectFactory (
                 }
 
                 const getBrowserRunInfo = (attemptRunId: string, attempt?: number): BrowserRunInfo => {
+                    const actions       = testRunToActionsMap[attemptRunId];
+                    const screenshotMap = testRunToScreenshotsMap[attemptRunId];
+
                     const result = {
                         browser,
-                        screenshotUploadIds: testRunToScreenshotsMap[attemptRunId],
+                        actions,
                         videoUploadIds,
-                        actions:             testRunToActionsMap[attemptRunId],
-                        thirdPartyError:     testRunToErrorsMap[attemptRunId],
-                        quarantineAttempt:   attempt,
-                        warnings:            testRunToWarningsMap[attemptRunId],
+                        screenshotMap,
+                        thirdPartyError:   testRunToErrorsMap[attemptRunId],
+                        quarantineAttempt: attempt,
+                        warnings:          testRunToWarningsMap[attemptRunId],
                     };
 
                     delete testRunToActionsMap[attemptRunId];

src/texts.ts

@@ -1,4 +1,4 @@
-import { MAX_BUILD_ID_LENGTH } from './consts';
+import { MAX_BUILD_ID_LENGTH } from './types/consts';
 import createReportUrl from './create-report-url';
 
 export const DASHBOARD_LOCATION_NOT_DEFINED  = 'The \'TESTCAFE_DASHBOARD_URL\' environment variable is not defined.';

src/types/common.ts

@@ -1,4 +1,4 @@
-import { MAX_BUILD_ID_LENGTH } from '../consts';
+import { MAX_BUILD_ID_LENGTH } from './consts';
 import * as t from 'io-ts';
 
 export interface MaxLengthString<N> {

src/types/consts.ts

@@ -0,0 +1 @@
+export const MAX_BUILD_ID_LENGTH = 100;

src/types/internal/dashboard.ts

@@ -49,6 +49,10 @@ export interface ReadFileMethod {
     (path: string): Promise<Buffer>;
 };
 
+export interface FileExistsMethod {
+    (path: string): Promise<boolean>;
+};
+
 export type DashboardSettings = {
     authenticationToken: string;
     buildId?: string;
@@ -62,6 +66,13 @@ export type DashboardSettings = {
     ciInfo?: CIInfo;
 };
 
+export type LayoutTestingSettings = {
+    outputRelativeDir: string;
+    resultsRelativeDir: string;
+    layoutTestingEnabled: boolean;
+    comparerBaseDir: string;
+};
+
 export type Logger = {
     log (...params): void;
     warn (...params): void;

src/types/internal/testcafe.ts

@@ -1,4 +1,4 @@
-import { BrowserInfo, Meta, ReportedTestStructureItem, TestRunInfo, Error, TestPhase, CommandType, TaskResult, Warning } from '../';
+import { BrowserInfo, Meta, ReportedTestStructureItem, TestRunInfo, Error, TestPhase, CommandType, TaskResult, Warning, TaskProperties } from '../';
 
 export type TestStartInfo = {
     testId: string;
@@ -35,7 +35,7 @@ export type decoratorFn = (str: string) => string;
 
 export type ReporterMethods = {
     init?: () => Promise<void>;
-    reportTaskStart: (startTime: Date, userAgents: string[], testCount: number, taskStructure: ReportedTestStructureItem[]) => Promise<void>;
+    reportTaskStart: (startTime: Date, userAgents: string[], testCount: number, taskStructure: ReportedTestStructureItem[], taskProperties: TaskProperties) => Promise<void>;
     reportFixtureStart: (name: string, path: string, meta: Meta) => Promise<void>;
     reportTestStart?: (name: string, meta: Meta, testStartInfo: TestStartInfo) => Promise<void>;
     reportTestActionStart?: (apiActionName: string, actionInfo: TestCafeActionInfo) => Promise<void>;