test(DIST-136): Increase resilience for visual tests in firefox

test(DIST-136): Fixes firefox75 bug

test(DIST-136): Fixes firefox version

test(DIST-136): Reverts functional tests

test(DIST-136): Split tests in separate jobs

test(DIST-136): Split tests in separate jobs

Update .travis.yml

Co-Authored-By: Matej Lednicky <matej.lednicky@typeform.com>

test(DIST-136): Run functional tests in parallel

Update .travis.yml

Co-Authored-By: Matej Lednicky <matej.lednicky@typeform.com>

.travis.yml

@@ -11,7 +11,7 @@ python:
 
 addons:
   chrome: stable
-  firefox: latest
+  firefox: "67.0"
 
 branches:
   only:
@@ -47,12 +47,25 @@ jobs:
       if: branch = master OR branch = release
       script:
         - yarn test:unit
-    - name: "Visual Tests (Internal)"
+    - name: "Visual Tests Chrome (Internal)"
       if: fork = false AND (branch = master OR branch = release)
       script:
         - yarn start --no-info & export EMBED_PID=$!
         - yarn test:visual:install
         - yarn test:visual:chrome
+        - kill $EMBED_PID
+    - name: "Visual Tests Mobile (Internal)"
+      if: fork = false AND (branch = master OR branch = release)
+      script:
+        - yarn start --no-info & export EMBED_PID=$!
+        - yarn test:visual:install
+        - yarn test:visual:mobile
+        - kill $EMBED_PID
+    - name: "Visual Tests Firefox (Internal)"
+      if: fork = false AND (branch = master OR branch = release)
+      script:
+        - yarn start --no-info & export EMBED_PID=$!
+        - yarn test:visual:install
         - yarn test:visual:firefox
         - kill $EMBED_PID
     - name: "Functional Tests (Internal)"
@@ -66,8 +79,8 @@ jobs:
       if: fork = true AND (branch = master OR branch = release)
       script:
         - yarn start --no-info & export EMBED_PID=$!
-        - yarn test:functional:chrome
-        - yarn test:functional:firefox
+        - yarn test:functional:chrome --parallel
+        - yarn test:functional:firefox --parallel
         - kill $EMBED_PID
     - name: "Deploy"
       # Deploy to preview preview URL for testing purposes. Only deploy pull requests to 'release' branch.

README.md

@@ -154,9 +154,6 @@ This is the list of all the test commands, if you want to run them one by one:
 - `yarn test:visual` --> Runs visual tests with CodeceptJs, WebDriver, and Applitools.
 - `yarn test:visual:chrome` --> Runs visual tests using Chrome
 - `yarn test:visual:firefox` --> Runs visual tests using Firefox (Gecko drivers)
-- `yarn test:visual:safari` --> Runs tests with Safari. This requires some extra steps for our local environment, check the [official apple documentation](https://developer.apple.com/documentation/webkit/testing_with_webdriver_in_safari). 
-
-***_Note: `yarn test:visual` will not run the Safari tests. You need to run them manually_**
 
 ## Feedback
 

codecept-visual.conf.js

@@ -35,11 +35,11 @@ exports.config = {
     WebDriver: {
       smartWait: 5000,
       url: process.env.URL || 'http://localhost:8080',
-      show: true,
       browser: process.profile || 'chrome',
       host: '127.0.0.1',
       port: 4444,
       path: '/wd/hub',
+      restart: false,
       desiredCapabilities: capabilities[process.profile] || capabilities.chrome,
       windowSize: '1200x825'
     },

demo/demo.js

@@ -1,20 +1,24 @@
+window.messages = []
 window.addEventListener('message', function (event) {
   console.log(`${event.data.type} event has been triggered`)
+  // Changes the document title when the form is ready
+  if (event.data.type === 'form-ready') {
+    window.document.title = event.data.type
+  }
 })
-
 var queryString = window.location.search
 
 var EMBED_DOM_CLASSES = [
   {
     selector: '.typeform-widget',
-    attribute: 'data-url',
+    attribute: 'data-url'
   }, {
     selector: '.typeform-share',
-    attribute: 'href',
+    attribute: 'href'
   }, {
     selector: '.typeform-full',
-    attribute: 'src',
-  },
+    attribute: 'src'
+  }
 ]
 
 window.addEventListener('DOMContentLoaded', function () {

demo/full.html

@@ -27,7 +27,7 @@
       width="100%"
       height="100%"
       frameborder="0"
-      src="https://admin.typeform.com/to/pDgR2Z"
+      src="https://admin.typeform.com/to/mzy0hT?disable-tracking=true"
     ></iframe>
 
     <script type="text/javascript" src="embed.js"></script>

demo/popup.html

@@ -20,7 +20,7 @@
   <body>
     <a
       class="typeform-share"
-      href="//admin.typeform.com/to/mzy0hT"
+      href="//admin.typeform.com/to/mzy0hT?disable-tracking=true"
       data-mode="1"
       data-submit-close-delay="4"
       target="_blank"
@@ -30,7 +30,7 @@
 
     <a
       class="typeform-share"
-      href="//admin.typeform.com/to/mzy0hT"
+      href="//admin.typeform.com/to/mzy0hT?disable-tracking=true"
       data-mode="2"
       data-submit-close-delay="4"
       target="_blank"
@@ -40,7 +40,7 @@
 
     <a
       class="typeform-share"
-      href="//admin.typeform.com/to/mzy0hT"
+      href="//admin.typeform.com/to/mzy0hT?disable-tracking=true"
       data-mode="3"
       data-submit-close-delay="4"
       target="_blank"

demo/widget.html

@@ -26,13 +26,11 @@
   <body>
     <div
       class="typeform-widget"
-      data-url="//admin.typeform.com/to/McAQk1"
+      data-url="//admin.typeform.com/to/mzy0hT?disable-tracking=true"
       style="width:100%; height:500px;"
     ></div>
 
     <script type="text/javascript" src="demo.js"></script>
     <script type="text/javascript" src="embed.js"></script>
-    <script type="text/javascript" src="embed.js"></script>
-    <script type="text/javascript" src="embed.js"></script>
   </body>
 </html>

e2e/codeceptjs-utils.js

@@ -6,7 +6,7 @@ module.exports = {
     opinionScale: '[data-qa*=blocktype-opinionScale]',
     closeButton: '[data-qa="popup-close-button"]'
   },
-  resizeWindow: (I) => {
-    I.resizeWindow(375, 600)
+  resizeWindow: async (I) => {
+    await I.resizeWindow(375, 600)
   }
 }

e2e/custom-steps.js

@@ -26,6 +26,16 @@ module.exports = function () {
     // For Visual Tests requires more waiting time
     waitForVisualElement (cssLocator) {
       this.waitForVisible({ css: cssLocator }, 10)
+    },
+    async waitForIFrameMessage () {
+      this.executeAsyncScript(function (done) {
+        const interval = setInterval(function () {
+          if (window.document.title === 'form-ready') {
+            clearInterval(interval)
+            done()
+          }
+        }, 100)
+      })
     }
   })
 }

e2e/cypress-utils.js

@@ -33,7 +33,7 @@ export const setupIframeTesting = (iframe) => {
   // We can only use it inside Chrome disabling web security --> see config
   // Won't work in popups
   if (Cypress.isBrowser('chrome')) {
-    getIframeBody(iframe).find('[data-qa="fixed-footer-progress"]').should('be.visible')
+    getIframeBody(iframe).find('[data-qa="start-button"]').should('be.visible')
   } else {
     cy.get(iframe).should('be.visible')
   }

e2e/functional/drawer.spec.js

@@ -8,7 +8,6 @@ describe('Drawer Embed Widget', () => {
       link
     })
   })
-
   it('Closes Embed Drawer using Keyboard', () => {
     setupPopupTesting({
       link,

e2e/functional/right-drawer.spec.js

@@ -1,7 +1,7 @@
 const { setupPopupTesting } = require('../cypress-utils')
 
 const link = 3
-describe('Right Drawer Embed Widget', () => {
+describe('Right Drawer Embed Widget handled with close button', () => {
   it('Closes Embed Drawer clicking on the close button', () => {
     setupPopupTesting({
       link

e2e/visual-helper.js

@@ -1,20 +1,28 @@
 const { Eyes, Target, BatchInfo } = require('@applitools/eyes-webdriverio')
-const { helper: Helper } = require('codeceptjs')
+const { helper: Helper, event } = require('codeceptjs')
+// const { ConsoleLogHandler } = require('@applitools/eyes-sdk-core')
+
+const eyes = new Eyes()
 
 class ApplitoolsHelper extends Helper {
   constructor (config) {
     super(config)
-    this.eyes = new Eyes()
-    this.eyes.setApiKey(process.env.EYES_API_KEY)
+    eyes.setApiKey(process.env.EYES_API_KEY)
     this.appName = 'Embed Widget'
     const batchInfo = new BatchInfo(this.appName)
-    this.eyes.setBatch(batchInfo)
+    eyes.setBatch(batchInfo)
   }
 
   async _before (test) {
     const windowSize = this.setWindowSize(this.helpers['WebDriver'].config)
     const browser = this.helpers['WebDriver'].browser
-    await this.eyes.open(browser, this.appName, test.title, windowSize)
+    // TODO Is this useful or makes tests more noisy?
+    // Print logs to console
+    // but makes the build noisy with a lot of logging events might makes it hard to read in Travis
+
+    // eyes.setLogHandler(new ConsoleLogHandler(true))
+
+    await eyes.open(browser, this.appName, test.title, windowSize)
   }
 
   setWindowSize (webDriverConfig) {
@@ -33,8 +41,20 @@ class ApplitoolsHelper extends Helper {
   }
 
   async eyesCheck (pageName) {
-    await this.eyes.check(pageName, Target.window())
-    await this.eyes.close()
+    await eyes.check(pageName, Target.window())
+    await eyes.close()
+    await eyes.abortIfNotClosed()
+  }
+
+  // Once visual tests have finished it takes Applitools more than
+  // 3 minutes to sent the exit code. So below the exit is forced
+  // as soon as Codecept tests results are printed.
+  // Forcing exit code to 0 because Applitools github integration will make
+  // the github status check fail if visuals fail
+  _afterSuite () {
+    event.dispatcher.on(event.all.result, () => {
+      process.exit(0)
+    })
   }
 }
 

e2e/visual/embed.js

@@ -1,74 +1,85 @@
 const { resizeWindow } = require('../codeceptjs-utils')
 
-const clickOnLink = (I, n = 1) => {
+const clickOnLink = async (I, n = 1) => {
   const cssLinkSelector = `[data-mode="${n}"]`
-  I.click({ css: cssLinkSelector })
+  await I.click({ css: cssLinkSelector })
 }
 
+const waitForIFrame = async (I) => {
+  await I.waitForVisible({ css: 'iframe' }, WAITING_TIME)
+  await I.waitForIFrameMessage()
+}
+const WAITING_TIME = 5
+
 Feature('Embed Widget')
 
-Scenario('Basic Widget', async (I) => {
-  I.amOnPage('widget.html')
-  I.waitForVisible({ css: 'iframe' }, 5)
+Scenario('Basic Widget @desktop', async (I) => {
+  await I.amOnPage('widget.html')
+  await waitForIFrame(I)
   await I.eyesCheck('Embed Basic Widget')
 })
 
-Scenario('Full Page Widget', async (I) => {
-  I.amOnPage('full.html')
-  I.waitForVisible({ css: 'iframe' }, 5)
+Scenario('Full Page Widget @desktop', async (I) => {
+  await I.amOnPage('full.html')
+  await waitForIFrame(I)
   await I.eyesCheck('Embed Full Page Widget')
 })
 
-Scenario('Basic Widget on Mobile', async (I) => {
-  resizeWindow(I)
-  I.amOnPage('widget.html')
+Scenario('Basic Widget @mobile', async (I) => {
+  await resizeWindow(I)
+  await I.amOnPage('widget.html')
+  await waitForIFrame(I)
   await I.eyesCheck('Embed Basic Widget - Mobile')
 })
 
-Scenario('Full Page Widget on Mobile', async (I) => {
-  resizeWindow(I)
-  I.amOnPage('full.html')
+Scenario('Full Page Widget @mobile', async (I) => {
+  await resizeWindow(I)
+  await I.amOnPage('full.html')
+  await waitForIFrame(I)
   await I.eyesCheck('Embed Full Page Widget - Mobile')
 })
 
-Scenario('Popup Widget', async (I) => {
-  I.amOnPage('popup.html')
-  clickOnLink(I, 1)
-  I.waitForVisible({ css: 'iframe' }, 10)
+Scenario('Popup Widget @desktop', async (I) => {
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 1)
+  await waitForIFrame(I)
   await I.eyesCheck('Popup Widget')
 })
 
-Scenario('Popup Widget on Mobile', async (I) => {
-  resizeWindow(I)
-  I.amOnPage('popup.html')
-  clickOnLink(I, 1)
+Scenario('Popup Widget @mobile', async (I) => {
+  await resizeWindow(I)
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 1)
+  await waitForIFrame(I)
   await I.eyesCheck('Popup Widget')
 })
 
-Scenario('Drawer Widget', async (I) => {
-  I.amOnPage('popup.html')
-  clickOnLink(I, 2)
-  I.waitForVisible({ css: 'iframe' }, 10)
+Scenario('Drawer Widget @desktop', async (I) => {
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 2)
+  await waitForIFrame(I)
   await I.eyesCheck('Drawer Widget')
 })
 
-Scenario('Drawer Widget on Mobile', async (I) => {
-  resizeWindow(I)
-  I.amOnPage('popup.html')
-  clickOnLink(I, 2)
+Scenario('Drawer Widget @mobile', async (I) => {
+  await resizeWindow(I)
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 2)
+  await waitForIFrame(I)
   await I.eyesCheck('Drawer Widget - Mobile')
 })
 
-Scenario('Right Drawer Widget', async (I) => {
-  I.amOnPage('popup.html')
-  clickOnLink(I, 3)
-  I.waitForVisible({ css: 'iframe' }, 10)
+Scenario('Right Drawer Widget @desktop', async (I) => {
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 3)
+  await waitForIFrame(I)
   await I.eyesCheck('Popup Widget')
 })
 
-Scenario('Right Drawer Widget on Mobile', async (I) => {
-  resizeWindow(I)
-  I.amOnPage('popup.html')
-  clickOnLink(I, 3)
+Scenario('Right Drawer Widget @mobile', async (I) => {
+  await resizeWindow(I)
+  await I.amOnPage('popup.html')
+  await clickOnLink(I, 3)
+  await waitForIFrame(I)
   await I.eyesCheck('Right Drawer Widget - Mobile')
 })