chore: Replace Juggler with Firefox Nightly in CI (#5395)

Fixes #5150.
puppeteer · Feb 20, 2020 · 0b1a9ce · 0b1a9ce
1 parent 0b1777e
commit 0b1a9ce
Show file tree

Hide file tree

Showing 33 changed files with 259 additions and 286 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -22,15 +22,6 @@ task:
   test_doclint_script: npm run test-doclint
   test_types_script: npm run test-types
 
-task:
-  matrix:
-    - name: Firefox Juggler (node8 + linux)
-      container:
-        dockerfile: .ci/node8/Dockerfile.linux
-      xvfb_start_background_script: Xvfb :99 -ac -screen 0 1024x768x24
-  install_script: npm install --unsafe-perm && cd experimental/puppeteer-firefox && npm install --unsafe-perm
-  test_script: npm run fjunit
-
 task:
   osx_instance:
     image: high-sierra-base

diff --git a/.travis.yml b/.travis.yml
@@ -15,11 +15,13 @@ before_install:
   - "sysctl kernel.unprivileged_userns_clone=1"
   - "export DISPLAY=:99.0"
   - "sh -e /etc/init.d/xvfb start"
+  - 'if [ "$FIREFOX" = "true" ]; then pyenv shell 3.6; pip3 install --user mozdownload; fi'
+  - 'if [ "$FIREFOX" = "true" ]; then rm -rf $FIREFOX_HOME; mozdownload -t daily -d $FIREFOX_HOME/latest.tar.bz --log-level DEBUG; fi'
+  - 'if [ "$FIREFOX" = "true" ]; then cd $FIREFOX_HOME; tar -xvf latest.tar.bz; cd -; ls $FIREFOX_HOME/firefox/firefox; fi'
 script:
   - 'if [ "$NODE8" = "true" ]; then npm run lint; fi'
   - 'if [ "$NODE8" = "true" ]; then npm run coverage; fi'
-  - 'if [ "$FIREFOX" = "true" ]; then cd experimental/puppeteer-firefox && npm i && cd ../..; fi'
-  - 'if [ "$FIREFOX" = "true" ]; then npm run fjunit; fi'
+  - 'if [ "$FIREFOX" = "true" ]; then BINARY=$FIREFOX_HOME/firefox/firefox npm run funit; fi'
   - 'if [ "$NODE8" = "true" ]; then npm run test-doclint; fi'
   - 'if [ "$NODE8" = "true" ]; then npm run test-types; fi'
   - 'if [ "$NODE8" = "true" ]; then npm run bundle; fi'
@@ -34,8 +36,9 @@ jobs:
     - node_js: "8.16.0"
       env:
         - FIREFOX=true
-        - FLAKINESS_DASHBOARD_NAME="Travis Firefox (node8 + linux)"
+        - FLAKINESS_DASHBOARD_NAME="Travis Firefox Nightly (node8 + linux)"
         - FLAKINESS_DASHBOARD_BUILD_URL="${TRAVIS_JOB_WEB_URL}"
+        - FIREFOX_HOME=$TRAVIS_HOME/firefox-latest
 before_deploy: "npm run apply-next-version"
 deploy:
   provider: npm

diff --git a/lib/Launcher.js b/lib/Launcher.js
@@ -398,6 +398,9 @@ class FirefoxLauncher {
     else
       firefoxArguments.push(...args);
 
+    if (!firefoxArguments.some(argument => argument.startsWith('--remote-debugging-')))
+      firefoxArguments.push('--remote-debugging-port=0');
+
     let temporaryUserDataDir = null;
 
     if (!firefoxArguments.includes('-profile') && !firefoxArguments.includes('--profile')) {
@@ -484,7 +487,6 @@ class FirefoxLauncher {
    */
   defaultArgs(options = {}) {
     const firefoxArguments = [
-      '--remote-debugging-port=0',
       '--no-remote',
       '--foreground',
     ];

diff --git a/package.json b/package.json
@@ -12,7 +12,6 @@
   },
   "scripts": {
     "unit": "node test/test.js",
-    "fjunit": "PUPPETEER_PRODUCT=juggler node test/test.js",
     "funit": "PUPPETEER_PRODUCT=firefox node test/test.js",
     "debug-unit": "node --inspect-brk test/test.js",
     "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js",

diff --git a/test/accessibility.spec.js b/test/accessibility.spec.js
@@ -19,7 +19,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) {
   const {it, fit, xit} = testRunner;
   const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
 
-  describe('Accessibility', function() {
+  describe_fails_ffox('Accessibility', function() {
     it('should work', async function({page}) {
       await page.setContent(`
       <head>
@@ -310,7 +310,7 @@ module.exports.addTests = function({testRunner, expect, FFOX}) {
         expect(snapshot.children[0]).toEqual(golden);
       });
 
-      describe_fails_ffox('root option', function() {
+      describe('root option', function() {
         it('should work a button', async({page}) => {
           await page.setContent(`<button>My Button</button>`);
 

diff --git a/test/browser.spec.js b/test/browser.spec.js
@@ -23,10 +23,7 @@ module.exports.addTests = function({testRunner, expect, headless, puppeteer, CHR
     it('should return whether we are in headless', async({browser}) => {
       const version = await browser.version();
       expect(version.length).toBeGreaterThan(0);
-      if (CHROME)
-        expect(version.startsWith('Headless')).toBe(headless);
-      else
-        expect(version.startsWith('Firefox/')).toBe(true);
+      expect(version.startsWith('Headless')).toBe(headless);
     });
   });
 

diff --git a/test/browsercontext.spec.js b/test/browsercontext.spec.js
@@ -18,11 +18,11 @@ const utils = require('./utils');
 
 module.exports.addTests = function({testRunner, expect, puppeteer}) {
   const {describe, xdescribe, fdescribe} = testRunner;
-  const {it, fit, xit} = testRunner;
+  const {it, fit, xit, it_fails_ffox} = testRunner;
   const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
 
   describe('BrowserContext', function() {
-    it('should have default context', async function({browser, server}) {
+    it_fails_ffox('should have default context', async function({browser, server}) {
       expect(browser.browserContexts().length).toBe(1);
       const defaultContext = browser.browserContexts()[0];
       expect(defaultContext.isIncognito()).toBe(false);
@@ -31,7 +31,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(browser.defaultBrowserContext()).toBe(defaultContext);
       expect(error.message).toContain('cannot be closed');
     });
-    it('should create new incognito context', async function({browser, server}) {
+    it_fails_ffox('should create new incognito context', async function({browser, server}) {
       expect(browser.browserContexts().length).toBe(1);
       const context = await browser.createIncognitoBrowserContext();
       expect(context.isIncognito()).toBe(true);
@@ -40,7 +40,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       await context.close();
       expect(browser.browserContexts().length).toBe(1);
     });
-    it('should close all belonging targets once closing context', async function({browser, server}) {
+    it_fails_ffox('should close all belonging targets once closing context', async function({browser, server}) {
       expect((await browser.pages()).length).toBe(1);
 
       const context = await browser.createIncognitoBrowserContext();
@@ -51,7 +51,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       await context.close();
       expect((await browser.pages()).length).toBe(1);
     });
-    it('window.open should use parent tab context', async function({browser, server}) {
+    it_fails_ffox('window.open should use parent tab context', async function({browser, server}) {
       const context = await browser.createIncognitoBrowserContext();
       const page = await context.newPage();
       await page.goto(server.EMPTY_PAGE);
@@ -62,7 +62,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(popupTarget.browserContext()).toBe(context);
       await context.close();
     });
-    it('should fire target events', async function({browser, server}) {
+    it_fails_ffox('should fire target events', async function({browser, server}) {
       const context = await browser.createIncognitoBrowserContext();
       const events = [];
       context.on('targetcreated', target => events.push('CREATED: ' + target.url()));
@@ -78,7 +78,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       ]);
       await context.close();
     });
-    it('should wait for a target', async function({browser, server}) {
+    it_fails_ffox('should wait for a target', async function({browser, server}) {
       const context = await browser.createIncognitoBrowserContext();
       let resolved = false;
       const targetPromise = context.waitForTarget(target => target.url() === server.EMPTY_PAGE);
@@ -96,7 +96,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(error).toBeInstanceOf(puppeteer.errors.TimeoutError);
       await context.close();
     });
-    it('should isolate localStorage and cookies', async function({browser, server}) {
+    it_fails_ffox('should isolate localStorage and cookies', async function({browser, server}) {
       // Create two incognito contexts.
       const context1 = await browser.createIncognitoBrowserContext();
       const context2 = await browser.createIncognitoBrowserContext();
@@ -140,7 +140,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       ]);
       expect(browser.browserContexts().length).toBe(1);
     });
-    it('should work across sessions', async function({browser, server}) {
+    it_fails_ffox('should work across sessions', async function({browser, server}) {
       expect(browser.browserContexts().length).toBe(1);
       const context = await browser.createIncognitoBrowserContext();
       expect(browser.browserContexts().length).toBe(2);

diff --git a/test/click.spec.js b/test/click.spec.js
@@ -27,7 +27,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       await page.click('button');
       expect(await page.evaluate(() => result)).toBe('Clicked');
     });
-    it('should click svg', async({page, server}) => {
+    it_fails_ffox('should click svg', async({page, server}) => {
       await page.setContent(`
         <svg height="100" width="100">
           <circle onclick="javascript:window.__CLICKED=42" cx="50" cy="50" r="40" stroke="black" stroke-width="3" fill="red" />
@@ -43,7 +43,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(await page.evaluate(() => result)).toBe('Clicked');
     });
     // @see https://github.com/puppeteer/puppeteer/issues/4281
-    it('should click on a span with an inline element inside', async({page, server}) => {
+    it_fails_ffox('should click on a span with an inline element inside', async({page, server}) => {
       await page.setContent(`
         <style>
         span::before {
@@ -104,7 +104,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
         return textarea.value.substring(textarea.selectionStart, textarea.selectionEnd);
       })).toBe(text);
     });
-    it('should click offscreen buttons', async({page, server}) => {
+    it_fails_ffox('should click offscreen buttons', async({page, server}) => {
       await page.goto(server.PREFIX + '/offscreenbuttons.html');
       const messages = [];
       page.on('console', msg => messages.push(msg.text()));
@@ -153,7 +153,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(await page.evaluate(() => result.check)).toBe(false);
     });
 
-    it('should click on checkbox label and toggle', async({page, server}) => {
+    it_fails_ffox('should click on checkbox label and toggle', async({page, server}) => {
       await page.goto(server.PREFIX + '/input/checkbox.html');
       expect(await page.evaluate(() => result.check)).toBe(null);
       await page.click('label[for="agree"]');
@@ -187,7 +187,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       await page.click('#button-80');
       expect(await page.evaluate(() => document.querySelector('#button-80').textContent)).toBe('clicked');
     });
-    it('should double click the button', async({page, server}) => {
+    it_fails_ffox('should double click the button', async({page, server}) => {
       await page.goto(server.PREFIX + '/input/button.html');
       await page.evaluate(() => {
         window.double = false;
@@ -223,12 +223,12 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       expect(await page.evaluate(() => document.querySelector('#button-8').textContent)).toBe('context menu');
     });
     // @see https://github.com/puppeteer/puppeteer/issues/206
-    it('should click links which cause navigation', async({page, server}) => {
+    it_fails_ffox('should click links which cause navigation', async({page, server}) => {
       await page.setContent(`<a href="${server.EMPTY_PAGE}">empty.html</a>`);
       // This await should not hang.
       await page.click('a');
     });
-    it('should click the button inside an iframe', async({page, server}) => {
+    it_fails_ffox('should click the button inside an iframe', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setContent('<div style="width:100px;height:100px">spacer</div>');
       await utils.attachFrame(page, 'button-test', server.PREFIX + '/input/button.html');
@@ -248,7 +248,7 @@ module.exports.addTests = function({testRunner, expect, puppeteer}) {
       await frame.click('button');
       expect(await frame.evaluate(() => window.result)).toBe('Clicked');
     });
-    it('should click the button with deviceScaleFactor set', async({page, server}) => {
+    it_fails_ffox('should click the button with deviceScaleFactor set', async({page, server}) => {
       await page.setViewport({width: 400, height: 400, deviceScaleFactor: 5});
       expect(await page.evaluate(() => window.devicePixelRatio)).toBe(5);
       await page.setContent('<div style="width:100px;height:100px">spacer</div>');

diff --git a/test/cookies.spec.js b/test/cookies.spec.js
@@ -51,7 +51,7 @@ module.exports.addTests = function({testRunner, expect}) {
       expect(cookies.length).toBe(1);
       expect(cookies[0].httpOnly).toBe(true);
     });
-    it_fails_ffox('should properly report "Strict" sameSite cookie', async({page, server}) => {
+    it('should properly report "Strict" sameSite cookie', async({page, server}) => {
       server.setRoute('/empty.html', (req, res) => {
         res.setHeader('Set-Cookie', ';SameSite=Strict');
         res.end();
@@ -61,7 +61,7 @@ module.exports.addTests = function({testRunner, expect}) {
       expect(cookies.length).toBe(1);
       expect(cookies[0].sameSite).toBe('Strict');
     });
-    it_fails_ffox('should properly report "Lax" sameSite cookie', async({page, server}) => {
+    it('should properly report "Lax" sameSite cookie', async({page, server}) => {
       server.setRoute('/empty.html', (req, res) => {
         res.setHeader('Set-Cookie', ';SameSite=Lax');
         res.end();
@@ -104,7 +104,7 @@ module.exports.addTests = function({testRunner, expect}) {
         },
       ]);
     });
-    it('should get cookies from multiple urls', async({page, server}) => {
+    it_fails_ffox('should get cookies from multiple urls', async({page, server}) => {
       await page.setCookie({
         url: 'https://foo.com',
         name: 'doggo',
@@ -145,15 +145,15 @@ module.exports.addTests = function({testRunner, expect}) {
   });
 
   describe('Page.setCookie', function() {
-    it('should work', async({page, server}) => {
+    it_fails_ffox('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'password',
         value: '123456'
       });
       expect(await page.evaluate(() => document.cookie)).toEqual('password=123456');
     });
-    it('should isolate cookies in browser contexts', async({page, server, browser}) => {
+    it_fails_ffox('should isolate cookies in browser contexts', async({page, server, browser}) => {
       const anotherContext = await browser.createIncognitoBrowserContext();
       const anotherPage = await anotherContext.newPage();
 
@@ -173,7 +173,7 @@ module.exports.addTests = function({testRunner, expect}) {
       expect(cookies2[0].value).toBe('page2value');
       await anotherContext.close();
     });
-    it('should set multiple cookies', async({page, server}) => {
+    it_fails_ffox('should set multiple cookies', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'password',
@@ -190,7 +190,7 @@ module.exports.addTests = function({testRunner, expect}) {
         'password=123456',
       ]);
     });
-    it('should have |expires| set to |-1| for session cookies', async({page, server}) => {
+    it_fails_ffox('should have |expires| set to |-1| for session cookies', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'password',
@@ -200,7 +200,7 @@ module.exports.addTests = function({testRunner, expect}) {
       expect(cookies[0].session).toBe(true);
       expect(cookies[0].expires).toBe(-1);
     });
-    it('should set cookie with reasonable defaults', async({page, server}) => {
+    it_fails_ffox('should set cookie with reasonable defaults', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'password',
@@ -219,7 +219,7 @@ module.exports.addTests = function({testRunner, expect}) {
         session: true
       }]);
     });
-    it('should set a cookie with a path', async({page, server}) => {
+    it_fails_ffox('should set a cookie with a path', async({page, server}) => {
       await page.goto(server.PREFIX + '/grid.html');
       await page.setCookie({
         name: 'gridcookie',
@@ -279,7 +279,7 @@ module.exports.addTests = function({testRunner, expect}) {
       }
       expect(error.message).toContain('At least one of the url and domain needs to be specified');
     });
-    it('should default to setting secure cookie for HTTPS websites', async({page, server}) => {
+    it_fails_ffox('should default to setting secure cookie for HTTPS websites', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       const SECURE_URL = 'https://example.com';
       await page.setCookie({
@@ -290,7 +290,7 @@ module.exports.addTests = function({testRunner, expect}) {
       const [cookie] = await page.cookies(SECURE_URL);
       expect(cookie.secure).toBe(true);
     });
-    it('should be able to set unsecure cookie for HTTP website', async({page, server}) => {
+    it_fails_ffox('should be able to set unsecure cookie for HTTP website', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       const HTTP_URL = 'http://example.com';
       await page.setCookie({
@@ -301,7 +301,7 @@ module.exports.addTests = function({testRunner, expect}) {
       const [cookie] = await page.cookies(HTTP_URL);
       expect(cookie.secure).toBe(false);
     });
-    it('should set a cookie on a different domain', async({page, server}) => {
+    it_fails_ffox('should set a cookie on a different domain', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         url: 'https://www.example.com',
@@ -322,7 +322,7 @@ module.exports.addTests = function({testRunner, expect}) {
         session: true
       }]);
     });
-    it('should set cookies from a frame', async({page, server}) => {
+    it_fails_ffox('should set cookies from a frame', async({page, server}) => {
       await page.goto(server.PREFIX + '/grid.html');
       await page.setCookie({name: 'localhost-cookie', value: 'best'});
       await page.evaluate(src => {
@@ -365,7 +365,7 @@ module.exports.addTests = function({testRunner, expect}) {
   });
 
   describe('Page.deleteCookie', function() {
-    it('should work', async({page, server}) => {
+    it_fails_ffox('should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'cookie1',

diff --git a/test/defaultbrowsercontext.spec.js b/test/defaultbrowsercontext.spec.js
@@ -19,7 +19,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p
   const {it, fit, xit, it_fails_ffox} = testRunner;
   const {beforeAll, beforeEach, afterAll, afterEach} = testRunner;
 
-  describe_fails_ffox('DefaultBrowserContext', function() {
+  describe('DefaultBrowserContext', function() {
     beforeEach(async state => {
       state.browser = await puppeteer.launch(defaultBrowserOptions);
       state.page = await state.browser.newPage();
@@ -46,7 +46,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p
         session: true
       }]);
     });
-    it('page.setCookie() should work', async({page, server}) => {
+    it_fails_ffox('page.setCookie() should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'username',
@@ -65,7 +65,7 @@ module.exports.addTests = function({testRunner, expect, defaultBrowserOptions, p
         session: true
       }]);
     });
-    it('page.deleteCookie() should work', async({page, server}) => {
+    it_fails_ffox('page.deleteCookie() should work', async({page, server}) => {
       await page.goto(server.EMPTY_PAGE);
       await page.setCookie({
         name: 'cookie1',