test: Late load tests, fix Chromecast test flake (#4115)

This change fixes tests on Chromecast by loading tests later in the process. Test scripts are now dynamically inserted by boot.js, rather than loaded by Karma. The bootstrapping code then awaits the completion of that before starting the Karma frameworks (Jasmine) to run the tests. This also removes the use of goog.provide/goog.require in tests and test utils. We don't need to load test utils or library sources dynamically in each test, and this gives us more explicit control over script loading and ordering. Closes #4094
shaka-project · Apr 11, 2022 · fbbd63d · fbbd63d
1 parent 169943f
commit fbbd63d
Show file tree

Hide file tree

Showing 142 changed files with 134 additions and 1,024 deletions.
diff --git a/demo/config.js b/demo/config.js
@@ -11,7 +11,6 @@ goog.require('goog.asserts');
 goog.require('shakaDemo.BoolInput');
 goog.require('shakaDemo.DatalistInput');
 goog.require('shakaDemo.InputContainer');
-goog.require('shakaDemo.Main');
 goog.require('shakaDemo.MessageIds');
 goog.require('shakaDemo.NumberInput');
 goog.require('shakaDemo.SelectInput');

diff --git a/demo/custom.js b/demo/custom.js
@@ -12,7 +12,6 @@ goog.require('ShakaDemoAssetInfo');
 goog.require('shakaDemo.AssetCard');
 goog.require('shakaDemo.Input');
 goog.require('shakaDemo.InputContainer');
-goog.require('shakaDemo.Main');
 goog.require('shakaDemo.MessageIds');
 goog.require('shakaDemo.TextInput');
 

diff --git a/karma.conf.js b/karma.conf.js
@@ -9,6 +9,7 @@
 
 const Jimp = require('jimp');
 const fs = require('fs');
+const glob = require('glob');
 const path = require('path');
 const rimraf = require('rimraf');
 const {ssim} = require('ssim.js');
@@ -107,39 +108,38 @@ module.exports = (config) => {
       'dist/deps.js',
       'shaka-player.uncompiled.js',
 
+      // the demo's config tab will register with shakaDemoMain, and will be
+      // tested in test/demo/demo_unit.js
+      'demo/config.js',
+
       // cajon module (an AMD variant of requirejs) next
       'node_modules/cajon/cajon.js',
 
-      // bootstrapping for the test suite
-      'test/test/boot.js',
+      // define the test namespace next (shaka.test)
+      'test/test/namespace.js',
 
-      // test utils next
+      // test utilities next, which fill in that namespace
       'test/test/util/*.js',
 
-      // list of test assets next
-      'demo/common/message_ids.js',
-      'demo/common/asset.js',
-      'demo/common/assets.js',
+      // bootstrapping for the test suite last; this will load the actual tests
+      'test/test/boot.js',
 
       // if --test-custom-asset *is not* present, we will add unit tests.
       // if --quick *is not* present, we will add integration tests.
       // if --external *is* present, we will add external asset tests.
 
-      // load relevant demo files
-      {
-        pattern: 'demo/!(main|load|demo_uncompiled|service_worker).js',
-        included: true,
-      },
-
-      // source files - these are only watched and served
+      // source files - these are only watched and served.
+      // anything not listed here can't be dynamically loaded by other scripts.
       {pattern: 'lib/**/*.js', included: false},
       {pattern: 'ui/**/*.js', included: false},
       {pattern: 'ui/**/*.less', included: false},
       {pattern: 'third_party/**/*.js', included: false},
+      {pattern: 'test/**/*.js', included: false},
       {pattern: 'test/test/assets/*', included: false},
       {pattern: 'test/test/assets/3675/*', included: false},
       {pattern: 'dist/shaka-player.ui.js', included: false},
       {pattern: 'dist/locales.js', included: false},
+      {pattern: 'demo/**/*.js', included: false},
       {pattern: 'demo/locales/en.json', included: false},
       {pattern: 'demo/locales/source.json', included: false},
       {pattern: 'node_modules/sprintf-js/src/sprintf.js', included: false},
@@ -284,26 +284,30 @@ module.exports = (config) => {
     });
   }
 
+  const clientArgs = config.client.args[0];
+  clientArgs.testFiles = [];
+
   if (settings.test_custom_asset) {
     // If testing custom assets, we don't serve other unit or integration tests.
     // External asset tests are the basis for custom asset testing, so this file
     // is automatically included.
-    config.files.push('test/player_external.js');
+    clientArgs.testFiles.push('test/player_external.js');
   } else {
     // In a normal test run, we serve unit tests.
-    config.files.push('test/**/*_unit.js');
+    clientArgs.testFiles.push('test/**/*_unit.js');
 
     if (!settings.quick) {
       // If --quick is present, we don't serve integration tests.
-      config.files.push('test/**/*_integration.js');
+      clientArgs.testFiles.push('test/**/*_integration.js');
     }
     if (settings.external) {
       // If --external is present, we serve external asset tests.
-      config.files.push('test/**/*_external.js');
+      clientArgs.testFiles.push('test/**/*_external.js');
     }
   }
-  // We just modified the config in-place.  No need for config.set() after we
-  // push to config.files.
+
+  // These are the test files that will be dynamically loaded by boot.js.
+  clientArgs.testFiles = resolveGlobs(clientArgs.testFiles);
 
   const reporters = [];
 
@@ -348,14 +352,34 @@ module.exports = (config) => {
     const seed = settings.seed == null ? new Date().getTime() : settings.seed;
 
     // Run tests in a random order.
-    const clientArgs = config.client.args[0];
     clientArgs.random = true;
     clientArgs.seed = seed;
 
     console.log('Using a random test order (--random) with --seed=' + seed);
   }
 };
 
+/**
+ * Resolves a list of paths using globs into a list of explicit paths.
+ * Paths are all relative to the source directory.
+ *
+ * @param {!Array.<string>} list
+ * @return {!Array.<string>}
+ */
+function resolveGlobs(list) {
+  const options = {
+    cwd: __dirname,
+  };
+
+  const resolved = [];
+  for (const path of list) {
+    for (const resolvedPath of glob.sync(path, options)) {
+      resolved.push(resolvedPath);
+    }
+  }
+  return resolved;
+}
+
 /**
  * Determines which launchers and customLaunchers can be used and returns an
  * array of strings.

diff --git a/test/abr/simple_abr_manager_unit.js b/test/abr/simple_abr_manager_unit.js
@@ -4,11 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.abr.SimpleAbrManager');
-goog.require('shaka.test.ManifestGenerator');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.PlayerConfiguration');
-
 describe('SimpleAbrManager', () => {
   const sufficientBWMultiplier = 1.06;
   const defaultBandwidthEstimate = 500e3; // 500kbps

diff --git a/test/ads/ad_manager_unit.js b/test/ads/ad_manager_unit.js
@@ -4,12 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.Player');
-goog.require('shaka.ads.AdManager');
-goog.require('shaka.test.FakeVideo');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-
 describe('Ad manager', () => {
   /** @type {!shaka.test.FakeVideo} */
   let mockVideo;

diff --git a/test/cast/cast_proxy_unit.js b/test/cast/cast_proxy_unit.js
@@ -4,14 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cast.CastProxy');
-goog.require('shaka.cast.CastSender');
-goog.require('shaka.test.FakeVideo');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-goog.require('shaka.util.FakeEvent');
-goog.require('shaka.util.PublicPromise');
-
 describe('CastProxy', () => {
   const CastProxy = shaka.cast.CastProxy;
   const FakeEvent = shaka.util.FakeEvent;

diff --git a/test/cast/cast_receiver_integration.js b/test/cast/cast_receiver_integration.js
@@ -4,20 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.Player');
-goog.require('shaka.cast.CastReceiver');
-goog.require('shaka.cast.CastUtils');
-goog.require('shaka.log');
-goog.require('shaka.media.DrmEngine');
-goog.require('shaka.media.ManifestParser');
-goog.require('shaka.media.StreamingEngine');
-goog.require('shaka.net.NetworkingEngine');
-goog.require('shaka.test.TestScheme');
-goog.require('shaka.test.UiUtils');
-goog.require('shaka.util.EventManager');
-goog.require('shaka.util.Platform');
-goog.require('shaka.util.PublicPromise');
-
 // The receiver is only meant to run on the Chromecast, so we have the
 // ability to use modern APIs there that may not be available on all of the
 // browsers our library supports.  Because of this, CastReceiver tests will

diff --git a/test/cast/cast_receiver_unit.js b/test/cast/cast_receiver_unit.js
@@ -4,14 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cast.CastReceiver');
-goog.require('shaka.cast.CastUtils');
-goog.require('shaka.test.FakeVideo');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-goog.require('shaka.util.Platform');
-goog.require('shaka.util.PublicPromise');
-
 // The receiver is only meant to run on the Chromecast, so we have the
 // ability to use modern APIs there that may not be available on all of the
 // browsers our library supports.  Because of this, CastReceiver tests will

diff --git a/test/cast/cast_sender_unit.js b/test/cast/cast_sender_unit.js
@@ -4,12 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cast.CastSender');
-goog.require('shaka.cast.CastUtils');
-goog.require('shaka.test.StatusPromise');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-
 describe('CastSender', () => {
   const CastSender = shaka.cast.CastSender;
   const CastUtils = shaka.cast.CastUtils;

diff --git a/test/cast/cast_utils_unit.js b/test/cast/cast_utils_unit.js
@@ -4,19 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.Player');
-goog.require('shaka.cast.CastUtils');
-goog.require('shaka.media.MediaSourceEngine');
-goog.require('shaka.media.TimeRangesUtils');
-goog.require('shaka.test.FakeClosedCaptionParser');
-goog.require('shaka.test.FakeTextDisplayer');
-goog.require('shaka.test.UiUtils');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.EventManager');
-goog.require('shaka.util.FakeEvent');
-goog.require('shaka.util.FakeEventTarget');
-goog.require('shaka.util.ManifestParserUtils');
-
 describe('CastUtils', () => {
   const CastUtils = shaka.cast.CastUtils;
   const FakeEvent = shaka.util.FakeEvent;

diff --git a/test/cea/cea608_memory_unit.js b/test/cea/cea608_memory_unit.js
@@ -4,11 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.Cea608Memory');
-goog.require('shaka.cea.CeaUtils');
-goog.require('shaka.test.CeaUtils');
-goog.require('shaka.text.Cue');
-
 describe('Cea608Memory', () => {
   const CeaUtils = shaka.test.CeaUtils;
 

diff --git a/test/cea/cea708_service_unit.js b/test/cea/cea708_service_unit.js
@@ -4,13 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.Cea708Service');
-goog.require('shaka.cea.CeaUtils');
-goog.require('shaka.cea.DtvccPacket');
-goog.require('shaka.cea.DtvccPacketBuilder');
-goog.require('shaka.test.CeaUtils');
-goog.require('shaka.text.Cue');
-
 describe('Cea708Service', () => {
   const CeaUtils = shaka.test.CeaUtils;
 

diff --git a/test/cea/cea708_window_unit.js b/test/cea/cea708_window_unit.js
@@ -4,11 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.Cea708Window');
-goog.require('shaka.cea.CeaUtils');
-goog.require('shaka.test.CeaUtils');
-goog.require('shaka.text.Cue');
-
 describe('Cea708Window', () => {
   const CeaUtils = shaka.test.CeaUtils;
 

diff --git a/test/cea/cea_decoder_unit.js b/test/cea/cea_decoder_unit.js
@@ -4,12 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.CeaDecoder');
-goog.require('shaka.cea.CeaUtils');
-goog.require('shaka.log');
-goog.require('shaka.test.CeaUtils');
-goog.require('shaka.text.Cue');
-
 describe('CeaDecoder', () => {
   const CeaUtils = shaka.test.CeaUtils;
 

diff --git a/test/cea/dtvcc_packet_builder_unit.js b/test/cea/dtvcc_packet_builder_unit.js
@@ -4,9 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.DtvccPacket');
-goog.require('shaka.cea.DtvccPacketBuilder');
-
 describe('DtvccPacketBuilder', () => {
   /** @type {!shaka.cea.DtvccPacketBuilder} */
   let dtvccPacketBuilder;

diff --git a/test/cea/dtvcc_packet_unit.js b/test/cea/dtvcc_packet_unit.js
@@ -4,11 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.DtvccPacket');
-goog.require('shaka.cea.DtvccPacketBuilder');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-
 describe('DtvccPacket', () => {
   /** @type {!shaka.cea.DtvccPacket} */
   let dtvccPacket;

diff --git a/test/cea/mp4_cea_parser_unit.js b/test/cea/mp4_cea_parser_unit.js
@@ -4,10 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.cea.Mp4CeaParser');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.Error');
-
 describe('Mp4CeaParser', () => {
   const ceaInitSegmentUri = '/base/test/test/assets/cea-init.mp4';
   const ceaSegmentUri = '/base/test/test/assets/cea-segment.mp4';

diff --git a/test/dash/dash_parser_content_protection_unit.js b/test/dash/dash_parser_content_protection_unit.js
@@ -4,14 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('shaka.dash.ContentProtection');
-goog.require('shaka.dash.DashParser');
-goog.require('shaka.test.Dash');
-goog.require('shaka.test.FakeNetworkingEngine');
-goog.require('shaka.util.Error');
-goog.require('shaka.util.PlayerConfiguration');
-goog.require('shaka.util.Uint8ArrayUtils');
-
 // Test DRM-related parsing.
 describe('DashParser ContentProtection', () => {
   const Dash = shaka.test.Dash;

diff --git a/test/dash/dash_parser_live_unit.js b/test/dash/dash_parser_live_unit.js
@@ -4,19 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('goog.asserts');
-goog.require('shaka.dash.DashParser');
-goog.require('shaka.media.SegmentReference');
-goog.require('shaka.net.NetworkingEngine');
-goog.require('shaka.test.FakeNetworkingEngine');
-goog.require('shaka.test.ManifestParser');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.AbortableOperation');
-goog.require('shaka.util.Error');
-goog.require('shaka.util.PlayerConfiguration');
-goog.require('shaka.util.StringUtils');
-goog.requireType('shaka.util.PublicPromise');
-
 describe('DashParser Live', () => {
   const Util = shaka.test.Util;
   const ManifestParser = shaka.test.ManifestParser;

diff --git a/test/dash/dash_parser_manifest_unit.js b/test/dash/dash_parser_manifest_unit.js
@@ -4,21 +4,6 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-goog.require('goog.asserts');
-goog.require('shaka.media.SegmentReference');
-goog.require('shaka.net.NetworkingEngine');
-goog.require('shaka.test.Dash');
-goog.require('shaka.test.FakeNetworkingEngine');
-goog.require('shaka.test.ManifestGenerator');
-goog.require('shaka.test.Util');
-goog.require('shaka.util.AbortableOperation');
-goog.require('shaka.util.Error');
-goog.require('shaka.util.LanguageUtils');
-goog.require('shaka.util.ManifestParserUtils');
-goog.require('shaka.util.PlayerConfiguration');
-goog.require('shaka.util.StringUtils');
-goog.requireType('shaka.dash.DashParser');
-
 // Test basic manifest parsing functionality.
 describe('DashParser Manifest', () => {
   const ContentType = shaka.util.ManifestParserUtils.ContentType;