Skip to content

Commit

Permalink
Merge pull request #2921 from video-dev/task/improve-func-test-runs
Browse files Browse the repository at this point in the history 
Debug functional test failure
  • Loading branch information
@robwalch
robwalch committed Jul 27, 2020
2 parents 8234f43 + 8b9e876 commit 8a29da3
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 48 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Expand Up @@ -68,7 +68,7 @@ jobs:
# - stage: testFuncOptional
# env: TRAVIS_MODE=funcTests UA=firefox OS="OS X 10.11"
- stage: testFuncOptional
env: TRAVIS_MODE=funcTests UA=safari OS="OS X 10.11" UA_VERSION="9.0"
env: TRAVIS_MODE=funcTests UA=safari OS="OS X 10.12" UA_VERSION="10.1"
addons:
sauce_connect:
tunnel_domains: localhost
Expand Down
2 changes: 1 addition & 1 deletion demo/main.js
View file
Expand Up @@ -12,7 +12,7 @@ const STORAGE_KEYS = {
};

const testStreams = require('../tests/test-streams');
const defaultTestStreamUrl = testStreams['bbb'].url;
const defaultTestStreamUrl = testStreams[Object.keys(testStreams)[0]].url;
const sourceURL = decodeURIComponent(getURLParam('src', defaultTestStreamUrl));

let demoConfig = getURLParam('demoConfig', null);
Expand Down
86 changes: 41 additions & 45 deletions tests/functional/auto/setup.js
View file
Expand Up @@ -62,6 +62,7 @@ HttpServer.createServer({
root: './'
}).listen(8000, hostname);

const stringifyResult = (result) => JSON.stringify(result, Object.keys(result).filter(k => k !== 'logs'), 2);
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
async function retry (attempt, numAttempts = 5, interval = 2000) {
try {
Expand All @@ -77,9 +78,8 @@ async function retry (attempt, numAttempts = 5, interval = 2000) {
}

async function testLoadedData (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -89,12 +89,11 @@ async function testLoadedData (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('loadeddata');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('loadeddata');
}

async function testIdleBufferLength (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
const autoplay = false;
window.startStream(url, config, callback, autoplay);
Expand All @@ -105,7 +104,8 @@ async function testIdleBufferLength (url, config) {
if (buffered.length) {
const bufferEnd = buffered.end(buffered.length - 1);
const duration = video.duration;
console.log(`[log] > progress: ${bufferEnd.toFixed(2)}/${duration.toFixed(2)} buffered.length: ${buffered.length}`);
console.log('[log] > progress: ' + bufferEnd.toFixed(2) + '/' + duration.toFixed(2) +
' buffered.length: ' + buffered.length);
if (bufferEnd >= maxBufferLength || bufferEnd > duration - 1) {
callback({ code: 'loadeddata', logs: window.logString });
}
Expand All @@ -115,30 +115,27 @@ async function testIdleBufferLength (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('loadeddata');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('loadeddata');
}

async function testSmoothSwitch (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
window.hls.once(window.Hls.Events.FRAG_CHANGED, (event, data) => {
window.switchToHighestLevel('next');
});
window.hls.on(window.Hls.Events.LEVEL_SWITCHED, (event, data) => {
console.log(`[test] > level switched: ${data.level}`);
console.log('[test] > level switched: ' + data.level);
let currentTime = video.currentTime;
if (data.level === window.hls.levels.length - 1) {
console.log(`[test] > switched on level: ${data.level}`);
console.log('[test] > switched on level: ' + data.level);
window.setTimeout(function () {
let newCurrentTime = video.currentTime;
console.log(
`[test] > currentTime delta : ${newCurrentTime - currentTime}`
);
console.log('[test] > currentTime delta : ' + (newCurrentTime - currentTime));
callback({
code: newCurrentTime > currentTime,
currentTimeDelta: newCurrentTime - currentTime,
logs: window.logString
});
}, 2000);
Expand All @@ -148,13 +145,12 @@ async function testSmoothSwitch (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals(true);
expect(result, stringifyResult(result)).to.have.property('currentTimeDelta').which.is.gt(0);
}

async function testSeekOnLive (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -169,18 +165,23 @@ async function testSeekOnLive (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('seeked');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('seeked');
}

async function testSeekOnVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
window.setTimeout(function () {
video.currentTime = video.duration - 5;
// Fail test early if more than 2 buffered ranges are found
video.onprogress = function () {
if (video.buffered.length > 2) {
callback({ code: 'buffer-gaps', bufferedRanges: video.buffered.length, logs: window.logString });
}
};
}, 5000);
};
video.onended = function () {
Expand All @@ -190,13 +191,12 @@ async function testSeekOnVOD (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('ended');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('ended');
}

async function testSeekEndVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -211,13 +211,12 @@ async function testSeekEndVOD (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('ended');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('ended');
}

async function testIsPlayingVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -229,29 +228,26 @@ async function testIsPlayingVOD (url, config) {
let currentTime = video.currentTime;
if (expectedPlaying) {
window.setTimeout(function () {
console.log(
`[test] > video expected playing. [last currentTime/new currentTime]=[${currentTime}/${video.currentTime}]`
);
console.log('[test] > video expected playing. last currentTime/new currentTime=' +
currentTime + '/' + video.currentTime);
callback({ playing: currentTime !== video.currentTime });
}, 5000);
} else {
console.log(
`[test] > video not playing. [paused/ended/buffered.length]=[${video.paused}/${video.ended}/${video.buffered.length}]`
);
console.log('[test] > video not playing. paused/ended/buffered.length=' +
video.paused + '/' + video.ended + '/' + video.buffered.length);
callback({ playing: false });
}
};
},
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('playing').which.is.true;
expect(result, stringifyResult(result)).to.have.property('playing').which.is.true;
}

async function testSeekBackToStart (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.ontimeupdate = function () {
Expand All @@ -275,7 +271,7 @@ async function testSeekBackToStart (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('playing').which.is.true;
expect(result, stringifyResult(result)).to.have.property('playing').which.is.true;
}

describe(`testing hls.js playback in the browser on "${browserDescription}"`, function () {
Expand Down Expand Up @@ -421,7 +417,7 @@ describe(`testing hls.js playback in the browser on "${browserDescription}"`, fu
testIsPlayingVOD.bind(null, url, config)
);
it(
`should seek 5s from end and receive video ended event for ${stream.description}`,
`should seek 5s from end and receive video ended event for ${stream.description} with 2 or less buffered ranges`,
testSeekOnVOD.bind(null, url, config)
);
// it(`should seek on end and receive video ended event for ${stream.description}`, testSeekEndVOD.bind(null, url));
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/auto/testbench.js
View file
Expand Up @@ -92,7 +92,7 @@ function startStream (streamUrl, config, callback, autoplay) {
var playPromise = video.play();
if (playPromise) {
playPromise.catch(function (error) {
console.log('[test] > video.play() failed with error:', error);
console.log('[test] > video.play() failed with error: ' + error.name + ' ' + error.message);
if (error.name === 'NotAllowedError') {
console.log('[test] > Attempting to play with video muted');
video.muted = true;
Expand Down

0 comments on commit 8a29da3

Please sign in to comment.