Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Functional test improvements & test failure investigation #2921

Merged
merged 3 commits into from Jul 27, 2020
Merged

Functional test improvements & test failure investigation #2921

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6cdeade
Update Safari version used in functional tests
Jul 26, 2020
383cb7e
Fix JavaScript functions executed in IE11 by Selenium
Jul 26, 2020
8b9e876
Improve test logging
Jul 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
View file
Open in desktop
Expand Up @@ -68,7 +68,7 @@ jobs:
# - stage: testFuncOptional
# env: TRAVIS_MODE=funcTests UA=firefox OS="OS X 10.11"
- stage: testFuncOptional
env: TRAVIS_MODE=funcTests UA=safari OS="OS X 10.11" UA_VERSION="9.0"
env: TRAVIS_MODE=funcTests UA=safari OS="OS X 10.12" UA_VERSION="10.1"
addons:
sauce_connect:
tunnel_domains: localhost
Expand Down
2 changes: 1 addition & 1 deletion demo/main.js
View file
Open in desktop
Expand Up @@ -12,7 +12,7 @@ const STORAGE_KEYS = {
};

const testStreams = require('../tests/test-streams');
const defaultTestStreamUrl = testStreams['bbb'].url;
const defaultTestStreamUrl = testStreams[Object.keys(testStreams)[0]].url;
const sourceURL = decodeURIComponent(getURLParam('src', defaultTestStreamUrl));

let demoConfig = getURLParam('demoConfig', null);
Expand Down
86 changes: 41 additions & 45 deletions tests/functional/auto/setup.js
View file
Open in desktop
Expand Up @@ -62,6 +62,7 @@ HttpServer.createServer({
root: './'
}).listen(8000, hostname);

const stringifyResult = (result) => JSON.stringify(result, Object.keys(result).filter(k => k !== 'logs'), 2);
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
async function retry (attempt, numAttempts = 5, interval = 2000) {
try {
Expand All @@ -77,9 +78,8 @@ async function retry (attempt, numAttempts = 5, interval = 2000) {
}

async function testLoadedData (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -89,12 +89,11 @@ async function testLoadedData (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('loadeddata');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('loadeddata');
}

async function testIdleBufferLength (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
const autoplay = false;
window.startStream(url, config, callback, autoplay);
Expand All @@ -105,7 +104,8 @@ async function testIdleBufferLength (url, config) {
if (buffered.length) {
const bufferEnd = buffered.end(buffered.length - 1);
const duration = video.duration;
console.log(`[log] > progress: ${bufferEnd.toFixed(2)}/${duration.toFixed(2)} buffered.length: ${buffered.length}`);
console.log('[log] > progress: ' + bufferEnd.toFixed(2) + '/' + duration.toFixed(2) +
' buffered.length: ' + buffered.length);
if (bufferEnd >= maxBufferLength || bufferEnd > duration - 1) {
callback({ code: 'loadeddata', logs: window.logString });
}
Expand All @@ -115,30 +115,27 @@ async function testIdleBufferLength (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('loadeddata');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('loadeddata');
}

async function testSmoothSwitch (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
window.hls.once(window.Hls.Events.FRAG_CHANGED, (event, data) => {
window.switchToHighestLevel('next');
});
window.hls.on(window.Hls.Events.LEVEL_SWITCHED, (event, data) => {
console.log(`[test] > level switched: ${data.level}`);
console.log('[test] > level switched: ' + data.level);
let currentTime = video.currentTime;
if (data.level === window.hls.levels.length - 1) {
console.log(`[test] > switched on level: ${data.level}`);
console.log('[test] > switched on level: ' + data.level);
window.setTimeout(function () {
let newCurrentTime = video.currentTime;
console.log(
`[test] > currentTime delta : ${newCurrentTime - currentTime}`
);
console.log('[test] > currentTime delta : ' + (newCurrentTime - currentTime));
callback({
code: newCurrentTime > currentTime,
currentTimeDelta: newCurrentTime - currentTime,
logs: window.logString
});
}, 2000);
Expand All @@ -148,13 +145,12 @@ async function testSmoothSwitch (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals(true);
expect(result, stringifyResult(result)).to.have.property('currentTimeDelta').which.is.gt(0);
}

async function testSeekOnLive (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -169,18 +165,23 @@ async function testSeekOnLive (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('seeked');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('seeked');
}

async function testSeekOnVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
window.setTimeout(function () {
video.currentTime = video.duration - 5;
// Fail test early if more than 2 buffered ranges are found
video.onprogress = function () {
if (video.buffered.length > 2) {
callback({ code: 'buffer-gaps', bufferedRanges: video.buffered.length, logs: window.logString });
}
};
}, 5000);
};
video.onended = function () {
Expand All @@ -190,13 +191,12 @@ async function testSeekOnVOD (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('ended');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('ended');
}

async function testSeekEndVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -211,13 +211,12 @@ async function testSeekEndVOD (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('code').which.equals('ended');
expect(result, stringifyResult(result)).to.have.property('code').which.equals('ended');
}

async function testIsPlayingVOD (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.onloadeddata = function () {
Expand All @@ -229,29 +228,26 @@ async function testIsPlayingVOD (url, config) {
let currentTime = video.currentTime;
if (expectedPlaying) {
window.setTimeout(function () {
console.log(
`[test] > video expected playing. [last currentTime/new currentTime]=[${currentTime}/${video.currentTime}]`
);
console.log('[test] > video expected playing. last currentTime/new currentTime=' +
currentTime + '/' + video.currentTime);
callback({ playing: currentTime !== video.currentTime });
}, 5000);
} else {
console.log(
`[test] > video not playing. [paused/ended/buffered.length]=[${video.paused}/${video.ended}/${video.buffered.length}]`
);
console.log('[test] > video not playing. paused/ended/buffered.length=' +
video.paused + '/' + video.ended + '/' + video.buffered.length);
callback({ playing: false });
}
};
},
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('playing').which.is.true;
expect(result, stringifyResult(result)).to.have.property('playing').which.is.true;
}

async function testSeekBackToStart (url, config) {
const result = await browser.executeAsyncScript(
(url, config) => {
let callback = arguments[arguments.length - 1];
const result = await browser.executeAsyncScript(function (url, config) {
const callback = arguments[arguments.length - 1];
window.startStream(url, config, callback);
const video = window.video;
video.ontimeupdate = function () {
Expand All @@ -275,7 +271,7 @@ async function testSeekBackToStart (url, config) {
url,
config
);
expect(result, JSON.stringify(result, null, 2)).to.have.property('playing').which.is.true;
expect(result, stringifyResult(result)).to.have.property('playing').which.is.true;
}

describe(`testing hls.js playback in the browser on "${browserDescription}"`, function () {
Expand Down Expand Up @@ -421,7 +417,7 @@ describe(`testing hls.js playback in the browser on "${browserDescription}"`, fu
testIsPlayingVOD.bind(null, url, config)
);
it(
`should seek 5s from end and receive video ended event for ${stream.description}`,
`should seek 5s from end and receive video ended event for ${stream.description} with 2 or less buffered ranges`,
testSeekOnVOD.bind(null, url, config)
);
// it(`should seek on end and receive video ended event for ${stream.description}`, testSeekEndVOD.bind(null, url));
Expand Down
2 changes: 1 addition & 1 deletion tests/functional/auto/testbench.js
View file
Open in desktop
Expand Up @@ -92,7 +92,7 @@ function startStream (streamUrl, config, callback, autoplay) {
var playPromise = video.play();
if (playPromise) {
playPromise.catch(function (error) {
console.log('[test] > video.play() failed with error:', error);
console.log('[test] > video.play() failed with error: ' + error.name + ' ' + error.message);
if (error.name === 'NotAllowedError') {
console.log('[test] > Attempting to play with video muted');
video.muted = true;
Expand Down