Skip to content

Commit

Permalink
Functional Test execution on Lamdatest (#4472)
Browse files Browse the repository at this point in the history
  • Loading branch information
@dsilhavy
dsilhavy committed May 6, 2024
1 parent 8239d6a commit a7f983e
Show file tree
Hide file tree
Showing 8 changed files with 266 additions and 224 deletions.
93 changes: 60 additions & 33 deletions .circleci/config.yml
View file
Expand Up @@ -29,23 +29,36 @@ commands:
- run:
name: Check if preconditions are met for functional tests
command: |
if [ -z "$BROWSERSTACK_ACCESS_KEY" ]; then
echo "BrowserStack not configured, functional tests will not be executed."
if [ -z "LAMBDATEST_ACCESS_KEY" ]; then
echo "Lambdatest not configured, functional tests will not be executed."
circleci-agent step halt
fi
functional_test_setup:
steps:
- run:
name: Download the browserstack binary file to create a tunnel
command: wget "https://www.browserstack.com/browserstack-local/BrowserStackLocal-linux-x64.zip"
name: "Downloading tunnel binary"
command: |
wget https://downloads.lambdatest.com/tunnel/v3/linux/64bit/LT_Linux.zip
- run:
name: Unzip the browserstack binary file
command: unzip BrowserStackLocal-linux-x64.zip
name: "Extracting tunnel binary"
command: |
sudo apt-get install unzip
unzip LT_Linux.zip
- run:
name: Run browserstack with provided access key
command: ./BrowserStackLocal $BROWSERSTACK_ACCESS_KEY
name: "Executing tunnel binary"
background: true
command: |
./LT --user ${LAMBDATEST_EMAIL} --key ${LAMBDATEST_KEY}
sleep 40
- run:
name: "Setup custom environment variables"
command: |
echo 'export LT_USERNAME="${LAMBDATEST_USER}"' >> $BASH_ENV
echo 'export LT_ACCESS_KEY="${LAMBDATEST_ACCESS_KEY}"' >> $BASH_ENV
run_testbuild:
steps:
Expand All @@ -67,10 +80,11 @@ commands:
command: |
mkdir test/functional/results/test/karma/junit/adjusted
cd test/functional/results/test/karma/junit
for file in *.xml; do
for file in **/*.xml; do
[ -f "$file" ] || continue
xmlstarlet ed -L -u '//testcase[@time="NaN"]/@time' -v '0' "$file"
cp "$file" adjusted/$file
modified_file="${file//\//_}"
cp "$file" adjusted/$modified_file
done
- store_test_results:
path: test/functional/results/test/karma/junit/adjusted
Expand All @@ -79,6 +93,17 @@ commands:
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
Expand All @@ -89,9 +114,13 @@ commands:
default: "single"
type: string
configfile:
default: "browserstack"
default: "lambdatest"
type: string
steps:
- run:
name: Sleep
command: |
sleep 30
- run:
name: Run functional tests <<parameters.configfile>> - <<parameters.streamsfile>>
when: always
Expand Down Expand Up @@ -138,31 +167,22 @@ jobs:
- dependencies_setup
- run_testbuild

functional-tests-smoke:
functional-tests-single:
executor: dashjs-executor
steps:
- functional_precondition
- checkout
- run:
name: Virtual merge into development branch
command: |
if [ "${CIRCLE_BRANCH}" = "development" ]; then
echo "On development branch already, no merge needed"
else
git config --global user.email "circleci@example.com"
git config --global user.name "CircleCI"
git checkout development
git merge --no-edit --no-ff $CIRCLE_BRANCH
fi
- dependencies_setup
- run_testbuild
- functional_test_setup
- functional_steps
- run_test_suite:
streamsfile: single
configfile: browserstack
configfile: lambdatest
- process_test_results

functional-tests-smoke:
executor: dashjs-executor
steps:
- functional_steps
- run_test_suite:
streamsfile: smoke
configfile: browserstack
configfile: lambdatest-smoke
- process_test_results

functional-tests-full-part-1:
Expand All @@ -171,7 +191,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: drm_emsg_eptdelta_gaps
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-2:
Expand All @@ -180,7 +200,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: multiperiod_subtitle_vendor
configfile: browserstack
configfile: lambdatest-full
- process_test_results

functional-tests-full-part-3:
Expand All @@ -189,7 +209,7 @@ jobs:
- functional_steps
- run_test_suite:
streamsfile: vod_live_lowlatency_mss_multiaudio
configfile: browserstack
configfile: lambdatest-full
- process_test_results

workflows:
Expand All @@ -202,7 +222,14 @@ workflows:
branches:
ignore:
- development # skiping redundant job if already on development
- functional-tests-single:
filters:
branches:
ignore: # as creds are available only for non-forked branches
- /pull\/[0-9]+/
- functional-tests-smoke:
requires:
- functional-tests-single
filters:
branches:
ignore: # as creds are available only for non-forked branches
Expand Down
72 changes: 30 additions & 42 deletions test/functional/config/karma.functional.conf.cjs
View file
Expand Up @@ -13,18 +13,26 @@ module.exports = function (config) {
return
}

const testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
let testConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/${configFileName}.json`, 'utf-8'));
const streamsConfiguration = JSON.parse(fs.readFileSync(`test/functional/config/test-configurations/streams/${streamsFileName}.json`, 'utf-8'));
const includedTestfiles = _getIncludedTestfiles(streamsConfiguration)
const excludedTestfiles = _getExcludedTestfiles(streamsConfiguration)
const customContextFile = testConfiguration.customContextFile ? testConfiguration.customContextFile : 'test/functional/view/index.html';
const testvectors = streamsConfiguration.testvectors

if (testConfiguration && testConfiguration.type && testConfiguration.type === 'lambdatest') {
testConfiguration = _adjustConfigurationForLambdatest(testConfiguration)
}

config.set({

// base path that will be used to resolve all patterns (eg. files, exclude)
basePath: '../../../',

// web server port
hostname: testConfiguration.hostname ? testConfiguration.hostname : 'localhost',
port: testConfiguration.port ? testConfiguration.port : 9876,
protocol: testConfiguration.protocol ? testConfiguration.protocol : 'http',

// frameworks to use
// available frameworks: https://npmjs.org/browse/keyword/karma-adapter
Expand Down Expand Up @@ -68,8 +76,7 @@ module.exports = function (config) {
outputDir: `test/functional/results/test/karma/junit`, // results will be saved as $outputDir/$browserName.xml
outputFile: `${Date.now()}.xml`, // if included, results will be saved as $outputDir/$browserName/$outputFile
suite: '', // suite will become the package name attribute in xml testsuite element
useBrowserName: false, // add browser name to report and classes names
nameFormatter: undefined, // function (browser, result) to customize the name attribute in xml testcase element
useBrowserName: true, // add browser name to report and classes names
classNameFormatter: undefined, // function (browser, result) to customize the classname attribute in xml testcase element
properties: {}, // key value pair of properties to add to the <properties> section of the report
xmlVersion: null // use '1' if reporting to be per SonarQube 6.2 XML format
Expand Down Expand Up @@ -101,17 +108,13 @@ module.exports = function (config) {
webpack: {},

client: {
useIframe: true,
useIframe: false,
mocha: {
timeout: 180000
timeout: 120000
},
testvectors
},

// web server port
port: 9876,


// enable / disable colors in the output (reporters and logs)
colors: true,

Expand All @@ -124,45 +127,15 @@ module.exports = function (config) {
// enable / disable watching file and executing tests whenever any file changes
autoWatch: false,

browserNoActivityTimeout: 180000,
browserNoActivityTimeout: 120000,
browserDisconnectTimeout: 20000,
browserDisconnectTolerance: 3,
browserDisconnectTolerance: 2,

// start these browsers
// available browser launchers: https://npmjs.org/browse/keyword/karma-launcher
browsers: testConfiguration.browsers,

customLaunchers: {
bs_chrome_win_11: {
base: 'BrowserStack',
browser: 'chrome',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_firefox_win_11: {
base: 'BrowserStack',
browser: 'firefox',
'browser_version': 'latest',
os: 'Windows',
'os_version': '11',
},
bs_safari_mac: {
'base': 'BrowserStack',
'browser_version': 'latest',
'os': 'OS X',
'os_version': 'Ventura',
'browser': 'safari',
},
chrome_custom: {
base: 'Chrome',
flags: ['--disable-web-security', '--autoplay-policy=no-user-gesture-required', '--disable-popup-blocking']
},
firefox_custom: {
base: 'Firefox',
prefs: {}
}
},
customLaunchers: testConfiguration.customLaunchers,

// Continuous Integration mode
// if true, Karma captures browsers, runs the tests and exits
Expand Down Expand Up @@ -198,3 +171,18 @@ function _getExcludedTestfiles(testConfiguration) {
return `test/functional/test/${entry}.js`
})
}

function _adjustConfigurationForLambdatest(testConfiguration) {
if (testConfiguration && testConfiguration.customLaunchers) {
Object.keys(testConfiguration.customLaunchers).forEach((key) => {
testConfiguration.customLaunchers[key].user = process.env.LAMBDATEST_USER;
testConfiguration.customLaunchers[key].accessKey = process.env.LAMBDATEST_ACCESS_KEY;
testConfiguration.customLaunchers[key].config = {
hostname: 'hub.lambdatest.com',
port: 80
};
})
}

return testConfiguration
}

0 comments on commit a7f983e

Please sign in to comment.