diff --git a/jest.config.js b/jest.config.js
index 4fe2e3c7bd8..8f29d7d6ed2 100644
--- a/jest.config.js
+++ b/jest.config.js
@@ -8,7 +8,8 @@ module.exports = {
     testPathIgnorePatterns: ignorePattern,
     testEnvironment: 'node',
     collectCoverage: true,
-    coverageReporters: ['none'],
+    coverageDirectory: '.nyc_output',
+    coverageReporters: ['json'],
     transform: {
         '^.+\\.(ts)?$': 'ts-jest',
     },
diff --git a/package.json b/package.json
index a21d62fd022..0528c2d7030 100644
--- a/package.json
+++ b/package.json
@@ -36,7 +36,7 @@
     "pretest": "yarn build && yarn lint && yarn prepsuite",
     "test": "jest --reporters=default",
     "test:smoketests": "nyc node smoketests",
-    "test:coverage": "nyc --require ts-node/register jest --forceExit",
+    "test:coverage": "nyc --no-clean --require ts-node/register jest --forceExit",
     "test:cli": "jest test --reporters=default --forceExit",
     "test:packages": "jest packages/ --reporters=default --forceExit",
     "test:ci": "yarn test:cli && yarn test:packages",
diff --git a/smoketests/helpers.js b/smoketests/helpers.js
index cfc25c9b8a9..fc13bef0043 100644
--- a/smoketests/helpers.js
+++ b/smoketests/helpers.js
@@ -76,6 +76,51 @@ const runTest = (package, cliArgs = [], logMessage, isSubPackage = false) => {
     });
 };
 
+const runTestStdout = ({ packageName, cliArgs, logMessage, isSubPackage } = {}) => {
+    // Simulate package missing
+    swapPkgName(packageName, isSubPackage);
+
+    const proc = execa(CLI_ENTRY_PATH, cliArgs, {
+        cwd: __dirname,
+    });
+
+    proc.stdin.setDefaultEncoding('utf-8');
+
+    return new Promise((resolve) => {
+        setTimeout(() => {
+            console.log('  timeout: killing process');
+            proc.kill();
+        }, 30000);
+
+        let hasPassed = false;
+
+        proc.stdout.on('data', (chunk) => {
+            let data = stripAnsi(chunk.toString());
+            console.log(`  stdout: ${data}`);
+
+            if (data.includes(logMessage)) {
+                hasPassed = true;
+                proc.kill();
+            }
+        });
+
+        proc.stderr.on('data', (chunk) => {
+            let data = stripAnsi(chunk.toString());
+            console.log(`  stderr: ${data}`);
+        });
+
+        proc.on('exit', () => {
+            swapPkgName(`.${packageName}`, isSubPackage);
+            resolve(hasPassed);
+        });
+
+        proc.on('error', () => {
+            swapPkgName(`.${packageName}`, isSubPackage);
+            resolve(false);
+        });
+    });
+};
+
 const runTestWithHelp = (package, cliArgs = [], logMessage, isSubPackage = false) => {
     // Simulate package missing
     swapPkgName(package, isSubPackage);
@@ -133,5 +178,6 @@ const runTestWithHelp = (package, cliArgs = [], logMessage, isSubPackage = false
 
 module.exports = {
     runTest,
+    runTestStdout,
     runTestWithHelp,
 };
diff --git a/smoketests/missing-packages/webpack-dev-server.test.js b/smoketests/missing-packages/webpack-dev-server.test.js
index 9533d86c3c8..d4596cad1b9 100644
--- a/smoketests/missing-packages/webpack-dev-server.test.js
+++ b/smoketests/missing-packages/webpack-dev-server.test.js
@@ -1,6 +1,6 @@
 'use strict';
 
-const { runTest } = require('../helpers');
+const { runTest, runTestStdout } = require('../helpers');
 
 const webpackDevServerTest = () => {
     const packageName = 'webpack-dev-server';
@@ -10,5 +10,13 @@ const webpackDevServerTest = () => {
     return runTest(packageName, args, logMessage);
 };
 
-module.exports.run = [webpackDevServerTest];
+const webpackDevServerWithHelpTest = () => {
+    const packageName = 'webpack-dev-server';
+    const cliArgs = ['help', 'serve'];
+    const logMessage = "To see all available options you need to install 'webpack-dev-server'";
+
+    return runTestStdout({ packageName, cliArgs, logMessage });
+};
+
+module.exports.run = [webpackDevServerTest, webpackDevServerWithHelpTest];
 module.exports.name = 'Missing webpack-dev-server';