feat(reporter): add check coverage thresholds

Add check option to coverageReporter options. Supports similar
options to istanbul's check-coverage feature.

It will cause karma to return a non-zero exit code if coverage
thresholds are not met.

Closes karma-runner#21

README.md

@@ -193,6 +193,45 @@ coverageReporter: {
 }
 ```
 
+#### check
+**Type:** Object
+
+**Description:** This will be used to configure minimum threshold enforcement for coverage results. If the thresholds are not met, karma will return failure. Thresholds, when specified as a positive number are taken to be the minimum percentage required. When a threshold is specified as a negative number it represents the maximum number of uncovered entities allowed.
+
+For example, `statements: 90` implies minimum statement coverage is 90%. `statements: -10` implies that no more than 10 uncovered statements are allowed.
+
+`global` applies to all files together and `each` on a per-file basis. A list of files or patterns can be excluded from enforcement via the `exclude` property. On a per-file or pattern basis, per-file thresholds can be overridden via the `overrides` property.
+
+```javascript
+coverageReporter: {
+  check: {
+    global: {
+      statements: 50,
+      branches: 50,
+      functions: 50,
+      lines: 50,
+      excludes: [
+        'foo/bar/**/*.js'
+      ]
+    },
+    each: {
+      statements: 50,
+      branches: 50,
+      functions: 50,
+      lines: 50,
+      excludes: [
+        'other/directory/**/*.js'
+      ],
+      overrides: {
+        'baz/component/**/*.js': {
+          statements: 98
+        }
+      }
+    }
+  }
+}
+```
+
 #### watermarks
 **Type:** Object
 

lib/reporter.js

@@ -3,6 +3,7 @@ var fs = require('fs');
 var util = require('util');
 var istanbul = require('istanbul');
 var dateformat = require('dateformat');
+var minimatch = require('minimatch');
 var globalSourceCache = require('./sourceCache');
 var coverageMap = require('./coverageMap');
 
@@ -32,6 +33,18 @@ Store.mix(SourceCacheStore, {
   }
 });
 
+/*
+ Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
+ Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+ */
+function isAbsolute(file) {
+  if (path.isAbsolute) {
+      return path.isAbsolute(file);
+  }
+
+  return path.resolve(file) === path.normalize(file);
+}
+
 
 // TODO(vojta): inject only what required (config.basePath, config.coverageReporter)
 var CoverageReporter = function(rootConfig, helper, logger) {
@@ -71,6 +84,132 @@ var CoverageReporter = function(rootConfig, helper, logger) {
     }
   }
 
+  /*
+   Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
+   Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+   */
+  function normalize(key) {
+    // Exclude keys will always be relative, but covObj keys can be absolute or relative
+    var excludeKey = isAbsolute(key) ? path.relative(basePath, key) : key;
+    // Also normalize for files that start with `./`, etc.
+    excludeKey = path.normalize(excludeKey);
+
+    return excludeKey;
+  }
+
+  /*
+   Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
+   Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+   */
+  function removeFiles(covObj, patterns) {
+    var obj = {};
+
+    Object.keys(covObj).forEach(function (key) {
+        // Do any patterns match the resolved key
+        var found = patterns.some(function(pattern) {
+          return minimatch(normalize(key), pattern, {dot: true});
+        });
+
+        // if no patterns match, keep the key
+        if (!found) {
+          obj[key] = covObj[key];
+        }
+    });
+
+    return obj;
+  }
+
+  /*
+   Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
+   Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+   */
+  function overrideThresholds(key, overrides) {
+    var thresholds = {};
+
+    // First match wins
+    Object.keys(overrides).some(function(pattern) {
+      if (minimatch(normalize(key), pattern, {dot: true})) {
+        thresholds = overrides[pattern];
+        return true;
+      }
+    });
+
+    return thresholds;
+  }
+
+  /*
+   Copyright (c) 2012, Yahoo! Inc.  All rights reserved.
+   Copyrights licensed under the New BSD License. See the accompanying LICENSE file for terms.
+   */
+  function checkCoverage(browser, collector) {
+    var defaultThresholds = {
+      global: {
+        statements: 0,
+        branches: 0,
+        lines: 0,
+        functions: 0,
+        excludes: []
+      },
+      each: {
+        statements: 0,
+        branches: 0,
+        lines: 0,
+        functions: 0,
+        excludes: [],
+        overrides: {}
+      }
+    };
+
+    var thresholds = helper.merge({}, defaultThresholds, config.check);
+
+    var rawCoverage = collector.getFinalCoverage();
+    var globalResults = istanbul.utils.summarizeCoverage(removeFiles(rawCoverage, thresholds.global.excludes));
+    var eachResults = removeFiles(rawCoverage, thresholds.each.excludes);
+
+    // Summarize per-file results and mutate original results.
+    Object.keys(eachResults).forEach(function (key) {
+      eachResults[key] = istanbul.utils.summarizeFileCoverage(eachResults[key]);
+    });
+
+    var coverageFailed = false;
+
+    function check(name, thresholds, actuals) {
+      [
+        'statements',
+        'branches',
+        'lines',
+        'functions'
+      ].forEach(function (key) {
+        var actual = actuals[key].pct;
+        var actualUncovered = actuals[key].total - actuals[key].covered;
+        var threshold = thresholds[key];
+
+        if (threshold < 0) {
+          if (threshold * -1 < actualUncovered) {
+            coverageFailed = true;
+            log.error(browser.name + ': Uncovered count for ' + key + ' (' + actualUncovered +
+              ') exceeds ' + name + ' threshold (' + -1 * threshold + ')');
+          }
+        } else {
+          if (actual < threshold) {
+            coverageFailed = true;
+            log.error(browser.name + ': Coverage for ' + key + ' (' + actual +
+              '%) does not meet ' + name + ' threshold (' + threshold + '%)');
+          }
+        }
+      });
+    }
+
+    check('global', thresholds.global, globalResults);
+
+    Object.keys(eachResults).forEach(function (key) {
+      var keyThreshold = helper.merge(thresholds.each, overrideThresholds(key, thresholds.each.overrides));
+      check('per-file' + ' (' + key + ') ', keyThreshold, eachResults[key]);
+    });
+
+    return coverageFailed;
+  }
+
   /**
    * Generate the output directory from the `coverageReporter.dir` and
    * `coverageReporter.subdir` options.
@@ -126,12 +265,25 @@ var CoverageReporter = function(rootConfig, helper, logger) {
     }
   };
 
-  this.onRunComplete = function(browsers) {
+  this.onRunComplete = function(browsers, results) {
+    var checkedCoverage = {};
+
     reporters.forEach(function(reporterConfig) {
       browsers.forEach(function(browser) {
 
         var collector = collectors[browser.id];
         if (collector) {
+          // If config.check is defined, check coverage levels for each browser
+          if (config.hasOwnProperty('check') && !checkedCoverage[browser.id]) {
+            checkedCoverage[browser.id] = true;
+            var coverageFailed = checkCoverage(browser, collector);
+            if (coverageFailed) {
+              if (results) {
+                results.exitCode = 1;
+              }
+            }
+          }
+
           pendingFileWritings++;
 
           var outputDir = helper.normalizeWinPath(path.resolve(basePath, generateOutputDir(browser.name,
@@ -144,6 +296,7 @@ var CoverageReporter = function(rootConfig, helper, logger) {
               sourceCache: sourceCache
             })
           });
+
           var reporter = istanbul.Report.create(reporterConfig.type || 'html', options);
 
           // If reporting to console, skip directory creation

package.json

@@ -19,9 +19,9 @@
   ],
   "author": "SATO taichi <ryushi@gmail.com>",
   "dependencies": {
-    "istanbul": "~0.3.0",
     "dateformat": "~1.0.6",
-    "minimatch": "~0.3.0"
+    "istanbul": "~0.3.0",
+    "minimatch": "^0.3.0"
   },
   "peerDependencies": {
     "karma": ">=0.9"

test/reporter.spec.coffee

@@ -26,10 +26,11 @@ describe 'reporter', ->
 
   mockAdd = sinon.spy()
   mockDispose = sinon.spy()
+  mockGetFinalCoverage = sinon.stub().returns {}
   mockCollector = class Collector
     add: mockAdd
     dispose: mockDispose
-    getFinalCoverage: -> null
+    getFinalCoverage: mockGetFinalCoverage
   mockWriteReport = sinon.spy()
   mockReportCreate = sinon.stub().returns writeReport: mockWriteReport
   mockMkdir = sinon.spy()
@@ -42,12 +43,22 @@ describe 'reporter', ->
     add: sinon.spy()
     get: sinon.spy()
 
+  mockSummarizeCoverage = sinon.stub().returns {
+    lines:      {total: 5, covered: 1, skipped: 0, pct: 20},
+    statements: {total: 5, covered: 1, skipped: 0, pct: 20},
+    functions:  {total: 5, covered: 1, skipped: 0, pct: 20},
+    branches:   {total: 5, covered: 1, skipped: 0, pct: 20}
+  }
+
   mocks =
     fs: mockFs
     istanbul:
       Store: mockStore
       Collector: mockCollector
       Report: create: mockReportCreate
+      utils:
+        summarizeCoverage: mockSummarizeCoverage
+        summarizeFileCoverage: mockSummarizeCoverage
     dateformat: require 'dateformat'
     './coverageMap': mockCoverageMap
 
@@ -293,3 +304,63 @@ describe 'reporter', ->
       browsers.forEach (b) -> reporter.onBrowserStart b
 
       expect(mockCoverageMap.get).not.to.have.been.called
+
+    it 'should log errors on low coverage and fail the build', ->
+      customConfig = _.merge {}, rootConfig,
+        coverageReporter:
+          check:
+            each:
+              statements: 50
+
+      mockGetFinalCoverage.returns
+        './foo/bar.js': {}
+        './foo/baz.js': {}
+
+      spy1 = sinon.spy()
+
+      customLogger = create: (name) ->
+        debug: -> null
+        info: -> null
+        warn: -> null
+        error: spy1
+
+      results = exitCode: 0
+
+      reporter = new m.CoverageReporter customConfig, mockHelper, customLogger
+      reporter.onRunStart()
+      browsers.forEach (b) -> reporter.onBrowserStart b
+      reporter.onRunComplete browsers, results
+
+      expect(spy1).to.have.been.called
+
+      expect(results.exitCode).to.not.equal 0
+
+    it 'should not log errors on sufficient coverage and not fail the build', ->
+      customConfig = _.merge {}, rootConfig,
+        coverageReporter:
+          check:
+            each:
+              statements: 10
+
+      mockGetFinalCoverage.returns
+        './foo/bar.js': {}
+        './foo/baz.js': {}
+
+      spy1 = sinon.spy()
+
+      customLogger = create: (name) ->
+        debug: -> null
+        info: -> null
+        warn: -> null
+        error: spy1
+
+      results = exitCode: 0
+
+      reporter = new m.CoverageReporter customConfig, mockHelper, customLogger
+      reporter.onRunStart()
+      browsers.forEach (b) -> reporter.onBrowserStart b
+      reporter.onRunComplete browsers, results
+
+      expect(spy1).to.not.have.been.called
+
+      expect(results.exitCode).to.equal 0