feat(reporter): add check coverage thresholds

Add check option to coverageReporter options. Supports similar
options to istanbul's check-coverage feature.

It will cause karma to return a non-zero exit code if coverage
thresholds are not met.

Closes karma-runner#21

LICENSE

@@ -17,4 +17,31 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+----
+
+Copyright 2012 Yahoo! Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the Yahoo! Inc. nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL YAHOO! INC. BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -190,6 +190,45 @@ coverageReporter: {
 }
 ```
 
+#### check
+**Type:** Object
+
+**Description:** This will be used to configure minimum threshold enforcement for coverage results. If the thresholds are not met, karma will return failure. Thresholds, when specified as a positive number are taken to be the minimum percentage required. When a threshold is specified as a negative number it represents the maximum number of uncovered entities allowed.
+
+For example, `statements: 90` implies minimum statement coverage is 90%. `statements: -10` implies that no more than 10 uncovered statements are allowed.
+
+`global` applies to all files together and `each` on a per-file basis. A list of files or patterns can be excluded from enforcement via the `exclude` property. On a per-file or pattern basis, per-file thresholds can be overridden via the `overrides` property.
+
+```javascript
+coverageReporter: {
+  check: {
+    global: {
+      statements: 50,
+      branches: 50,
+      functions: 50,
+      lines: 50,
+      excludes: [
+        'foo/bar/**/*.js'
+      ]
+    },
+    each: {
+      statements: 50,
+      branches: 50,
+      functions: 50,
+      lines: 50,
+      excludes: [
+        'other/directory/**/*.js'
+      ],
+      overrides: {
+        'baz/component/**/*.js': {
+          statements: 98
+        }
+      }
+    }
+  }
+}
+```
+
 #### watermarks
 **Type:** Object
 

lib/reporter.js

@@ -1,6 +1,12 @@
+// Part of this code is based on [1], which is licensed under the New BSD License.
+// For more information see the See the accompanying LICENSE file for terms.
+//
+// [1]: https://github.com/gotwarlost/istanbul/blob/master/lib/command/check-coverage.js
+
 var path = require('path')
 var util = require('util')
 var istanbul = require('istanbul')
+var minimatch = require('minimatch')
 var globalSourceCache = require('./sourceCache')
 var coverageMap = require('./coverageMap')
 
@@ -29,6 +35,14 @@ Store.mix(SourceCacheStore, {
   }
 })
 
+function isAbsolute (file) {
+  if (path.isAbsolute) {
+    return path.isAbsolute(file)
+  }
+
+  return path.resolve(file) === path.normalize(file)
+}
+
 // TODO(vojta): inject only what required (config.basePath, config.coverageReporter)
 var CoverageReporter = function (rootConfig, helper, logger) {
   var _ = helper._
@@ -71,6 +85,116 @@ var CoverageReporter = function (rootConfig, helper, logger) {
     }
   }
 
+  function normalize (key) {
+    // Exclude keys will always be relative, but covObj keys can be absolute or relative
+    var excludeKey = isAbsolute(key) ? path.relative(basePath, key) : key
+    // Also normalize for files that start with `./`, etc.
+    excludeKey = path.normalize(excludeKey)
+
+    return excludeKey
+  }
+
+  function removeFiles (covObj, patterns) {
+    var obj = {}
+
+    Object.keys(covObj).forEach(function (key) {
+      // Do any patterns match the resolved key
+      var found = patterns.some(function (pattern) {
+        return minimatch(normalize(key), pattern, {dot: true})
+      })
+
+      // if no patterns match, keep the key
+      if (!found) {
+        obj[key] = covObj[key]
+      }
+    })
+
+    return obj
+  }
+
+  function overrideThresholds (key, overrides) {
+    var thresholds = {}
+
+    // First match wins
+    Object.keys(overrides).some(function (pattern) {
+      if (minimatch(normalize(key), pattern, {dot: true})) {
+        thresholds = overrides[pattern]
+        return true
+      }
+    })
+
+    return thresholds
+  }
+
+  function checkCoverage (browser, collector) {
+    var defaultThresholds = {
+      global: {
+        statements: 0,
+        branches: 0,
+        lines: 0,
+        functions: 0,
+        excludes: []
+      },
+      each: {
+        statements: 0,
+        branches: 0,
+        lines: 0,
+        functions: 0,
+        excludes: [],
+        overrides: {}
+      }
+    }
+
+    var thresholds = helper.merge({}, defaultThresholds, config.check)
+
+    var rawCoverage = collector.getFinalCoverage()
+    var globalResults = istanbul.utils.summarizeCoverage(removeFiles(rawCoverage, thresholds.global.excludes))
+    var eachResults = removeFiles(rawCoverage, thresholds.each.excludes)
+
+    // Summarize per-file results and mutate original results.
+    Object.keys(eachResults).forEach(function (key) {
+      eachResults[key] = istanbul.utils.summarizeFileCoverage(eachResults[key])
+    })
+
+    var coverageFailed = false
+
+    function check (name, thresholds, actuals) {
+      [
+        'statements',
+        'branches',
+        'lines',
+        'functions'
+      ].forEach(function (key) {
+        var actual = actuals[key].pct
+        var actualUncovered = actuals[key].total - actuals[key].covered
+        var threshold = thresholds[key]
+
+        if (threshold < 0) {
+          if (threshold * -1 < actualUncovered) {
+            coverageFailed = true
+            log.error(browser.name + ': Uncovered count for ' + key + ' (' + actualUncovered +
+              ') exceeds ' + name + ' threshold (' + -1 * threshold + ')')
+          }
+        } else {
+          if (actual < threshold) {
+            coverageFailed = true
+            log.error(browser.name + ': Coverage for ' + key + ' (' + actual +
+              '%) does not meet ' + name + ' threshold (' + threshold + '%)')
+          }
+        }
+      })
+    }
+
+    check('global', thresholds.global, globalResults)
+
+    Object.keys(eachResults).forEach(function (key) {
+      var keyThreshold = helper.merge(thresholds.each, overrideThresholds(key, thresholds.each.overrides))
+      check('per-file' + ' (' + key + ') ', keyThreshold, eachResults[key])
+    })
+
+    return coverageFailed
+  }
+
   /**
    * Generate the output directory from the `coverageReporter.dir` and
    * `coverageReporter.subdir` options.
@@ -126,11 +250,24 @@ var CoverageReporter = function (rootConfig, helper, logger) {
     }
   }
 
-  this.onRunComplete = function (browsers) {
+  this.onRunComplete = function (browsers, results) {
+    var checkedCoverage = {}
+
     reporters.forEach(function (reporterConfig) {
       browsers.forEach(function (browser) {
         var collector = collectors[browser.id]
         if (collector) {
+          // If config.check is defined, check coverage levels for each browser
+          if (config.hasOwnProperty('check') && !checkedCoverage[browser.id]) {
+            checkedCoverage[browser.id] = true
+            var coverageFailed = checkCoverage(browser, collector)
+            if (coverageFailed) {
+              if (results) {
+                results.exitCode = 1
+              }
+            }
+          }
+
           pendingFileWritings++
 
           var mainDir = reporterConfig.dir || config.dir
@@ -146,6 +283,7 @@ var CoverageReporter = function (rootConfig, helper, logger) {
           }, config, reporterConfig, {
             dir: outputDir
           })
+
           var reporter = istanbul.Report.create(reporterConfig.type || 'html', options)
 
           // If reporting to console, skip directory creation

package.json

@@ -19,8 +19,8 @@
   ],
   "author": "SATO taichi <ryushi@gmail.com>",
   "dependencies": {
-    "istanbul": "^0.3.0",
     "dateformat": "^1.0.6",
+    "istanbul": "^0.3.0",
     "minimatch": "^2.0.8"
   },
   "license": "MIT",

test/reporter.spec.coffee

@@ -26,10 +26,11 @@ describe 'reporter', ->
 
   mockAdd = sinon.spy()
   mockDispose = sinon.spy()
+  mockGetFinalCoverage = sinon.stub().returns {}
   mockCollector = class Collector
     add: mockAdd
     dispose: mockDispose
-    getFinalCoverage: -> null
+    getFinalCoverage: mockGetFinalCoverage
   mockWriteReport = sinon.spy()
   mockReportCreate = sinon.stub().returns writeReport: mockWriteReport
   mockMkdir = sinon.spy()
@@ -48,13 +49,23 @@ describe 'reporter', ->
     functions: [50, 80]
     lines: [50, 80]
 
+  mockSummarizeCoverage = sinon.stub().returns {
+    lines:      {total: 5, covered: 1, skipped: 0, pct: 20},
+    statements: {total: 5, covered: 1, skipped: 0, pct: 20},
+    functions:  {total: 5, covered: 1, skipped: 0, pct: 20},
+    branches:   {total: 5, covered: 1, skipped: 0, pct: 20}
+  }
+
   mocks =
     fs: mockFs
     istanbul:
       Store: mockStore
       Collector: mockCollector
       Report: create: mockReportCreate
       config: defaultConfig: sinon.stub().returns(reporting: watermarks: mockDefaultWatermarks)
+      utils:
+        summarizeCoverage: mockSummarizeCoverage
+        summarizeFileCoverage: mockSummarizeCoverage
     dateformat: require 'dateformat'
     './coverageMap': mockCoverageMap
 
@@ -378,3 +389,62 @@ describe 'reporter', ->
 
       expect(mockDispose).not.to.have.been.calledBefore mockWriteReport
 
+    it 'should log errors on low coverage and fail the build', ->
+      customConfig = _.merge {}, rootConfig,
+        coverageReporter:
+          check:
+            each:
+              statements: 50
+
+      mockGetFinalCoverage.returns
+        './foo/bar.js': {}
+        './foo/baz.js': {}
+
+      spy1 = sinon.spy()
+
+      customLogger = create: (name) ->
+        debug: -> null
+        info: -> null
+        warn: -> null
+        error: spy1
+
+      results = exitCode: 0
+
+      reporter = new m.CoverageReporter customConfig, mockHelper, customLogger
+      reporter.onRunStart()
+      browsers.forEach (b) -> reporter.onBrowserStart b
+      reporter.onRunComplete browsers, results
+
+      expect(spy1).to.have.been.called
+
+      expect(results.exitCode).to.not.equal 0
+
+    it 'should not log errors on sufficient coverage and not fail the build', ->
+      customConfig = _.merge {}, rootConfig,
+        coverageReporter:
+          check:
+            each:
+              statements: 10
+
+      mockGetFinalCoverage.returns
+        './foo/bar.js': {}
+        './foo/baz.js': {}
+
+      spy1 = sinon.spy()
+
+      customLogger = create: (name) ->
+        debug: -> null
+        info: -> null
+        warn: -> null
+        error: spy1
+
+      results = exitCode: 0
+
+      reporter = new m.CoverageReporter customConfig, mockHelper, customLogger
+      reporter.onRunStart()
+      browsers.forEach (b) -> reporter.onBrowserStart b
+      reporter.onRunComplete browsers, results
+
+      expect(spy1).to.not.have.been.called
+
+      expect(results.exitCode).to.equal 0