feat(coverage): thresholds to support maximum uncovered items (#7061)

docs/config/index.md

@@ -1481,7 +1481,26 @@ Do not show files with 100% statement, branch, and function coverage.
 
 #### coverage.thresholds
 
-Options for coverage thresholds
+Options for coverage thresholds.
+
+If a threshold is set to a positive number, it will be interpreted as the minimum percentage of coverage required. For example, setting the lines threshold to `90` means that 90% of lines must be covered.
+
+If a threshold is set to a negative number, it will be treated as the maximum number of uncovered items allowed. For example, setting the lines threshold to `-10` means that no more than 10 lines may be uncovered.
+
+<!-- eslint-skip -->
+```ts
+{
+  coverage: {
+    thresholds: {
+      // Requires 90% function coverage
+      functions: 90,
+
+      // Require that no more than 10 lines are uncovered
+      lines: -10,
+    }
+  }
+}
+```
 
 ##### coverage.thresholds.lines
 
@@ -1490,7 +1509,6 @@ Options for coverage thresholds
 - **CLI:** `--coverage.thresholds.lines=<number>`
 
 Global threshold for lines.
-See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
 
 ##### coverage.thresholds.functions
 
@@ -1499,7 +1517,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
 - **CLI:** `--coverage.thresholds.functions=<number>`
 
 Global threshold for functions.
-See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
 
 ##### coverage.thresholds.branches
 
@@ -1508,7 +1525,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
 - **CLI:** `--coverage.thresholds.branches=<number>`
 
 Global threshold for branches.
-See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
 
 ##### coverage.thresholds.statements
 
@@ -1517,7 +1533,6 @@ See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-threshol
 - **CLI:** `--coverage.thresholds.statements=<number>`
 
 Global threshold for statements.
-See [istanbul documentation](https://github.com/istanbuljs/nyc#coverage-thresholds) for more information.
 
 ##### coverage.thresholds.perFile
 
@@ -1535,7 +1550,7 @@ Check thresholds per file.
 - **Available for providers:** `'v8' | 'istanbul'`
 - **CLI:** `--coverage.thresholds.autoUpdate=<boolean>`
 
-Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is above the configured thresholds.
+Update all threshold values `lines`, `functions`, `branches` and `statements` to configuration file when current coverage is better than the configured thresholds.
 This option helps to maintain thresholds when coverage is improved.
 
 ##### coverage.thresholds.100

packages/vitest/src/utils/coverage.ts

@@ -363,25 +363,54 @@ export class BaseCoverageProvider<Options extends ResolvedCoverageOptions<'istan
         for (const thresholdKey of THRESHOLD_KEYS) {
           const threshold = thresholds[thresholdKey]
 
-          if (threshold !== undefined) {
+          if (threshold === undefined) {
+            continue
+          }
+
+          /**
+           * Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered),
+           * while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered).
+           */
+          if (threshold >= 0) {
             const coverage = summary.data[thresholdKey].pct
 
             if (coverage < threshold) {
               process.exitCode = 1
 
-              /*
+              /**
                * Generate error message based on perFile flag:
                * - ERROR: Coverage for statements (33.33%) does not meet threshold (85%) for src/math.ts
                * - ERROR: Coverage for statements (50%) does not meet global threshold (85%)
                */
-              let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${
-                name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
+              let errorMessage = `ERROR: Coverage for ${thresholdKey} (${coverage}%) does not meet ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
               } threshold (${threshold}%)`
 
               if (this.options.thresholds?.perFile && file) {
                 errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}`
               }
 
+              this.ctx.logger.error(errorMessage)
+            }
+          }
+          else {
+            const uncovered = summary.data[thresholdKey].total - summary.data[thresholdKey].covered
+            const absoluteThreshold = threshold * -1
+
+            if (uncovered > absoluteThreshold) {
+              process.exitCode = 1
+
+              /**
+               * Generate error message based on perFile flag:
+               * - ERROR: Uncovered statements (33) exceed threshold (30) for src/math.ts
+               * - ERROR: Uncovered statements (33) exceed global threshold (30)
+               */
+              let errorMessage = `ERROR: Uncovered ${thresholdKey} (${uncovered}) exceed ${name === GLOBAL_THRESHOLDS_KEY ? name : `"${name}"`
+              } threshold (${absoluteThreshold})`
+
+              if (this.options.thresholds?.perFile && file) {
+                errorMessage += ` for ${relative('./', file).replace(/\\/g, '/')}`
+              }
+
               this.ctx.logger.error(errorMessage)
             }
           }
@@ -416,12 +445,30 @@ export class BaseCoverageProvider<Options extends ResolvedCoverageOptions<'istan
 
       for (const key of THRESHOLD_KEYS) {
         const threshold = thresholds[key] ?? 100
-        const actual = Math.min(
-          ...summaries.map(summary => summary[key].pct),
-        )
+        /**
+         * Positive thresholds are treated as minimum coverage percentages (X means: X% of lines must be covered),
+         * while negative thresholds are treated as maximum uncovered counts (-X means: X lines may be uncovered).
+         */
+        if (threshold >= 0) {
+          const actual = Math.min(
+            ...summaries.map(summary => summary[key].pct),
+          )
 
-        if (actual > threshold) {
-          thresholdsToUpdate.push([key, actual])
+          if (actual > threshold) {
+            thresholdsToUpdate.push([key, actual])
+          }
+        }
+        else {
+          const absoluteThreshold = threshold * -1
+          const actual = Math.max(
+            ...summaries.map(summary => summary[key].total - summary[key].covered),
+          )
+
+          if (actual < absoluteThreshold) {
+            // If everything was covered, set new threshold to 100% (since a threshold of 0 would be considered as 0%)
+            const updatedThreshold = actual === 0 ? 100 : actual * -1
+            thresholdsToUpdate.push([key, updatedThreshold])
+          }
         }
       }
 

test/coverage-test/fixtures/configs/vitest.config.thresholds-auto-update.ts

@@ -9,14 +9,14 @@ export default defineConfig({
         // Global ones
         lines: 0.1,
         functions: 0.2,
-        branches: 0.3,
-        statements: 0.4,
+        branches: -1000,
+        statements: -2000,
 
         '**/src/math.ts': {
           branches: 0.1,
           functions: 0.2,
-          lines: 0.3,
-          statements: 0.4
+          lines: -1000,
+          statements: -2000,
         }
       }
     }

test/coverage-test/test/threshold-auto-update.test.ts

@@ -20,14 +20,14 @@ test('thresholds.autoUpdate updates thresholds', async () => {
             // Global ones
             lines: 0.1,
             functions: 0.2,
-            branches: 0.3,
-            statements: 0.4,
+            branches: -1000,
+            statements: -2000,
 
             '**/src/math.ts': {
               branches: 0.1,
               functions: 0.2,
-              lines: 0.3,
-              statements: 0.4
+              lines: -1000,
+              statements: -2000,
             }
           }
         }
@@ -56,13 +56,13 @@ test('thresholds.autoUpdate updates thresholds', async () => {
               lines: 55.55,
               functions: 33.33,
               branches: 100,
-              statements: 55.55,
+              statements: -8,
 
               '**/src/math.ts': {
                 branches: 100,
                 functions: 25,
-                lines: 50,
-                statements: 50
+                lines: -6,
+                statements: -6,
               }
             }
           }
@@ -84,13 +84,13 @@ test('thresholds.autoUpdate updates thresholds', async () => {
               lines: 33.33,
               functions: 33.33,
               branches: 100,
-              statements: 33.33,
+              statements: -4,
 
               '**/src/math.ts': {
                 branches: 100,
                 functions: 25,
-                lines: 25,
-                statements: 25
+                lines: -3,
+                statements: -3,
               }
             }
           }

test/coverage-test/test/threshold-failure.test.ts

@@ -2,7 +2,7 @@ import { expect } from 'vitest'
 import { sum } from '../fixtures/src/math'
 import { coverageTest, isV8Provider, normalizeURL, runVitest, test } from '../utils'
 
-test('failing thresholds', async () => {
+test('failing percentage thresholds', async () => {
   const { exitCode, stderr } = await runVitest({
     include: [normalizeURL(import.meta.url)],
     coverage: {
@@ -28,6 +28,36 @@ test('failing thresholds', async () => {
   expect(stderr).toContain('ERROR: Coverage for functions (25%) does not meet "**/fixtures/src/math.ts" threshold (100%)')
 })
 
+test('failing absolute thresholds', async () => {
+  const { exitCode, stderr } = await runVitest({
+    include: [normalizeURL(import.meta.url)],
+    coverage: {
+      all: false,
+      include: ['**/fixtures/src/math.ts'],
+      thresholds: {
+        '**/fixtures/src/math.ts': {
+          branches: -1,
+          functions: -2,
+          lines: -5,
+          statements: -1,
+        },
+      },
+    },
+  }, { throwOnError: false })
+
+  expect(exitCode).toBe(1)
+
+  if (isV8Provider()) {
+    expect(stderr).toContain('ERROR: Uncovered lines (6) exceed "**/fixtures/src/math.ts" threshold (5)')
+    expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)')
+    expect(stderr).toContain('ERROR: Uncovered statements (6) exceed "**/fixtures/src/math.ts" threshold (1)')
+  }
+  else {
+    expect(stderr).toContain('ERROR: Uncovered functions (3) exceed "**/fixtures/src/math.ts" threshold (2)')
+    expect(stderr).toContain('ERROR: Uncovered statements (3) exceed "**/fixtures/src/math.ts" threshold (1)')
+  }
+})
+
 coverageTest('cover some lines, but not too much', () => {
   expect(sum(1, 2)).toBe(3)
 })