fix: Fix issue with maxDuplicateProblems setting (#2237)

integration-tests/snapshots/AdaDoom3/AdaDoom3/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.*"]
 Summary:
   files: 105
   filesWithIssues: 98
-  issues: 3081
+  issues: 3332
   errors: 0
 Errors: []
 

integration-tests/snapshots/AdaDoom3/AdaDoom3/snapshot.txt

@@ -3,7 +3,7 @@ Repository: AdaDoom3/AdaDoom3
 Url: "https://github.com/AdaDoom3/AdaDoom3.git"
 Args: ["**/*.*"]
 Lines:
- CSpell: Files checked: 105, Issues found: 3081 in 98 files
+ CSpell: Files checked: 105, Issues found: 3332 in 98 files
  exit code: 1
 ./Engine/Assembly/PPC/neo-engine-cpu.adb:5:78      - Unknown word (Squirek)    -- Copyright (C) 2016 Justin Squirek
 ./Engine/Assembly/x86-64/neo-engine-cpu.adb:101:111   - Unknown word (OSXSAVE)    -- ECX_Register, 27); -- OSXSAVE

integration-tests/snapshots/Azure/azure-rest-api-specs/snapshot.txt

@@ -3,7 +3,7 @@ Repository: Azure/azure-rest-api-specs
 Url: "https://github.com/Azure/azure-rest-api-specs.git"
 Args: ["--config=cSpell.json","**/*.{md,ts,js}"]
 Lines:
- CSpell: Files checked: 1761, Issues found: 1929 in 634 files
+ CSpell: Files checked: 1761, Issues found: 2070 in 634 files
  exit code: 1
 ./README.md:20:179    - Unknown word (Dataplane)  -- RPs' Resource types or Dataplane API versions that represent
 ./README.md:30:134    - Unknown word (dataplane)  -- per resource type or dataplane service version. This

integration-tests/snapshots/MartinThoma/LaTeX-examples/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.{md,tex}"]
 Summary:
   files: 1402
   filesWithIssues: 557
-  issues: 4948
+  issues: 5077
   errors: 0
 Errors: []
 

integration-tests/snapshots/MartinThoma/LaTeX-examples/snapshot.txt

@@ -3,7 +3,7 @@ Repository: MartinThoma/LaTeX-examples
 Url: "https://github.com/MartinThoma/LaTeX-examples.git"
 Args: ["**/*.{md,tex}"]
 Lines:
- CSpell: Files checked: 1402, Issues found: 4948 in 557 files
+ CSpell: Files checked: 1402, Issues found: 5077 in 557 files
  exit code: 1
 ./README.md:12:35     - Unknown word (Wikpedia)   -- can be found on [my Wikpedia Commons user page](http
 ./README.md:20:26     - Unknown word (seperate)   -- Every LaTeX file is in a seperate folder and has its own

integration-tests/snapshots/TheAlgorithms/Python/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.{md,py}"]
 Summary:
   files: 1020
   filesWithIssues: 418
-  issues: 2785
+  issues: 3011
   errors: 0
 Errors: []
 

integration-tests/snapshots/TheAlgorithms/Python/snapshot.txt

@@ -3,7 +3,7 @@ Repository: TheAlgorithms/Python
 Url: "https://github.com/TheAlgorithms/Python.git"
 Args: ["**/*.{md,py}"]
 Lines:
- CSpell: Files checked: 1020, Issues found: 2785 in 418 files
+ CSpell: Files checked: 1020, Issues found: 3011 in 418 files
  exit code: 1
 ./CONTRIBUTING.md:131:33    - Unknown word (pytest)     -- doctests will be run by pytest as part of our automated
 ./CONTRIBUTING.md:134:14    - Unknown word (doctest)    -- python3 -m doctest -v my_submission.py

integration-tests/snapshots/alexiosc/megistos/report.yaml

@@ -5,7 +5,7 @@ Args: ["--config=${repoConfig}/cspell.json","**/*.{md,c,h,html}"]
 Summary:
   files: 746
   filesWithIssues: 719
-  issues: 28136
+  issues: 29574
   errors: 0
 Errors: []
 

integration-tests/snapshots/alexiosc/megistos/snapshot.txt

@@ -3,7 +3,7 @@ Repository: alexiosc/megistos
 Url: "https://github.com/alexiosc/megistos.git"
 Args: ["--config=../../../../config/repositories/alexiosc/megistos/cspell.json","**/*.{md,c,h,html}"]
 Lines:
- CSpell: Files checked: 746, Issues found: 28136 in 719 files
+ CSpell: Files checked: 746, Issues found: 29574 in 719 files
  exit code: 1
 ./ATTIC/doc/examples/mod_dialog.c:18:13     - Unknown word (margv)      -- if (sameas (margv [6], "OK") || sameas
 ./ATTIC/doc/examples/mod_dialog.c:9:29      - Unknown word (FOOVT)      -- dialog_run ("foo", FOOVT, FOOLT, inp_buffer)

integration-tests/snapshots/django/django/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.{md,py}"]
 Summary:
   files: 2732
   filesWithIssues: 859
-  issues: 7131
+  issues: 7726
   errors: 0
 Errors: []
 

integration-tests/snapshots/django/django/snapshot.txt

@@ -3,7 +3,7 @@ Repository: django/django
 Url: "https://github.com/django/django.git"
 Args: ["**/*.{md,py}"]
 Lines:
- CSpell: Files checked: 2732, Issues found: 7131 in 859 files
+ CSpell: Files checked: 2732, Issues found: 7726 in 859 files
  exit code: 1
 ./django/apps/config.py:125:76    - Unknown word (isclass)    -- getmembers(mod, inspect.isclass)
 ./django/apps/registry.py:132:40    - Unknown word (unconfigured) -- not ready" is due to unconfigured settings, accessing

integration-tests/snapshots/googleapis/google-cloud-cpp/report.yaml

@@ -5,7 +5,7 @@ Args: ["--config=${repoConfig}/cspell.json","**/*","-e","{*.BUILD,BUILD,CHANGELO
 Summary:
   files: 2538
   filesWithIssues: 1732
-  issues: 14547
+  issues: 16150
   errors: 0
 Errors: []
 

integration-tests/snapshots/googleapis/google-cloud-cpp/snapshot.txt

@@ -3,7 +3,7 @@ Repository: googleapis/google-cloud-cpp
 Url: "https://github.com/googleapis/google-cloud-cpp.git"
 Args: ["--config=../../../../config/repositories/googleapis/google-cloud-cpp/cspell.json","**/*","-e","{*.BUILD,BUILD,CHANGELOG.md,*.sh,*.cfg,*.ps1,Dockerfile.*,*.Dockerfile,*.{yaml,xml,json,cmake}}"]
 Lines:
- CSpell: Files checked: 2538, Issues found: 14547 in 1732 files
+ CSpell: Files checked: 2538, Issues found: 16150 in 1732 files
  exit code: 1
 ./ARCHITECTURE.md:186:1     - Unknown word (discoverability) -- discoverability of available option
 ./ARCHITECTURE.md:22:61     - Unknown word (retriable)  -- RPCs that fail with a retriable error,

integration-tests/snapshots/graphql/graphql-spec/snapshot.txt

@@ -3,7 +3,7 @@ Repository: graphql/graphql-spec
 Url: "https://github.com/graphql/graphql-spec.git"
 Args: ["**/*.md"]
 Lines:
- CSpell: Files checked: 14, Issues found: 135 in 2 files
+ CSpell: Files checked: 14, Issues found: 143 in 2 files
  exit code: 1
 ./CONTRIBUTING.md:109:62    - Unknown word (strawman)   -- a series of stages: _strawman_,
 ./CONTRIBUTING.md:122:14    - Unknown word (Strawman)   -- ## Stage 0: _Strawman_

integration-tests/snapshots/liriliri/licia/snapshot.txt

@@ -3,7 +3,7 @@ Repository: liriliri/licia
 Url: "https://github.com/liriliri/licia.git"
 Args: ["**/*.*"]
 Lines:
- CSpell: Files checked: 1384, Issues found: 345 in 84 files
+ CSpell: Files checked: 1384, Issues found: 359 in 84 files
  exit code: 1
 ./CHANGELOG.md:240:26    - Unknown word (unenumerable) -- feat(allKeys): support unenumerable and symbol
 ./DOC.md:10155:10  - Unknown word (dasherize)   -- Create a dasherize version.

integration-tests/snapshots/mdx-js/mdx/report.yaml

@@ -5,7 +5,7 @@ Args: ["--config=${repoConfig}/cspell.json","**/*.*"]
 Summary:
   files: 160
   filesWithIssues: 99
-  issues: 1188
+  issues: 1268
   errors: 0
 Errors: []
 

integration-tests/snapshots/mdx-js/mdx/snapshot.txt

@@ -3,7 +3,7 @@ Repository: mdx-js/mdx
 Url: "https://github.com/mdx-js/mdx"
 Args: ["--config=../../../../config/repositories/mdx-js/mdx/cspell.json","**/*.*"]
 Lines:
- CSpell: Files checked: 160, Issues found: 1188 in 99 files
+ CSpell: Files checked: 160, Issues found: 1268 in 99 files
  exit code: 1
 ./docs/_asset/index.client.js:16:9      - Unknown word (nljson)     -- const nljson = document.querySelector
 ./docs/_asset/index.css:286:21    - Unknown word (squircle)           -- mask-image: paint(squircle);

integration-tests/snapshots/microsoft/TypeScript-Website/report.yaml

@@ -5,7 +5,7 @@ Args: ["--config=${repoConfig}/cspell.json","**/*.*"]
 Summary:
   files: 757
   filesWithIssues: 179
-  issues: 660
+  issues: 682
   errors: 0
 Errors: []
 

integration-tests/snapshots/microsoft/TypeScript-Website/snapshot.txt

@@ -3,7 +3,7 @@ Repository: microsoft/TypeScript-Website
 Url: "https://github.com/microsoft/TypeScript-Website.git"
 Args: ["--config=../../../../config/repositories/microsoft/TypeScript-Website/cspell.json","**/*.*"]
 Lines:
- CSpell: Files checked: 757, Issues found: 660 in 179 files
+ CSpell: Files checked: 757, Issues found: 682 in 179 files
  exit code: 1
 ./docs/Deprecating a page.md:3:109     - Unknown word (Dont)       -- deleting because [Cool URIs Dont Change](https://www
 ./docs/New TypeScript Version.md:135:3     - Unknown word (Shippit)    -- # Shippit

integration-tests/snapshots/pagekit/pagekit/report.yaml

@@ -5,7 +5,7 @@ Args: ["-e","/app/assets/codemirror","-e","/app/system/languages","**"]
 Summary:
   files: 742
   filesWithIssues: 225
-  issues: 699
+  issues: 734
   errors: 0
 Errors: []
 

integration-tests/snapshots/pagekit/pagekit/snapshot.txt

@@ -3,7 +3,7 @@ Repository: pagekit/pagekit
 Url: "https://github.com/pagekit/pagekit.git"
 Args: ["-e","/app/assets/codemirror","-e","/app/system/languages","**"]
 Lines:
- CSpell: Files checked: 742, Issues found: 699 in 225 files
+ CSpell: Files checked: 742, Issues found: 734 in 225 files
  exit code: 1
 ./CHANGELOG.md:102:50    - Unknown word (Onur)         -- page, discovered by Onur Yilmaz (https://www
 ./CHANGELOG.md:102:55    - Unknown word (Yilmaz)       -- discovered by Onur Yilmaz (https://www.netsparker

integration-tests/snapshots/php/php-src/report.yaml

@@ -5,7 +5,7 @@ Args: ["--config=${repoConfig}/cspell.json","**/*.{md,c,h,php}"]
 Summary:
   files: 1828
   filesWithIssues: 1089
-  issues: 19771
+  issues: 21956
   errors: 0
 Errors: []
 

integration-tests/snapshots/php/php-src/snapshot.txt

@@ -3,7 +3,7 @@ Repository: php/php-src
 Url: "https://github.com/php/php-src.git"
 Args: ["--config=../../../../config/repositories/php/php-src/cspell.json","**/*.{md,c,h,php}"]
 Lines:
- CSpell: Files checked: 1828, Issues found: 19771 in 1089 files
+ CSpell: Files checked: 1828, Issues found: 21956 in 1089 files
  exit code: 1
 ./CODING_STANDARDS.md:105:8     - Unknown word (setclientencoding) -- pg_setclientencoding
 ./CODING_STANDARDS.md:126:5     - Unknown word (fooselect)         -- fooselect_bar

integration-tests/snapshots/pycontribs/jira/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.*"]
 Summary:
   files: 75
   filesWithIssues: 34
-  issues: 439
+  issues: 464
   errors: 0
 Errors: []
 

integration-tests/snapshots/pycontribs/jira/snapshot.txt

@@ -3,7 +3,7 @@ Repository: pycontribs/jira
 Url: "https://github.com/pycontribs/jira.git"
 Args: ["**/*.*"]
 Lines:
- CSpell: Files checked: 75, Issues found: 439 in 34 files
+ CSpell: Files checked: 75, Issues found: 464 in 34 files
  exit code: 1
 ./AUTHORS.rst:3:21      - Unknown word (Contribs)   -- Development Team (PyContribs)
 ./AUTHORS.rst:5:7       - Unknown word (Speakmon)   -- - Ben Speakmon <ben.speakmon@gmail

integration-tests/snapshots/typescript-cheatsheets/react/report.yaml

@@ -5,7 +5,7 @@ Args: ["**/*.{ts,js,md}"]
 Summary:
   files: 50
   filesWithIssues: 42
-  issues: 523
+  issues: 539
   errors: 0
 Errors: []
 

integration-tests/snapshots/typescript-cheatsheets/react/snapshot.txt

@@ -3,7 +3,7 @@ Repository: typescript-cheatsheets/react
 Url: "https://github.com/typescript-cheatsheets/react.git"
 Args: ["**/*.{ts,js,md}"]
 Lines:
- CSpell: Files checked: 50, Issues found: 523 in 42 files
+ CSpell: Files checked: 50, Issues found: 539 in 42 files
  exit code: 1
 ./CONTRIBUTING.md:20:39     - Unknown word (swyx's)     -- deployed to Netlify on swyx's personal account.
 ./CONTRIBUTING.md:22:12     - Unknown word (docsite)    -- To run the docsite locally:

packages/cspell-lib/src/textValidator.test.ts

@@ -11,9 +11,7 @@ import { createSpellingDictionary } from './SpellingDictionary/createSpellingDic
 import { FreqCounter } from './util/FreqCounter';
 import * as Text from './util/text';
 import { genSequence } from 'gensequence';
-import { settingsToValidateOptions } from './validator';
-
-const sToV = settingsToValidateOptions;
+import { settingsToValidateOptions as sToV } from './validator';
 
 // cspell:ignore whiteberry redmango lightbrown redberry
 
@@ -195,6 +193,20 @@ describe('Validate textValidator functions', () => {
         expect(words.join(' ')).toBe('Test the line breaks from begin to end eol');
     });
 
+    test('make sure maxDuplicateProblems is honored', () => {
+        const dictWords = getSpellingDictionaryCollectionSync();
+        // cspell:ignore unword
+        const text = 'unword '.repeat(20);
+
+        const resultDef = [...validateText(text, dictWords, sToV({}))];
+        expect(resultDef).toHaveLength(5);
+
+        const result10 = [
+            ...validateText(text, dictWords, sToV({ allowCompoundWords: false, maxDuplicateProblems: 10 })),
+        ];
+        expect(result10).toHaveLength(10);
+    });
+
     test('tests words crossing exclude boundaries out of order', async () => {
         const text = '_Test the _line_breaks___from __begin to end__ _eol_';
         const inclusionRanges = calcTextInclusionRanges(text, { ignoreRegExpList: [/_/g] });

packages/cspell-lib/src/textValidator.ts

@@ -66,7 +66,7 @@ export function validateText(
             const n = (mapOfProblems.get(word) || 0) + 1;
             mapOfProblems.set(word, n);
             // Filter out if there is too many
-            return n < maxDuplicateProblems;
+            return n <= maxDuplicateProblems;
         })
         .take(maxNumberOfProblems);
 }

test-fixtures/issues/issue-2231/cspell.yml

@@ -0,0 +1,2 @@
+version: "0.2"
+maxDuplicateProblems: 5

test-fixtures/issues/issue-2231/sample.txt

@@ -0,0 +1,9 @@
+
+/api/cp/accounts/personal/paymentqistory
+/api/x/accounts/personal/paymentqistory
+
+/api/c/accounts/personal/paymentqistory
+/api/x/accounts/personal/paymentqistory
+
+/api/cp/accounts/personal/paymentqistory more
+/api/x/accounts/personal/paymentqistory