feature: Support using suggestion costs to improve suggestions (#2281)

Dictionary Definitions now support Suggestion Maps. See: [cspell/rfc/rfc-0002 improve dictionary suggestions · streetsidesoftware/cspell](https://github.com/streetsidesoftware/cspell/tree/main/rfc/rfc-0002%20improve%20dictionary%20suggestions) * dev: Work toward supporting weighted suggestions. * dev: sort scripts * dev: Add some comments to types. * refactor: refactor Hunspell reader * Refactor distance weights Add walker for WeightMap * test: be able to use Hunspell reader in tests * Add WeightMaps to suggestion options. * Have suggestion collector sort using weights.
streetsidesoftware · Jan 19, 2022 · 879a0c8 · 879a0c8
1 parent f219824
commit 879a0c8
Show file tree

Hide file tree

Showing 56 changed files with 295,140 additions and 612 deletions.
diff --git a/cspell-dict.txt b/cspell-dict.txt
@@ -25,6 +25,7 @@ popd
 prebuilds
 pushd
 pycontribs
+résumé
 serializers
 specberus
 tsbuildinfo

diff --git a/cspell.json b/cspell.json
@@ -55,5 +55,8 @@
     "ignoreWords": [
         "commitcomment"
     ],
-    "words": []
+    "words": [],
+    "features": {
+        "weighted-suggestions": true
+    }
 }
diff --git a/cspell.schema.json b/cspell.schema.json
@@ -177,7 +177,7 @@
         },
         "suggestionEditCosts": {
           "$ref": "#/definitions/SuggestionCostsDefs",
-          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list."
+          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list.\n\nAdded with `v5.16.0`."
         },
         "type": {
           "$ref": "#/definitions/DictionaryFileTypes",
@@ -239,7 +239,7 @@
         },
         "suggestionEditCosts": {
           "$ref": "#/definitions/SuggestionCostsDefs",
-          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list."
+          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list.\n\nAdded with `v5.16.0`."
         },
         "type": {
           "$ref": "#/definitions/DictionaryFileTypes",
@@ -283,7 +283,7 @@
         },
         "suggestionEditCosts": {
           "$ref": "#/definitions/SuggestionCostsDefs",
-          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list."
+          "description": "Used in making suggestions. The lower the value, the more likely the suggestion will be near the top of the suggestion list.\n\nAdded with `v5.16.0`."
         },
         "type": {
           "$ref": "#/definitions/DictionaryFileTypes",
@@ -344,6 +344,20 @@
       "$ref": "#/definitions/FsPath",
       "description": "A File System Path.\n\nSpecial Properties:\n- `${cwd}` prefix - will be replaced with the current working directory.\n- Relative paths are relative to the configuration file."
     },
+    "FeatureEnableOnly": {
+      "type": "boolean"
+    },
+    "Features": {
+      "additionalProperties": false,
+      "description": "Features are behaviors or settings that can be explicitly configured.",
+      "properties": {
+        "weighted-suggestions": {
+          "$ref": "#/definitions/FeatureEnableOnly",
+          "description": "Enable/disable using weighted suggestions."
+        }
+      },
+      "type": "object"
+    },
     "FsPath": {
       "description": "A File System Path. Relative paths are relative to the configuration file.",
       "type": "string"
@@ -950,6 +964,10 @@
       },
       "type": "array"
     },
+    "features": {
+      "$ref": "#/definitions/Features",
+      "description": "Configure CSpell features.\n\n- Added with `v5.16.0`."
+    },
     "files": {
       "description": "Glob patterns of files to be checked. Glob patterns are relative to the `globRoot` of the configuration file that defines them.",
       "items": {

diff --git a/package.json b/package.json
@@ -9,27 +9,29 @@
   },
   "private": true,
   "scripts": {
-    "build": "lerna run build && lerna run prepare-cspell-action",
+    "bootstrap": "lerna bootstrap --no-ci",
+    "bt": "npm run build && npm run test",
     "build-dev": "lerna run build-dev",
-    "build-schema": "lerna run build-schema",
     "build-integration-tests": "cd ./integration-tests && npm i",
+    "build-schema": "lerna run build-schema",
+    "build": "lerna run build && lerna run prepare-cspell-action",
     "check-spelling": "npx cspell -c cspellrc.json",
     "clean-build": "npm run clean && lerna run clean-build",
     "clean": "rimraf \"{*,*/*}/{.tsbuildinfo,temp,coverage}\" lcov.info && lerna run clean",
-    "coverage": "lerna run coverage --stream && npm run coverage-collect",
     "coverage-collect": "globcat packages/*/coverage/lcov.info > lcov.info",
-    "install": "lerna bootstrap --ci",
+    "coverage": "lerna run coverage --stream && npm run coverage-collect",
+    "eslint-fix": "eslint . --fix --cache",
     "ibt": "npm install && npm run build && npm test",
-    "bootstrap": "lerna bootstrap --no-ci",
-    "lint": "eslint . --fix --cache && prettier -w \"**/*.{md,yaml,yml,json}\"",
+    "install": "lerna bootstrap --ci",
     "lint-ci": "eslint . && prettier -c \"**/*.{md,yaml,yml,json}\"",
-    "lint-docs": "prettier -w \"docs/**/*.{md,markdown,yaml,yml,json,html,htm}\"",
     "lint-docs-ci": "prettier -c \"docs/**/*.{md,markdown,yaml,yml,json,html,htm}\"",
+    "lint-docs": "prettier -w \"docs/**/*.{md,markdown,yaml,yml,json,html,htm}\"",
+    "lint": "npm run eslint-fix && prettier -w \"**/*.{md,yaml,yml,json}\"",
     "prepare-cspell-action": "npm i && lerna run prepare-cspell-action",
     "pub-lerna": "lerna publish from-git",
-    "pub-version": "lerna version --conventional-commits",
-    "pub-version-pre": "lerna version --conventional-commits --conventional-prerelease",
     "pub-version-grad": "lerna version --conventional-commits --conventional-graduate",
+    "pub-version-pre": "lerna version --conventional-commits --conventional-prerelease",
+    "pub-version": "lerna version --conventional-commits",
     "pub": "npm run ibt && lerna publish --conventional-commits",
     "pub-grad": "npm run ibt && lerna publish --conventional-commits --conventional-graduate",
     "pub-next": "npm run ibt && lerna publish --conventional-commits --conventional-prerelease --dist-tag next",

diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.test.ts
@@ -188,6 +188,18 @@ describe('Validate CSpellSettingsServer', () => {
         expect(setting4).toBe(setting3);
     });
 
+    test.each`
+        left                                              | right                                              | expected
+        ${{}}                                             | ${{}}                                              | ${{}}
+        ${{ dictionaries: ['a'] }}                        | ${{ dictionaries: ['b'] }}                         | ${oc({ dictionaries: ['a', 'b'] })}
+        ${{ features: {} }}                               | ${{}}                                              | ${oc({ features: {} })}
+        ${{ features: { 'weighted-suggestions': true } }} | ${{}}                                              | ${oc({ features: { 'weighted-suggestions': true } })}
+        ${{ features: { 'weighted-suggestions': true } }} | ${{ features: { 'weighted-suggestions': false } }} | ${oc({ features: { 'weighted-suggestions': false } })}
+        ${{ features: { 'weighted-suggestions': true } }} | ${{ features: { 'new-feature': true } }}           | ${oc({ features: { 'weighted-suggestions': true, 'new-feature': true } })}
+    `('mergeSettings $left with $right', ({ left, right, expected }) => {
+        expect(mergeSettings(left, right)).toEqual(expected);
+    });
+
     test.each`
         filename                                              | relativeTo   | refFilename
         ${r('../../cspell.config.json')}                      | ${undefined} | ${r('../../cspell.config.json')}

diff --git a/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts b/packages/cspell-lib/src/Settings/CSpellSettingsServer.ts
@@ -389,6 +389,16 @@ function mergeList<T>(left: T[] | undefined, right: T[] | undefined): T[] | unde
     return left.concat(right);
 }
 
+function mergeObjects(left: undefined, right: undefined): undefined;
+function mergeObjects<T>(left: T, right: undefined): T;
+function mergeObjects<T>(left: T, right: T): T;
+function mergeObjects<T>(left: undefined, right: T): T;
+function mergeObjects<T>(left?: T, right?: T): T | undefined {
+    if (left === undefined) return right;
+    if (right === undefined) return left;
+    return { ...left, ...right };
+}
+
 function tagLanguageSettings(tag: string, settings: LanguageSetting[] = []): LanguageSetting[] {
     return settings.map((s) => ({
         id: tag + '.' + (s.id || s.locale || s.languageId),
@@ -464,6 +474,7 @@ function merge(left: CSpellSettings, right: CSpellSettings): CSpellSettings {
         files: mergeListUnique(left.files, right.files),
         ignorePaths: versionBasedMergeList(left.ignorePaths, right.ignorePaths, version),
         overrides: versionBasedMergeList(left.overrides, right.overrides, version),
+        features: mergeObjects(left.features, right.features),
         source: mergeSources(left, right),
         globRoot: undefined,
         import: undefined,
@@ -927,6 +938,7 @@ function validateRawConfig(config: CSpellUserSettings, fileRef: ImportFileRef):
 }
 
 export const __testing__ = {
+    mergeObjects,
     normalizeCacheSettings,
     normalizeSettings,
     validateRawConfigExports,

diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionary.ts
@@ -1,4 +1,4 @@
-import type { ReplaceMap } from '@cspell/cspell-types';
+import type { ReplaceMap, SuggestionCostsDefs } from '@cspell/cspell-types';
 import { CompoundWordsMethod, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib';
 
 export { CompoundWordsMethod, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib';
@@ -57,6 +57,7 @@ export interface SpellingDictionaryOptions {
     useCompounds?: boolean;
     caseSensitive?: boolean;
     noSuggest?: boolean;
+    suggestionDefs?: SuggestionCostsDefs | undefined;
 }
 
 export interface SpellingDictionary {

diff --git a/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts b/packages/cspell-lib/src/SpellingDictionary/SpellingDictionaryFromTrie.ts
@@ -1,5 +1,11 @@
-import type { FindFullResult, FindWordOptions, SuggestionCollector, SuggestionResult } from 'cspell-trie-lib';
-import { CompoundWordsMethod, importTrie, suggestionCollector, Trie } from 'cspell-trie-lib';
+import type {
+    FindFullResult,
+    FindWordOptions,
+    SuggestionCollector,
+    SuggestionResult,
+    WeightMap,
+} from 'cspell-trie-lib';
+import { CompoundWordsMethod, importTrie, suggestionCollector, Trie, createWeightedMap } from 'cspell-trie-lib';
 import { getDefaultSettings } from '../Settings';
 import { memorizer } from '../util/Memorizer';
 import { createMapper } from '../util/repMap';
@@ -29,6 +35,8 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
     readonly isDictionaryCaseSensitive: boolean;
     readonly containsNoSuggestWords: boolean;
 
+    private weightMap: WeightMap | undefined;
+
     constructor(
         readonly trie: Trie,
         readonly name: string,
@@ -40,6 +48,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
         this.isDictionaryCaseSensitive = options.caseSensitive ?? !trie.isLegacy;
         this.containsNoSuggestWords = options.noSuggest || false;
         this._size = size || 0;
+        this.weightMap = options.suggestionDefs ? createWeightedMap(options.suggestionDefs) : undefined;
     }
 
     public get size(): number {
@@ -158,6 +167,7 @@ export class SpellingDictionaryFromTrie implements SpellingDictionary {
             includeTies,
             ignoreCase,
             timeout,
+            weightMap: this.weightMap,
         });
         this.genSuggestions(collector, suggestOptions);
         return collector.suggestions.map((r) => ({ ...r, word: r.word }));

diff --git a/packages/cspell-tools/src/compiler/Reader.ts b/packages/cspell-tools/src/compiler/Reader.ts
@@ -10,7 +10,7 @@ import {
     CASE_INSENSITIVE_PREFIX,
 } from 'cspell-trie-lib';
 import * as zlib from 'zlib';
-import { AffWord } from 'hunspell-reader/dist/aff';
+import { AffWord } from 'hunspell-reader';
 
 const regHunspellFile = /\.(dic|aff)$/i;
 

diff --git a/packages/cspell-trie-lib/package-lock.json b/packages/cspell-trie-lib/package-lock.json
diff --git a/packages/cspell-trie-lib/package.json b/packages/cspell-trie-lib/package.json
@@ -48,6 +48,7 @@
     "@cspell/dict-es-es": "^2.1.0",
     "@types/fs-extra": "^9.0.13",
     "@types/node": "^17.0.9",
+    "hunspell-reader": "^5.15.2",
     "jest": "^27.4.7",
     "rimraf": "^3.0.2"
   }