From 8ed855b317ade9e13a21f219b72cf8a25b2d36d1 Mon Sep 17 00:00:00 2001 From: efebarlas Date: Sat, 13 Nov 2021 18:06:42 +0000 Subject: [PATCH] option regExp to specify RegExp engine (e.g. re2) #1684 commit 1835f3517ffb750ea4c75ce3ee8d9c262374e8f4 Author: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat Nov 13 18:04:08 2021 +0000 simplify regExp option commit e7f1eb9b4ae4dea95ceb493e2e3f389f80d4d005 Merge: 98f04d37 f68ef8f7 Author: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat Nov 13 17:20:15 2021 +0000 Merge branch 'master' into master commit 98f04d3742e0697483b7149938ca38504210ee9b Author: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sat Nov 13 17:20:04 2021 +0000 Update docs/options.md commit 0ff99ed3e4cc1b9bee33ae97646c7d07b336cc8f Merge: d9ea90c9 8fccddb4 Author: Efe Barlas <43009963+efebarlas@users.noreply.github.com> Date: Wed Nov 10 00:15:33 2021 -0500 Merge branch 'master' into master commit d9ea90c9049a6945913d5b3a862d301aa41cf27a Author: efebarlas Date: Wed Nov 10 00:09:17 2021 -0500 prettier:write to pass CI Signed-off-by: efebarlas commit b29cd91ae8f9b7758abafd4ee17b1cbbf3b7fb84 Merge: f50eb438 20089ed4 Author: efebarlas Date: Tue Nov 9 21:54:45 2021 -0500 Merge branch 'master' of github.com:efebarlas/ajv Tests added for code.regExp option commit f50eb4380b4586e9341cf314bfe36f1c3f1dd7bb Author: efebarlas Date: Tue Nov 9 21:54:28 2021 -0500 Tests added Signed-off-by: efebarlas commit 20089ed47eaa37a41da333fcff9415e54bdd8b4d Author: Efe Barlas <43009963+efebarlas@users.noreply.github.com> Date: Tue Nov 9 21:53:34 2021 -0500 Update options.md commit fd3e290f43cead6720151d4d1ec7749c7c67a46a Merge: 41dd4bc8 6ef0c66e Author: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun Sep 12 19:07:28 2021 +0100 Merge branch 'master' into master commit 41dd4bc8629ad1538ecc16967aec50b138cbd336 Merge: 698f4111 a9f38cd2 Author: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> Date: Sun Sep 12 11:35:20 2021 +0100 Merge branch 'master' into master commit 698f41117d655abc8b870caf0f0f13aba28f61c7 Author: Efe Barlas Date: Thu Aug 12 14:55:17 2021 -0400 dev-dependency to node-re2 added commit a0720f881b8331db1c8c38a805e24a71f5daacbb Author: Efe Barlas Date: Thu Aug 12 14:43:39 2021 -0400 re2 runtime lib + regExp code option added commit 1470c23f94021f8a7433c3b9599a466ee8ebd53d Author: Efe Barlas Date: Fri Jul 9 14:14:45 2021 -0400 variable name changes Signed-off-by: Efe Barlas commit 8f7ca34e549950b6be3d1454373d258a7f9bc561 Author: Efe Barlas Date: Fri Jul 9 13:22:38 2021 -0400 minor changes Signed-off-by: Efe Barlas commit 9791cce0d3f28b75cafa898b8725214b86a78b41 Author: Efe Barlas Date: Fri Jul 9 13:20:47 2021 -0400 remove comments Signed-off-by: Efe Barlas commit b07542d081e5d3b411a576071a670746f2789e99 Author: Efe Barlas Date: Fri Jul 9 11:28:29 2021 -0400 added: RE2 Option with fallback Signed-off-by: Efe Barlas Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> --- docs/options.md | 7 ++++++ lib/core.ts | 10 ++++++++- lib/runtime/re2.ts | 6 ++++++ lib/types/index.ts | 9 ++++++++ lib/vocabularies/code.ts | 10 ++++++--- package.json | 1 + spec/issues/1683_re2_engine.spec.ts | 33 +++++++++++++++++++++++++++++ spec/issues/re2.ts | 4 ++++ 8 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 lib/runtime/re2.ts create mode 100644 spec/issues/1683_re2_engine.spec.ts create mode 100644 spec/issues/re2.ts diff --git a/docs/options.md b/docs/options.md index 4330d4959..ed0db7b02 100644 --- a/docs/options.md +++ b/docs/options.md @@ -69,6 +69,7 @@ const defaultOptions = { source: false, process: undefined, // (code: string) => string optimize: true, + regExp: RegExp }, } ``` @@ -361,6 +362,12 @@ type CodeOptions = { // Code snippet created with `_` tagged template literal that contains all format definitions, // it can be the code of actual definitions or `require` call: // _`require("./my-formats")` + regExp: RegExpEngine + // Pass non-standard RegExp engine to mitigate ReDoS, e.g. node-re2. + // During validation of a schema, code.regExp will be + // used to match strings against regular expressions. + // The supplied function must support the interface: + // regExp(regex, unicodeFlag).test(string) => boolean } type Source = { diff --git a/lib/core.ts b/lib/core.ts index 7d3079c62..2cf93c62c 100644 --- a/lib/core.ts +++ b/lib/core.ts @@ -48,6 +48,7 @@ import type { ErrorObject, Format, AddedFormat, + RegExpEngine, } from "./types" import type {JSONSchemaType} from "./types/json-schema" import type {JTDSchemaType, SomeJTDSchemaType, JTDDataType} from "./types/jtd-schema" @@ -62,6 +63,9 @@ import {eachItem} from "./compile/util" import * as $dataRefSchema from "./refs/data.json" +const defaultRegExp: RegExpEngine = (str, flags) => new RegExp(str, flags) +defaultRegExp.code = "new RegExp" + const META_IGNORE_OPTIONS: (keyof Options)[] = ["removeAdditional", "useDefaults", "coerceTypes"] const EXT_SCOPE_NAMES = new Set([ "validate", @@ -141,9 +145,11 @@ export interface CodeOptions { formats?: Code // code to require (or construct) map of available formats - for standalone code source?: boolean process?: (code: string, schema?: SchemaEnv) => string + regExp?: RegExpEngine } interface InstanceCodeOptions extends CodeOptions { + regExp: RegExpEngine optimize: number } @@ -231,13 +237,14 @@ function requiredOptions(o: Options): RequiredInstanceOptions { const s = o.strict const _optz = o.code?.optimize const optimize = _optz === true || _optz === undefined ? 1 : _optz || 0 + const regExp = o.code?.regExp ?? defaultRegExp return { strictSchema: o.strictSchema ?? s ?? true, strictNumbers: o.strictNumbers ?? s ?? true, strictTypes: o.strictTypes ?? s ?? "log", strictTuples: o.strictTuples ?? s ?? "log", strictRequired: o.strictRequired ?? s ?? false, - code: o.code ? {...o.code, optimize} : {optimize}, + code: o.code ? {...o.code, optimize, regExp} : {optimize, regExp}, loopRequired: o.loopRequired ?? MAX_EXPRESSION, loopEnum: o.loopEnum ?? MAX_EXPRESSION, meta: o.meta ?? true, @@ -279,6 +286,7 @@ export default class Ajv { constructor(opts: Options = {}) { opts = this.opts = {...opts, ...requiredOptions(opts)} const {es5, lines} = this.opts.code + this.scope = new ValueScope({scope: {}, prefixes: EXT_SCOPE_NAMES, es5, lines}) this.logger = getLogger(opts.logger) const formatOpt = opts.validateFormats diff --git a/lib/runtime/re2.ts b/lib/runtime/re2.ts new file mode 100644 index 000000000..0c769bc7a --- /dev/null +++ b/lib/runtime/re2.ts @@ -0,0 +1,6 @@ +import * as re2 from "re2" + +type Re2 = typeof re2 & {code: string} +;(re2 as Re2).code = 'require("ajv/dist/runtime/re2").default' + +export default re2 as Re2 diff --git a/lib/types/index.ts b/lib/types/index.ts index 509b9df98..03a06e642 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -222,3 +222,12 @@ export type AddedFormat = | AsyncFormatDefinition export type Format = AddedFormat | string + +export interface RegExpEngine { + (pattern: string, u: string): RegExpLike + code: string +} + +export interface RegExpLike { + test: (s: string) => boolean +} diff --git a/lib/vocabularies/code.ts b/lib/vocabularies/code.ts index ca1cf5f0a..93d522926 100644 --- a/lib/vocabularies/code.ts +++ b/lib/vocabularies/code.ts @@ -4,7 +4,7 @@ import type {KeywordCxt} from "../compile/validate" import {CodeGen, _, and, or, not, nil, strConcat, getProperty, Code, Name} from "../compile/codegen" import {alwaysValidSchema, Type} from "../compile/util" import N from "../compile/names" - +import {useFunc} from "../compile/util" export function checkReportMissingProp(cxt: KeywordCxt, prop: string): void { const {gen, data, it} = cxt gen.if(noPropertyInData(gen, data, prop, it.opts.ownProperties), () => { @@ -90,12 +90,16 @@ export function callValidateCode( return context !== nil ? _`${func}.call(${context}, ${args})` : _`${func}(${args})` } +const newRegExp = _`new RegExp` + export function usePattern({gen, it: {opts}}: KeywordCxt, pattern: string): Name { const u = opts.unicodeRegExp ? "u" : "" + const {regExp} = opts.code + return gen.scopeValue("pattern", { key: pattern, - ref: new RegExp(pattern, u), - code: _`new RegExp(${pattern}, ${u})`, + ref: regExp(pattern, u), + code: _`${regExp.code === "new RegExp" ? newRegExp : useFunc(gen, regExp)}(${pattern}, ${u})`, }) } diff --git a/package.json b/package.json index 5c90c3533..f04319813 100644 --- a/package.json +++ b/package.json @@ -97,6 +97,7 @@ "node-fetch": "^3.0.0", "nyc": "^15.0.0", "prettier": "^2.3.1", + "re2": "^1.16.0", "rollup": "^2.44.0", "rollup-plugin-terser": "^7.0.2", "ts-node": "^10.0.0", diff --git a/spec/issues/1683_re2_engine.spec.ts b/spec/issues/1683_re2_engine.spec.ts new file mode 100644 index 000000000..475a0a541 --- /dev/null +++ b/spec/issues/1683_re2_engine.spec.ts @@ -0,0 +1,33 @@ +import getAjvAllInstances from "../ajv_all_instances" +import {withStandalone} from "../ajv_standalone" +import {_} from "../../dist/compile/codegen/code" +import jsonSchemaTest = require("json-schema-test") +import options from "../ajv_options" +import {afterError, afterEach} from "../after_test" +import chai from "../chai" +import re2 from "../../dist/runtime/re2" +import re2tests from "./re2" + +const instances = getAjvAllInstances(options, { + $data: true, + formats: {allowedUnknown: true}, + strictTypes: false, + strictTuples: false, +}) + +instances.forEach((ajv) => { + ajv.opts.code.source = true + ajv.opts.code.formats = _`{allowedUnknown: true}` + ajv.opts.code.regExp = re2 +}) + +jsonSchemaTest(withStandalone(instances), { + description: "Test with re2 RegExp engine with " + instances.length + " ajv instances", + suites: {"regular expressions": re2tests}, + assert: chai.assert, + afterError, + afterEach, + cwd: __dirname, + hideFolder: "extras/", + timeout: 90000, +}) diff --git a/spec/issues/re2.ts b/spec/issues/re2.ts new file mode 100644 index 000000000..8bb9f944c --- /dev/null +++ b/spec/issues/re2.ts @@ -0,0 +1,4 @@ +export default [ + {name: "$data/format", test: require("../extras/$data/format.json")}, + {name: "$data/pattern", test: require("../extras/$data/pattern.json")}, +]