diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js
index 779fabd7b1f5..a0757f347a36 100644
--- a/packages/babel-parser/src/tokenizer/index.js
+++ b/packages/babel-parser/src/tokenizer/index.js
@@ -51,7 +51,7 @@ const VALID_REGEX_FLAGS = new Set([
 // an immediate sibling of NumericLiteralSeparator _
 
 const forbiddenNumericSeparatorSiblings = {
-  decBinOct: [
+  decBinOct: new Set([
     charCodes.dot,
     charCodes.uppercaseB,
     charCodes.uppercaseE,
@@ -60,22 +60,22 @@ const forbiddenNumericSeparatorSiblings = {
     charCodes.lowercaseB,
     charCodes.lowercaseE,
     charCodes.lowercaseO,
-  ],
-  hex: [
+  ]),
+  hex: new Set([
     charCodes.dot,
     charCodes.uppercaseX,
     charCodes.underscore, // multiple separators are not allowed
     charCodes.lowercaseX,
-  ],
+  ]),
 };
 
 const allowedNumericSeparatorSiblings = {};
-allowedNumericSeparatorSiblings.bin = [
+allowedNumericSeparatorSiblings.bin = new Set([
   // 0 - 1
   charCodes.digit0,
   charCodes.digit1,
-];
-allowedNumericSeparatorSiblings.oct = [
+]);
+allowedNumericSeparatorSiblings.oct = new Set([
   // 0 - 7
   ...allowedNumericSeparatorSiblings.bin,
 
@@ -85,16 +85,16 @@ allowedNumericSeparatorSiblings.oct = [
   charCodes.digit5,
   charCodes.digit6,
   charCodes.digit7,
-];
-allowedNumericSeparatorSiblings.dec = [
+]);
+allowedNumericSeparatorSiblings.dec = new Set([
   // 0 - 9
   ...allowedNumericSeparatorSiblings.oct,
 
   charCodes.digit8,
   charCodes.digit9,
-];
+]);
 
-allowedNumericSeparatorSiblings.hex = [
+allowedNumericSeparatorSiblings.hex = new Set([
   // 0 - 9, A - F, a - f,
   ...allowedNumericSeparatorSiblings.dec,
 
@@ -111,7 +111,7 @@ allowedNumericSeparatorSiblings.hex = [
   charCodes.lowercaseD,
   charCodes.lowercaseE,
   charCodes.lowercaseF,
-];
+]);
 
 // Object type used to represent tokens. Note that normally, tokens
 // simply exist as properties on the parser object. This is only
@@ -1199,26 +1199,22 @@ export default class Tokenizer extends CommentsParser {
       if (code === charCodes.underscore && allowNumSeparator !== "bail") {
         const prev = this.input.charCodeAt(this.state.pos - 1);
         const next = this.input.charCodeAt(this.state.pos + 1);
-        if (allowedSiblings.indexOf(next) === -1) {
-          this.raise(Errors.UnexpectedNumericSeparator, {
+
+        if (!allowNumSeparator) {
+          this.raise(Errors.NumericSeparatorInEscapeSequence, {
             at: this.state.curPosition(),
           });
         } else if (
-          forbiddenSiblings.indexOf(prev) > -1 ||
-          forbiddenSiblings.indexOf(next) > -1 ||
-          Number.isNaN(next)
+          Number.isNaN(next) ||
+          !allowedSiblings.has(next) ||
+          forbiddenSiblings.has(prev) ||
+          forbiddenSiblings.has(next)
         ) {
           this.raise(Errors.UnexpectedNumericSeparator, {
             at: this.state.curPosition(),
           });
         }
 
-        if (!allowNumSeparator) {
-          this.raise(Errors.NumericSeparatorInEscapeSequence, {
-            at: this.state.curPosition(),
-          });
-        }
-
         // Ignore this _ character
         ++this.state.pos;
         continue;
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/input.js
new file mode 100644
index 000000000000..789dc5505d9c
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/input.js
@@ -0,0 +1 @@
+\u{_0061};
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/output.json
new file mode 100644
index 000000000000..d7a3c492cfb0
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-3/output.json
@@ -0,0 +1,25 @@
+{
+  "type": "File",
+  "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+  "errors": [
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:3)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":9,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":9,"index":9},"identifierName":"a"},
+          "name": "a"
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/input.js
new file mode 100644
index 000000000000..af2b9f9aabf8
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/input.js
@@ -0,0 +1 @@
+\u{00__61};
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/output.json
new file mode 100644
index 000000000000..4b8d534991bd
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-4/output.json
@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":11,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":11,"index":11}},
+  "errors": [
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:5)",
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:6)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":11,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":11,"index":11}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":11,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":11,"index":11}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10},"identifierName":"a"},
+          "name": "a"
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/input.js
new file mode 100644
index 000000000000..80d668780753
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/input.js
@@ -0,0 +1 @@
+\u_0061;
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/output.json
new file mode 100644
index 000000000000..c3e15937dddf
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-5/output.json
@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+  "errors": [
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:2)",
+    "SyntaxError: Invalid Unicode escape. (1:0)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":7,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":7,"index":7},"identifierName":"\u00061"},
+          "name": "\u00061"
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/input.js
new file mode 100644
index 000000000000..06a4a77603ac
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/input.js
@@ -0,0 +1 @@
+\u00__61;
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/output.json
new file mode 100644
index 000000000000..332f9837f4fb
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-6/output.json
@@ -0,0 +1,27 @@
+{
+  "type": "File",
+  "start":0,"end":9,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":9,"index":9}},
+  "errors": [
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:4)",
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:5)",
+    "SyntaxError: Invalid Unicode escape. (1:0)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":9,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":9,"index":9}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":9,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":9,"index":9}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8},"identifierName":"\u000061"},
+          "name": "\u000061"
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/input.js
new file mode 100644
index 000000000000..69085c281941
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/input.js
@@ -0,0 +1 @@
+\u{0061_};
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/output.json
new file mode 100644
index 000000000000..f699d513cc76
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/invalid-unicode-7/output.json
@@ -0,0 +1,25 @@
+{
+  "type": "File",
+  "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+  "errors": [
+    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (1:7)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":10,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":10,"index":10}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":9,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":9,"index":9},"identifierName":"a"},
+          "name": "a"
+        }
+      }
+    ],
+    "directives": []
+  }
+}
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/input.js b/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/input.js
new file mode 100644
index 000000000000..df81ab2ab4cb
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/input.js
@@ -0,0 +1 @@
+\u0061_;
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/output.json b/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/output.json
new file mode 100644
index 000000000000..7c43e69e9047
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2021/numeric-separator/unicode-followed-by-_/output.json
@@ -0,0 +1,22 @@
+{
+  "type": "File",
+  "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":8,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":8,"index":8}},
+        "expression": {
+          "type": "Identifier",
+          "start":0,"end":7,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":7,"index":7},"identifierName":"a_"},
+          "name": "a_"
+        }
+      }
+    ],
+    "directives": []
+  }
+}