fix: cannot use underscore as indentifier (#372)

closes #371

packages/message-compiler/src/tokenizer.ts

@@ -178,8 +178,9 @@ export function createTokenizer(
     const cc = ch.charCodeAt(0)
     return (
       (cc >= 97 && cc <= 122) || // a-z
-      (cc >= 65 && cc <= 90)
-    ) // A-Z
+      (cc >= 65 && cc <= 90) || // A-Z
+      cc === 95 // _
+    )
   }
 
   function isNumberStart(ch: string): boolean {
@@ -399,9 +400,9 @@ export function createTokenizer(
         (cc >= 97 && cc <= 122) || // a-z
         (cc >= 65 && cc <= 90) || // A-Z
         (cc >= 48 && cc <= 57) || // 0-9
-        cc === 95 ||
-        cc === 36
-      ) // _ $
+        cc === 95 || // _
+        cc === 36 // $
+      )
     }
     return takeChar(scnr, closure)
   }

packages/message-compiler/test/__snapshots__/compiler.test.ts.snap

@@ -311,6 +311,15 @@ exports[`edge cases %: code 1`] = `
 }"
 `;
 
+exports[`edge cases {_field} with the same value already exists.: code 1`] = `
+"function __msg__ (ctx) {
+  const { normalize: _normalize, interpolate: _interpolate, named: _named } = ctx
+  return _normalize([
+    _interpolate(_named(\\"_field\\")), \\" with the same value already exists.\\"
+  ])
+}"
+`;
+
 exports[`edge cases hi %s !: code 1`] = `
 "function __msg__ (ctx) {
   const { normalize: _normalize } = ctx
@@ -320,6 +329,15 @@ exports[`edge cases hi %s !: code 1`] = `
 }"
 `;
 
+exports[`edge cases hi @._upper:{_name} !: code 1`] = `
+"function __msg__ (ctx) {
+  const { normalize: _normalize, interpolate: _interpolate, named: _named, linked: _linked } = ctx
+  return _normalize([
+    \\"hi \\", _linked(_interpolate(_named(\\"_name\\")), \\"_upper\\"), \\" !\\"
+  ])
+}"
+`;
+
 exports[`edge cases no apples %| one apple % |  too much apples  : code 1`] = `
 "function __msg__ (ctx) {
   const { normalize: _normalize, plural: _plural } = ctx

packages/message-compiler/test/__snapshots__/parser.test.ts.snap

@@ -247,6 +247,135 @@ Array [
 ]
 `;
 
+exports[`parse: foo 1`] = `
+Object {
+  "body": Object {
+    "end": 21,
+    "items": Array [
+      Object {
+        "end": 3,
+        "loc": Object {
+          "end": Object {
+            "column": 4,
+            "line": 1,
+            "offset": 3,
+          },
+          "start": Object {
+            "column": 1,
+            "line": 1,
+            "offset": 0,
+          },
+        },
+        "start": 0,
+        "type": 3,
+        "value": "hi ",
+      },
+      Object {
+        "end": 19,
+        "key": Object {
+          "end": 19,
+          "key": "_name",
+          "loc": Object {
+            "end": Object {
+              "column": 20,
+              "line": 1,
+              "offset": 19,
+            },
+            "start": Object {
+              "column": 13,
+              "line": 1,
+              "offset": 12,
+            },
+          },
+          "start": 12,
+          "type": 4,
+        },
+        "loc": Object {
+          "end": Object {
+            "column": 20,
+            "line": 1,
+            "offset": 19,
+          },
+          "start": Object {
+            "column": 4,
+            "line": 1,
+            "offset": 3,
+          },
+        },
+        "modifier": Object {
+          "end": 11,
+          "loc": Object {
+            "end": Object {
+              "column": 12,
+              "line": 1,
+              "offset": 11,
+            },
+            "start": Object {
+              "column": 5,
+              "line": 1,
+              "offset": 4,
+            },
+          },
+          "start": 4,
+          "type": 8,
+          "value": "_upper",
+        },
+        "start": 3,
+        "type": 6,
+      },
+      Object {
+        "end": 21,
+        "loc": Object {
+          "end": Object {
+            "column": 22,
+            "line": 1,
+            "offset": 21,
+          },
+          "start": Object {
+            "column": 20,
+            "line": 1,
+            "offset": 19,
+          },
+        },
+        "start": 19,
+        "type": 3,
+        "value": " !",
+      },
+    ],
+    "loc": Object {
+      "end": Object {
+        "column": 22,
+        "line": 1,
+        "offset": 21,
+      },
+      "start": Object {
+        "column": 1,
+        "line": 1,
+        "offset": 0,
+      },
+    },
+    "start": 0,
+    "type": 2,
+  },
+  "end": 21,
+  "loc": Object {
+    "end": Object {
+      "column": 22,
+      "line": 1,
+      "offset": 21,
+    },
+    "source": "hi @._upper:{_name} !",
+    "start": Object {
+      "column": 1,
+      "line": 1,
+      "offset": 0,
+    },
+  },
+  "start": 0,
+  "type": 0,
+}
+`;
+
 exports[`parse: linked 1`] = `
 Object {
   "body": Object {
@@ -1285,6 +1414,100 @@ Object {
 }
 `;
 
+exports[`parse: named 2`] = `
+Object {
+  "body": Object {
+    "end": 15,
+    "items": Array [
+      Object {
+        "end": 6,
+        "loc": Object {
+          "end": Object {
+            "column": 7,
+            "line": 1,
+            "offset": 6,
+          },
+          "start": Object {
+            "column": 1,
+            "line": 1,
+            "offset": 0,
+          },
+        },
+        "start": 0,
+        "type": 3,
+        "value": "hello ",
+      },
+      Object {
+        "end": 13,
+        "key": "_name",
+        "loc": Object {
+          "end": Object {
+            "column": 14,
+            "line": 1,
+            "offset": 13,
+          },
+          "start": Object {
+            "column": 7,
+            "line": 1,
+            "offset": 6,
+          },
+        },
+        "start": 6,
+        "type": 4,
+      },
+      Object {
+        "end": 15,
+        "loc": Object {
+          "end": Object {
+            "column": 16,
+            "line": 1,
+            "offset": 15,
+          },
+          "start": Object {
+            "column": 14,
+            "line": 1,
+            "offset": 13,
+          },
+        },
+        "start": 13,
+        "type": 3,
+        "value": " !",
+      },
+    ],
+    "loc": Object {
+      "end": Object {
+        "column": 16,
+        "line": 1,
+        "offset": 15,
+      },
+      "start": Object {
+        "column": 1,
+        "line": 1,
+        "offset": 0,
+      },
+    },
+    "start": 0,
+    "type": 2,
+  },
+  "end": 15,
+  "loc": Object {
+    "end": Object {
+      "column": 16,
+      "line": 1,
+      "offset": 15,
+    },
+    "source": "hello {_name} !",
+    "start": Object {
+      "column": 1,
+      "line": 1,
+      "offset": 0,
+    },
+  },
+  "start": 0,
+  "type": 0,
+}
+`;
+
 exports[`parse: plural 1`] = `
 Object {
   "body": Object {

packages/message-compiler/test/compiler.test.ts

@@ -95,4 +95,14 @@ describe('edge cases', () => {
     const { code } = compile(`no apples %| one apple % |  too much apples  `)
     expect(code).toMatchSnapshot('code')
   })
+
+  test(`{_field} with the same value already exists.`, () => {
+    const { code } = compile(`{_field} with the same value already exists.`)
+    expect(code).toMatchSnapshot('code')
+  })
+
+  test(`hi @._upper:{_name} !`, () => {
+    const { code } = compile(`hi @._upper:{_name} !`)
+    expect(code).toMatchSnapshot('code')
+  })
 })

packages/message-compiler/test/parser.test.ts

@@ -5,6 +5,7 @@ test('parse', () => {
   ;[
     { code: 'hello world', name: 'message' },
     { code: 'hello {name} !', name: 'named' },
+    { code: 'hello {_name} !', name: 'named' },
     { code: 'hello {0} !', name: 'list' },
     { code: `hello {'kazupon'} !`, name: 'literal' },
     { code: '@:apples', name: 'linked' },
@@ -17,6 +18,7 @@ test('parse', () => {
     { code: `@.: a`, name: 'linked error' },
     { code: `@.:foo`, name: 'linked modifier error' },
     { code: `@:(foo)`, name: 'linked key paren error' },
+    { code: `hi @._upper:{_name} !`, name: 'foo' },
     { code: `@.lower:(foo)`, name: 'linked key paren error with modifier' },
     { code: `@.`, name: 'EOF in linked modifier' },
     { code: `|`, name: 'empty plural' }