feat(cli-lib): adding support for gts, gjs and hbs files.

.swcrc

@@ -27,4 +27,4 @@
     },
     "keepClassNames": false
   }
-}
+}

jest.config.js

@@ -1,7 +1,22 @@
+const config = {
+  jsc: {
+    parser: {
+      syntax: 'typescript',
+      tsx: true,
+      dynamicImport: true,
+      topLevelAwait: true,
+      importMeta: true,
+    },
+    target: 'es2022',
+  },
+}
 module.exports = {
   testEnvironment: 'node',
   transform: {
-    '^.+\\.(t|j)sx?$': '@swc/jest',
+    '^.+\\.m?(t|j)sx?$': ['@swc/jest', config],
+    '^.+\\.m(t|j)s$': ['@swc/jest', config],
   },
   verbose: true,
+  moduleFileExtensions: ['js', 'jsx', 'mjs', 'ts', 'tsx', 'mts'],
+  extensionsToTreatAsEsm: ['.mts'],
 }

package.json

@@ -33,15 +33,19 @@
     "@bazel/ibazel": "0.24",
     "@commitlint/cli": "17",
     "@commitlint/config-angular": "17",
+    "@glimmer/env": "^0.1.7",
+    "@glimmer/reference": "^0.91.1",
+    "@glimmer/syntax": "^0.91.1",
+    "@glimmer/validator": "^0.91.1",
     "@jest/transform": "29",
     "@jest/types": "29",
     "@lerna-lite/cli": "2",
     "@lerna-lite/list": "2",
     "@lerna-lite/version": "2",
     "@napi-rs/cli": "^2.11.0",
-    "@swc-node/register": "^1.5.4",
-    "@swc/core": "^1.3.56",
-    "@swc/jest": "^0.2.23",
+    "@swc-node/register": "^1.9.0",
+    "@swc/core": "^1.4.12",
+    "@swc/jest": "^0.2.36",
     "@taplo/cli": "^0.5.0",
     "@testing-library/jest-dom": "^6.0.0",
     "@testing-library/react": "^14.0.0",
@@ -88,6 +92,7 @@
     "clsx": "2",
     "commander": "8",
     "core-js": "^3.6.5",
+    "ember-template-recast": "^6.1.4",
     "emoji-regex": "^10.2.1",
     "eslint": "7 || 8",
     "fast-glob": "^3.2.7",
@@ -96,7 +101,7 @@
     "http-server": "^14.0.0",
     "husky": "^8.0.0",
     "jasmine-expect": "^5.0.0",
-    "jest": "29",
+    "jest": "^29.7.0",
     "jest-environment-jsdom": "29",
     "json-stable-stringify": "^1.0.1",
     "karma": "^6.0.4",
@@ -118,8 +123,8 @@
     "rimraf": "^3.0.2",
     "serialize-javascript": "^6.0.0",
     "test262-harness": "10",
-    "ts-jest": "29",
-    "ts-loader": "^9.1.2",
+    "ts-jest": "^29.1.2",
+    "ts-loader": "^9.5.1",
     "tsd": "^0.30.0",
     "tslib": "^2.4.0",
     "typescript": "5.2.2",
@@ -140,6 +145,12 @@
     "source-map-js": "1.0.2"
   },
   "pnpm": {
+    "overrides": {
+      "ember-template-recast": "^6.1.4",
+      "@glimmer/reference": "^0.91.1",
+      "@glimmer/syntax": "^0.91.1",
+      "@glimmer/validator": "^0.91.1"
+    },
     "packageExtensions": {
       "ts-jest": {
         "dependencies": {
@@ -172,6 +183,7 @@
   "author": "Seth Bertalotto <sbertal@verizonmedia.com>",
   "dependencies": {
     "@swc/helpers": "^0.5.1",
+    "content-tag": "^2.0.1",
     "jest-cli": "^29.5.0",
     "jest-junit": "^16.0.0",
     "magic-string": "^0.30.0"

packages/cli-lib/BUILD

@@ -23,7 +23,7 @@ npm_package(
     visibility = ["//visibility:public"],
 )
 
-SRCS = glob(["src/**/*.ts"]) + [
+SRCS = glob(["src/**/*.ts", "src/**/*.mts"]) + [
     "index.ts",
     "main.ts",
     # "package.json",
@@ -35,6 +35,16 @@ VUE_DEPS = [
     "//:node_modules/vue",
 ]
 
+GLIMMER_HBS_DEPS = [
+    "//:node_modules/@glimmer/env",
+    "//:node_modules/@glimmer/reference",
+    "//:node_modules/@glimmer/syntax",
+    "//:node_modules/@glimmer/validator",
+    "//:node_modules/content-tag",
+    "//:node_modules/ember-template-recast",
+    ":node_modules/@babel/parser",
+]
+
 SRC_DEPS = [
     "//:node_modules/@types/estree",
     "//:node_modules/@types/fs-extra",
@@ -49,7 +59,7 @@ SRC_DEPS = [
     "//:node_modules/typescript",
     ":node_modules/@formatjs/icu-messageformat-parser",
     ":node_modules/@formatjs/ts-transformer",
-] + VUE_DEPS
+] + VUE_DEPS + GLIMMER_HBS_DEPS
 
 ts_compile_node(
     name = "dist",

packages/cli-lib/main.mts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+
+'use strict'
+
+import cli from './src/cli.js'
+
+cli.default(process.argv)

packages/cli-lib/package.json

@@ -31,14 +31,20 @@
     "url": "https://github.com/formatjs/formatjs/issues"
   },
   "dependencies": {
+    "@babel/parser": "^7.22.10",
     "@formatjs/icu-messageformat-parser": "workspace:*",
     "@formatjs/ts-transformer": "workspace:*",
+    "@glimmer/env": "^0.1.7",
+    "@glimmer/reference": "^0.91.1",
+    "@glimmer/syntax": "^0.91.1",
+    "@glimmer/validator": "^0.91.1",
     "@types/estree": "^1.0.0",
     "@types/fs-extra": "^9.0.1",
     "@types/json-stable-stringify": "^1.0.32",
     "@types/node": "14 || 16 || 17",
     "chalk": "^4.0.0",
     "commander": "8",
+    "ember-template-recast": "^6.1.4",
     "fast-glob": "^3.2.7",
     "fs-extra": "10",
     "json-stable-stringify": "^1.0.1",
@@ -60,5 +66,10 @@
   },
   "engines": {
     "node": ">= 16"
+  },
+  "devDependencies": {
+    "vite": "^5.2.8",
+    "vite-plugin-dts": "^3.8.1",
+    "vitest": "^1.4.0"
   }
 }

packages/cli-lib/src/extract.ts

@@ -158,6 +158,20 @@ async function processFile(
     debug('Processing %s using vue extractor', fn)
     const {parseFile} = await import('./vue_extractor.js')
     parseFile(source, fn, scriptParseFn)
+  } else if (fn.endsWith('.hbs')) {
+    debug('Processing %s using hbs extractor', fn)
+    // SAFETY: The TS config does not understand that this is a
+    //         safe way to import a module in a cjs project
+    // @ts-ignore
+    const {parseFile} = await import('./hbs_extractor.mjs')
+    parseFile(source, fn, opts)
+  } else if (fn.endsWith('.gts') || fn.endsWith('.gjs')) {
+    debug('Processing %s as gts/gjs file', fn)
+    // SAFETY: The TS config does not understand that this is a
+    //         safe way to import a module in a cjs project
+    // @ts-ignore
+    const {parseFile} = await import('./gts_extractor.mjs')
+    parseFile(source, fn, opts)
   } else {
     debug('Processing %s using typescript extractor', fn)
     scriptParseFn(source)
@@ -219,7 +233,7 @@ export async function extract(
       const {id, description, defaultMessage} = message
       if (!id) {
         const error = new Error(
-          `[FormatJS CLI] Missing message id for message: 
+          `[FormatJS CLI] Missing message id for message:
 ${JSON.stringify(message, undefined, 2)}`
         )
         if (throws) {

packages/cli-lib/src/gts_extractor.mts

@@ -0,0 +1,18 @@
+import {parseFile as parseHbsFile} from './hbs_extractor.mjs'
+import {parseScript} from './parse_script.js'
+import {Preprocessor} from 'content-tag'
+let p = new Preprocessor()
+
+export function parseFile(source: string, fileName: string, options: any) {
+  const scriptParseFn = parseScript(options, fileName)
+  const transformedSource = p.process(source, {filename: fileName})
+
+  scriptParseFn(transformedSource)
+
+  // extract template from transformed source to then run through hbs processor
+  const parseResult = p.parse(source, {filename: fileName})
+
+  for (let parsed of parseResult) {
+    parseHbsFile(parsed.contents, fileName, options)
+  }
+}

packages/cli-lib/src/hbs_extractor.mts

@@ -0,0 +1,59 @@
+import {transform} from 'ember-template-recast'
+import type {AST} from '@glimmer/syntax'
+import {Opts} from '@formatjs/ts-transformer'
+
+function extractText(
+  node: AST.MustacheStatement | AST.SubExpression,
+  fileName: string,
+  options: Opts
+) {
+  if (!options.onMsgExtracted) return
+  if (!options.overrideIdFn) return
+
+  if (node.path.type !== 'PathExpression') return
+
+  if (['format-message', 'formatMessage'].includes(node.path.original)) {
+    let [first, second] = node.params
+
+    if (first.type !== 'StringLiteral') return
+
+    let message = first?.value
+
+    let desc: string | undefined
+    if (second?.type === 'StringLiteral') {
+      desc = second.value?.trim().replace(/\s+/gm, ' ')
+    }
+
+    let defaultMessage = message?.trim().replace(/\s+/gm, ' ')
+
+    let id =
+      typeof options.overrideIdFn === 'string'
+        ? options.overrideIdFn
+        : options.overrideIdFn(undefined, defaultMessage, desc, fileName)
+
+    options.onMsgExtracted(fileName, [
+      {
+        id: id,
+        defaultMessage: defaultMessage,
+        description: desc,
+      },
+    ])
+  }
+}
+
+export function parseFile(source: string, fileName: string, options: any) {
+  let visitor = function () {
+    return {
+      MustacheStatement(node: AST.MustacheStatement) {
+        extractText(node, fileName, options)
+      },
+      SubExpression(node: AST.SubExpression) {
+        extractText(node, fileName, options)
+      },
+    }
+  }
+
+  // SAFETY: ember-template-recast's types are out of date,
+  // but it does not affect runtime
+  transform(source, visitor as any)
+}