add ESM support (take 2) (#1649)

* Revert "temporarily revert ESM change (#1647)"

This reverts commit 084c1f2.

* add failing scenario for deep imports

* define entry point with dot

* make deep imports work via export patterns

* move doc to own file

* link to doc from readme

* add changelog entry

* add example to doc

* remove confusing comment

* remove cli option, use import by default

* update documentation

* remove redundant describe

* fix ordering

* Update features/esm.feature

Co-authored-by: Aurélien Reeves <aurelien.reeves@smartbear.com>

* Update features/esm.feature

Co-authored-by: Aurélien Reeves <aurelien.reeves@smartbear.com>

* simplify tagging

* use import only if a javascript file

* add note about no transpilers

* inline to avoid confusing reassignment

* whoops, re-add try/catch

* use require with transpilers; import otherwise

* remove pointless return

* support .cjs config file

* type and import the importer

* actually dont import - causes issues

Co-authored-by: Aurélien Reeves <aurelien.reeves@smartbear.com>

CHANGELOG.md

@@ -19,6 +19,7 @@ See the [migration guide](./docs/migration.md) for details of how to migrate fro
 
 ### Added
 
+* Add support for user code as native ES modules
 * `BeforeStep` and `AfterStep` hook functions now have access to the `pickleStep` in their argument object.
 * `--config` option to the CLI. It allows you to specify a configuration file other than `cucumber.js`.
   See [docs/profiles.md](./docs/profiles.md#using-another-file-than-cucumberjs) for more info.

README.md

@@ -67,6 +67,7 @@ The following documentation is for master. See below the documentation for older
   * [Attachments](/docs/support_files/attachments.md)
   * [API Reference](/docs/support_files/api_reference.md)
 * Guides
+  * [ES Modules](./docs/esm.md)
   * [Running in parallel](./docs/parallel.md)
   * [Retrying failing scenarios](./docs/retry.md)
   * [Profiles](./docs/profiles.md)

dependency-lint.yml

@@ -44,6 +44,7 @@ requiredModules:
       - 'dist/**/*'
       - 'lib/**/*'
       - 'node_modules/**/*'
+      - 'src/importer.js'
       - 'tmp/**/*'
     root: '**/*.{js,ts}'
   stripLoaders: false

docs/esm.md

@@ -0,0 +1,34 @@
+# ES Modules (experimental)
+
+You can optionally write your support code (steps, hooks, etc) with native ES modules syntax - i.e. using `import` and `export` statements without transpiling. This is enabled without any additional configuration, and you can use either of the `.js` or `.mjs` file extensions.
+
+Example (adapted from [our original example](./nodejs_example.md)):
+
+```javascript
+// features/support/steps.mjs
+import { Given, When, Then } from '@cucumber/cucumber'
+import { strict as assert } from 'assert'
+
+Given('a variable set to {int}', function (number) {
+  this.setTo(number)
+})
+
+When('I increment the variable by {int}', function (number) {
+  this.incrementBy(number)
+})
+
+Then('the variable should contain {int}', function (number) {
+  assert.equal(this.variable, number)
+})
+```
+
+As well as support code, these things can also be in ES modules syntax:
+
+- Custom formatters
+- Custom snippets
+
+You can use ES modules selectively/incrementally - so you can have a mixture of CommonJS and ESM in the same project.
+
+When using a transpiler for e.g. TypeScript, ESM isn't supported - you'll need to configure your transpiler to output modules in CommonJS syntax (for now).
+
+The config file referenced for [Profiles](./profiles.md) can only be in CommonJS syntax. In a project with `type=module`, you can name the file `cucumber.cjs`, since Node expects `.js` files to be in ESM syntax in such projects.

features/direct_imports.feature

@@ -44,3 +44,20 @@ Feature: Core feature elements execution using direct imports
       """
       features/step_definitions/cucumber_steps.js:3
       """
+
+  Scenario: deep imports don't break everything
+    Given a file named "features/a.feature" with:
+      """
+      Feature: some feature
+        Scenario: some scenario
+          Given a step passes
+      """
+    And a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      const {Given} = require('@cucumber/cucumber')
+      const TestCaseHookDefinition = require('@cucumber/cucumber/lib/models/test_case_hook_definition')
+
+      Given(/^a step passes$/, function() {});
+      """
+    When I run cucumber-js
+    Then it passes

features/esm.feature

@@ -0,0 +1,63 @@
+@esm
+Feature: ES modules support
+
+  cucumber-js works with native ES modules
+
+  Scenario Outline: native module syntax works in support code, formatters and snippets
+    Given a file named "features/a.feature" with:
+      """
+      Feature:
+        Scenario: one
+          Given a step passes
+
+        Scenario: two
+          Given a step passes
+      """
+    And a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      import {Given} from '@cucumber/cucumber'
+
+      Given(/^a step passes$/, function() {});
+      """
+    And a file named "custom-formatter.js" with:
+      """
+      import {SummaryFormatter} from '@cucumber/cucumber'
+
+      export default class CustomFormatter extends SummaryFormatter {}
+      """
+    And a file named "custom-snippet-syntax.js" with:
+      """
+      export default class CustomSnippetSyntax {
+          build(opts) {
+              return 'hello world'
+          }
+      }
+      """
+    And a file named "cucumber.cjs" with:
+      """
+      module.exports = {
+        'default': '--format summary'
+      }
+      """
+    When I run cucumber-js with `<options> --format ./custom-formatter.js --format-options '{"snippetSyntax": "./custom-snippet-syntax.js"}' <args>`
+    Then it passes
+    Examples:
+      | args         |
+      |              |
+      | --parallel 2 |
+
+  Scenario: .mjs support code files are matched by default
+    Given a file named "features/a.feature" with:
+      """
+      Feature:
+        Scenario:
+          Given a step passes
+      """
+    And a file named "features/step_definitions/cucumber_steps.mjs" with:
+      """
+      import {Given} from '@cucumber/cucumber'
+
+      Given(/^a step passes$/, function() {});
+      """
+    When I run cucumber-js
+    Then it passes

features/profiles.feature

@@ -91,3 +91,7 @@ Feature: default command line arguments
       | OPT      |
       | -c       |
       | --config |
+
+  Scenario: specifying a configuration file that doesn't exist
+    When I run cucumber-js with `--config doesntexist.js`
+    Then it fails

features/support/hooks.ts

@@ -13,7 +13,7 @@ Before('@debug', function (this: World) {
   this.debug = true
 })
 
-Before('@spawn', function (this: World) {
+Before('@spawn or @esm', function (this: World) {
   this.spawn = true
 })
 
@@ -43,6 +43,13 @@ Before(function (
   this.localExecutablePath = path.join(projectPath, 'bin', 'cucumber-js')
 })
 
+Before('@esm', function (this: World) {
+  fsExtra.writeJSONSync(path.join(this.tmpDir, 'package.json'), {
+    name: 'feature-test-pickle',
+    type: 'module',
+  })
+})
+
 Before('@global-install', function (this: World) {
   const tmpObject = tmp.dirSync({ unsafeCleanup: true })
 

package.json

@@ -163,6 +163,15 @@
     "lib": "./lib"
   },
   "main": "./lib/index.js",
+  "exports": {
+    ".": {
+      "import": "./lib/wrapper.mjs",
+      "require": "./lib/index.js"
+    },
+    "./lib/*": {
+      "require": "./lib/*.js"
+    }
+  },
   "types": "./lib/index.d.ts",
   "engines": {
     "node": ">=12"
@@ -180,7 +189,6 @@
     "cli-table3": "^0.6.0",
     "colors": "^1.4.0",
     "commander": "^8.0.0",
-    "create-require": "^1.1.1",
     "duration": "^0.2.2",
     "durations": "^3.4.2",
     "figures": "^3.2.0",
@@ -252,7 +260,7 @@
     "typescript": "4.4.2"
   },
   "scripts": {
-    "build-local": "tsc --build tsconfig.node.json",
+    "build-local": "tsc --build tsconfig.node.json && cp src/importer.js lib/ && cp src/wrapper.mjs lib/",
     "cck-test": "mocha 'compatibility/**/*_spec.ts'",
     "feature-test": "node ./bin/cucumber-js",
     "html-formatter": "node ./bin/cucumber-js --profile htmlFormatter",

src/cli/argv_parser.ts

@@ -107,11 +107,7 @@ const ArgvParser = {
       .usage('[options] [<GLOB|DIR|FILE[:LINE]>...]')
       .version(version, '-v, --version')
       .option('-b, --backtrace', 'show full backtrace for errors')
-      .option(
-        '-c, --config <TYPE[:PATH]>',
-        'specify configuration file',
-        'cucumber.js'
-      )
+      .option('-c, --config <TYPE[:PATH]>', 'specify configuration file')
       .option(
         '-d, --dry-run',
         'invoke formatters without executing steps',

src/cli/configuration_builder.ts

@@ -76,7 +76,7 @@ export default class ConfigurationBuilder {
       }
       supportCodePaths = await this.expandPaths(
         unexpandedSupportCodePaths,
-        '.js'
+        '.@(js|mjs)'
       )
     }
     return {

src/cli/configuration_builder_spec.ts

@@ -71,8 +71,10 @@ describe('Configuration', () => {
       const relativeFeaturePath = path.join('features', 'a.feature')
       const featurePath = path.join(cwd, relativeFeaturePath)
       await fsExtra.outputFile(featurePath, '')
-      const supportCodePath = path.join(cwd, 'features', 'a.js')
-      await fsExtra.outputFile(supportCodePath, '')
+      const jsSupportCodePath = path.join(cwd, 'features', 'a.js')
+      await fsExtra.outputFile(jsSupportCodePath, '')
+      const esmSupportCodePath = path.join(cwd, 'features', 'a.mjs')
+      await fsExtra.outputFile(esmSupportCodePath, '')
       const argv = baseArgv.concat([relativeFeaturePath])
 
       // Act
@@ -82,7 +84,7 @@ describe('Configuration', () => {
       // Assert
       expect(featurePaths).to.eql([featurePath])
       expect(pickleFilterOptions.featurePaths).to.eql([relativeFeaturePath])
-      expect(supportCodePaths).to.eql([supportCodePath])
+      expect(supportCodePaths).to.eql([jsSupportCodePath, esmSupportCodePath])
     })
 
     it('deduplicates the .feature files before returning', async function () {

src/cli/index.ts

@@ -29,7 +29,10 @@ import { IParsedArgvFormatOptions } from './argv_parser'
 import HttpStream from '../formatter/http_stream'
 import { promisify } from 'util'
 import { Writable } from 'stream'
+import { pathToFileURL } from 'url'
 
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { importer } = require('../importer')
 const { incrementing, uuid } = IdGenerator
 
 export interface ICliRunResult {
@@ -143,29 +146,28 @@ export default class Cli {
           )
           type = 'progress'
         }
-        return FormatterBuilder.build(type, typeOptions)
+        return await FormatterBuilder.build(type, typeOptions)
       })
     )
     return async function () {
       await Promise.all(formatters.map(async (f) => await f.finished()))
     }
   }
 
-  getSupportCodeLibrary({
+  async getSupportCodeLibrary({
     newId,
     supportCodeRequiredModules,
     supportCodePaths,
-  }: IGetSupportCodeLibraryRequest): ISupportCodeLibrary {
+  }: IGetSupportCodeLibraryRequest): Promise<ISupportCodeLibrary> {
     supportCodeRequiredModules.map((module) => require(module))
     supportCodeLibraryBuilder.reset(this.cwd, newId)
-    supportCodePaths.forEach((codePath) => {
-      try {
+    for (const codePath of supportCodePaths) {
+      if (supportCodeRequiredModules.length) {
         require(codePath)
-      } catch (e) {
-        console.error(e.stack)
-        console.error('codepath: ' + codePath)
+      } else {
+        await importer(pathToFileURL(codePath))
       }
-    })
+    }
     return supportCodeLibraryBuilder.finalize()
   }
 
@@ -184,7 +186,7 @@ export default class Cli {
       configuration.predictableIds && configuration.parallel <= 1
         ? incrementing()
         : uuid()
-    const supportCodeLibrary = this.getSupportCodeLibrary({
+    const supportCodeLibrary = await this.getSupportCodeLibrary({
       newId,
       supportCodePaths: configuration.supportCodePaths,
       supportCodeRequiredModules: configuration.supportCodeRequiredModules,

src/cli/profile_loader.ts

@@ -3,24 +3,31 @@ import path from 'path'
 import stringArgv from 'string-argv'
 import { doesHaveValue, doesNotHaveValue } from '../value_checker'
 
-export default class ProfileLoader {
-  private readonly directory: string
+const DEFAULT_FILENAMES = ['cucumber.cjs', 'cucumber.js']
 
-  constructor(directory: string) {
-    this.directory = directory
-  }
+export default class ProfileLoader {
+  constructor(private readonly directory: string) {}
 
   async getDefinitions(configFile?: string): Promise<Record<string, string>> {
-    const definitionsFilePath: string = path.join(
-      this.directory,
-      configFile || 'cucumber.js'
+    if (configFile) {
+      return this.loadFile(configFile)
+    }
+
+    const defaultFile = DEFAULT_FILENAMES.find((filename) =>
+      fs.existsSync(path.join(this.directory, filename))
     )
 
-    const exists = await fs.exists(definitionsFilePath)
-    if (!exists) {
-      return {}
+    if (defaultFile) {
+      return this.loadFile(defaultFile)
     }
-    const definitions = require(definitionsFilePath) // eslint-disable-line @typescript-eslint/no-var-requires
+
+    return {}
+  }
+
+  loadFile(configFile: string): Record<string, string> {
+    const definitionsFilePath: string = path.join(this.directory, configFile)
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const definitions = require(definitionsFilePath)
     if (typeof definitions !== 'object') {
       throw new Error(`${definitionsFilePath} does not export an object`)
     }