From 42fdbc2b20c29ca4cfad404a4c222c65077c6f5e Mon Sep 17 00:00:00 2001 From: Rob Hogan Date: Thu, 8 Jun 2023 01:19:20 -0700 Subject: [PATCH] Generate function map via Babel plugin, remove transformer->source map dependency Summary: ## Motivation As part of work to decouple `metro-react-native-babel-transformer` from Metro, and allow more integration with React Native (in particular, for Static ViewConfigs), we'd like to obviate the current dependency of the transformer on `metro-source-map`. This will isolate the transformer and `metro-react-native-babel-preset` so that they're only loosely coupled to Metro via stable interfaces, and frees them up to be moved together into RN. ## Function maps Metro uses "function maps" to augment source maps and provide richer symbolication. They are derived from the source AST (ideally before any mutation) - consequently, they're currently generated between parsing and transformation by *each* transformer via an imperative API, and returned explicitly alongside the transform result. ## This change Invert the dependency on `metro-source-map` by extracting function map metadata in a plugin pass, with a new Babel plugin, rather than with a call to `generateFunctionMap`. We take advantage of the existing API to pass a `plugins` array to the transformer implementation. ### Changes to contracts - Transformer implementations (`transformer.babelTransformerPath`) *may* return a `metadata` object from Babel's transform result metadata. (As an implementation detail, this may contain a `functionMap` property added by the plugin, but the transformer doesn't need to know about this or handle it explicitly.) - Returning a top-level `functionMap` from `transform()`, which was always optional, is now deprecated. It will be temporarily used as a fallback so that this change is non-breaking. ``` * **[Feature]** `metro-babel-transformer` and `metro-react-native-babel-transformer` will return `metadata` from Babel transform results. ``` Reviewed By: motiz88 Differential Revision: D46149279 fbshipit-source-id: e8629be9a90d6b1bec6c2d60d97859d3febaa897 --- flow-typed/babel.js.flow | 61 +++++----- packages/metro-babel-transformer/package.json | 1 - packages/metro-babel-transformer/src/index.js | 38 +++++-- .../metro-babel-transformer/types/index.d.ts | 4 +- .../package.json | 1 - .../src/index.js | 28 ++--- .../src/__tests__/generateFunctionMap-test.js | 39 +++++++ .../src/generateFunctionMap.js | 105 ++++++++++++++---- packages/metro-source-map/src/source-map.js | 6 +- packages/metro-transform-worker/src/index.js | 15 ++- 10 files changed, 212 insertions(+), 86 deletions(-) diff --git a/flow-typed/babel.js.flow b/flow-typed/babel.js.flow index f416e3e070..6bfaf9065f 100644 --- a/flow-typed/babel.js.flow +++ b/flow-typed/babel.js.flow @@ -298,15 +298,14 @@ declare module '@babel/core' { }; declare export type BabelFileMetadata = { - usedHelpers: Array, - - marked: Array<{ + usedHelpers?: Array, + marked?: Array<{ type: string, message: string, loc: BabelNodeSourceLocation, }>, - - modules: BabelFileModulesMetadata, + modules?: BabelFileModulesMetadata, + ... }; declare class Store { @@ -372,9 +371,9 @@ declare module '@babel/core' { addAst(ast: BabelNode): void; - transform(): TransformResult; + transform(): TransformResult<>; - wrap(code: string, callback: () => mixed): TransformResult; + wrap(code: string, callback: () => mixed): TransformResult<>; addCode(code: string): void; @@ -384,9 +383,9 @@ declare module '@babel/core' { parseShebang(): void; - makeResult(TransformResult): TransformResult; + makeResult(TransformResult): TransformResult; - generate(): TransformResult; + generate(): TransformResult<>; } declare export type MatchPattern = @@ -919,8 +918,8 @@ declare module '@babel/core' { moduleRoot?: string, |}; - declare export type TransformResult = {| - metadata: BabelFileMetadata, + declare export type TransformResult = {| + metadata: T, options: BabelCoreOptions, code: string, map: _BabelSourceMap | null, @@ -928,87 +927,87 @@ declare module '@babel/core' { ignored?: boolean, |}; - declare type TransformCallback = + declare type TransformCallback = | ((Error, null) => mixed) - | ((null, TransformResult | null) => mixed); + | ((null, TransformResult | null) => mixed); /** * Transforms the passed in code. Calling a callback with an object with the generated code, source map, and AST. */ - declare export function transform( + declare export function transform( code: string, options: ?BabelCoreOptions, - callback: TransformCallback, + callback: TransformCallback, ): void; /*** * Transforms the passed in code. Returning an object with the generated code, source map, and AST. */ - declare export function transformSync( + declare export function transformSync( code: string, options?: BabelCoreOptions, - ): TransformResult; + ): TransformResult; /** * Transforms the passed in code. Returning an promise for an object with the generated code, source map, and AST. */ - declare export function transformAsync( + declare export function transformAsync( code: string, options?: BabelCoreOptions, - ): Promise; + ): Promise>; /** * Asynchronously transforms the entire contents of a file. */ - declare export function transformFile( + declare export function transformFile( filename: string, options?: BabelCoreOptions, - callback: TransformCallback, + callback: TransformCallback, ): void; /** * Synchronous version of babel.transformFile. Returns the transformed contents of the filename. */ - declare export function transformFileSync( + declare export function transformFileSync( filename: string, options?: BabelCoreOptions, - ): TransformResult; + ): TransformResult; /** * Promise version of babel.transformFile. Returns a promise for the transformed contents of the filename. */ - declare export function transformFileAsync( + declare export function transformFileAsync( filename: string, options?: BabelCoreOptions, - ): Promise; + ): Promise>; /** * Given an AST, transform it. */ - declare export function transformFromAst( + declare export function transformFromAst( ast: BabelNodeFile | BabelNodeProgram, code?: string, options?: BabelCoreOptions, - callback: TransformCallback, + callback: TransformCallback, ): void; /** * Given an AST, transform it. */ - declare export function transformFromAstSync( + declare export function transformFromAstSync( ast: BabelNodeFile | BabelNodeProgram, code?: string, options?: BabelCoreOptions, - ): TransformResult; + ): TransformResult; /** * Given an AST, transform it. */ - declare export function transformFromAstAsync( + declare export function transformFromAstAsync( ast: BabelNodeFile | BabelNodeProgram, code?: string, options?: BabelCoreOptions, - ): Promise; + ): Promise>; /** * Given some code, parse it using Babel's standard behavior. Referenced presets and plugins will be loaded such that optional syntax plugins are automatically enabled. diff --git a/packages/metro-babel-transformer/package.json b/packages/metro-babel-transformer/package.json index c6aadb93f4..3b3598193c 100644 --- a/packages/metro-babel-transformer/package.json +++ b/packages/metro-babel-transformer/package.json @@ -19,7 +19,6 @@ "dependencies": { "@babel/core": "^7.20.0", "hermes-parser": "0.12.0", - "metro-source-map": "0.76.6", "nullthrows": "^1.1.1" }, "engines": { diff --git a/packages/metro-babel-transformer/src/index.js b/packages/metro-babel-transformer/src/index.js index 060f7bb8d9..d7fc50c1fb 100644 --- a/packages/metro-babel-transformer/src/index.js +++ b/packages/metro-babel-transformer/src/index.js @@ -11,11 +11,9 @@ 'use strict'; -import type {BabelCoreOptions} from '@babel/core'; -import type {FBSourceFunctionMap} from 'metro-source-map'; +import type {BabelCoreOptions, BabelFileMetadata} from '@babel/core'; const {parseSync, transformFromAstSync} = require('@babel/core'); -const {generateFunctionMap} = require('metro-source-map'); const nullthrows = require('nullthrows'); export type CustomTransformOptions = { @@ -54,10 +52,28 @@ export type BabelTransformerArgs = $ReadOnly<{ src: string, }>; +export type BabelFileFunctionMapMetadata = $ReadOnly<{ + names: $ReadOnlyArray, + mappings: string, +}>; + +export type MetroBabelFileMetadata = { + ...BabelFileMetadata, + metro?: ?{ + functionMap?: ?BabelFileFunctionMapMetadata, + ... + }, + ... +}; + export type BabelTransformer = { transform: BabelTransformerArgs => { ast: BabelNodeFile, - functionMap: ?FBSourceFunctionMap, + // Deprecated, will be removed in a future breaking release. Function maps + // will be generated by an input Babel plugin instead and written into + // `metadata` - transformers don't need to return them explicitly. + functionMap?: BabelFileFunctionMapMetadata, + metadata?: MetroBabelFileMetadata, ... }, getCacheKey?: () => string, @@ -94,12 +110,16 @@ function transform({filename, options, plugins, src}: BabelTransformerArgs) { }) : parseSync(src, babelConfig); - // Generate the function map before we transform the AST to - // ensure we aren't reading from mutated AST. - const functionMap = generateFunctionMap(sourceAst, {filename}); - const {ast} = transformFromAstSync(sourceAst, src, babelConfig); + const transformResult = transformFromAstSync( + sourceAst, + src, + babelConfig, + ); - return {ast: nullthrows(ast), functionMap}; + return { + ast: nullthrows(transformResult.ast), + metadata: transformResult.metadata, + }; } finally { if (OLD_BABEL_ENV) { process.env.BABEL_ENV = OLD_BABEL_ENV; diff --git a/packages/metro-babel-transformer/types/index.d.ts b/packages/metro-babel-transformer/types/index.d.ts index da365472bf..732af301e4 100644 --- a/packages/metro-babel-transformer/types/index.d.ts +++ b/packages/metro-babel-transformer/types/index.d.ts @@ -8,8 +8,6 @@ * @oncall react_native */ -import type {FBSourceFunctionMap} from 'metro-source-map'; - export interface CustomTransformOptions { [key: string]: unknown; } @@ -46,7 +44,7 @@ export interface BabelTransformerArgs { export interface BabelTransformer { transform: (args: BabelTransformerArgs) => { ast: unknown; - functionMap: FBSourceFunctionMap | null; + metadata: unknown; }; getCacheKey?: () => string; } diff --git a/packages/metro-react-native-babel-transformer/package.json b/packages/metro-react-native-babel-transformer/package.json index d9da3ee430..da8ebdac87 100644 --- a/packages/metro-react-native-babel-transformer/package.json +++ b/packages/metro-react-native-babel-transformer/package.json @@ -22,7 +22,6 @@ "babel-preset-fbjs": "^3.4.0", "hermes-parser": "0.12.0", "metro-react-native-babel-preset": "0.76.6", - "metro-source-map": "0.76.6", "nullthrows": "^1.1.1" }, "peerDependencies": { diff --git a/packages/metro-react-native-babel-transformer/src/index.js b/packages/metro-react-native-babel-transformer/src/index.js index afed881e8d..f7e1b194e9 100644 --- a/packages/metro-react-native-babel-transformer/src/index.js +++ b/packages/metro-react-native-babel-transformer/src/index.js @@ -16,16 +16,14 @@ import type {BabelCoreOptions, Plugins} from '@babel/core'; import type { BabelTransformer, - BabelTransformerArgs, + MetroBabelFileMetadata, } from 'metro-babel-transformer'; -import type {FBSourceFunctionMap} from 'metro-source-map/src/source-map'; const {parseSync, transformFromAstSync} = require('@babel/core'); const inlineRequiresPlugin = require('babel-preset-fbjs/plugins/inline-requires'); const crypto = require('crypto'); const fs = require('fs'); const makeHMRConfig = require('metro-react-native-babel-preset/src/configs/hmr'); -const {generateFunctionMap} = require('metro-source-map'); const nullthrows = require('nullthrows'); const path = require('path'); @@ -186,11 +184,12 @@ function buildBabelConfig( }; } -function transform({filename, options, src, plugins}: BabelTransformerArgs): { - ast: BabelNodeFile, - functionMap: ?FBSourceFunctionMap, - ... -} { +const transform: BabelTransformer['transform'] = ({ + filename, + options, + src, + plugins, +}) => { const OLD_BABEL_ENV = process.env.BABEL_ENV; process.env.BABEL_ENV = options.dev ? 'development' @@ -220,23 +219,26 @@ function transform({filename, options, src, plugins}: BabelTransformerArgs): { sourceType: babelConfig.sourceType, }); - const functionMap = generateFunctionMap(sourceAst, {filename}); - const result = transformFromAstSync(sourceAst, src, babelConfig); + const result = transformFromAstSync( + sourceAst, + src, + babelConfig, + ); // The result from `transformFromAstSync` can be null (if the file is ignored) if (!result) { /* $FlowFixMe BabelTransformer specifies that the `ast` can never be null but * the function returns here. Discovered when typing `BabelNode`. */ - return {ast: null, functionMap}; + return {ast: null}; } - return {ast: nullthrows(result.ast), functionMap}; + return {ast: nullthrows(result.ast), metadata: result.metadata}; } finally { if (OLD_BABEL_ENV) { process.env.BABEL_ENV = OLD_BABEL_ENV; } } -} +}; function getCacheKey() { var key = crypto.createHash('md5'); diff --git a/packages/metro-source-map/src/__tests__/generateFunctionMap-test.js b/packages/metro-source-map/src/__tests__/generateFunctionMap-test.js index 02e6c25fed..5e1b91540c 100644 --- a/packages/metro-source-map/src/__tests__/generateFunctionMap-test.js +++ b/packages/metro-source-map/src/__tests__/generateFunctionMap-test.js @@ -11,9 +11,11 @@ 'use strict'; +import type {MetroBabelFileMetadata} from 'metro-babel-transformer'; import type {Context} from '../generateFunctionMap'; const { + functionMapBabelPlugin, generateFunctionMap, generateFunctionMappingsArray, } = require('../generateFunctionMap'); @@ -1605,6 +1607,43 @@ function parent2() { }); }); + describe('functionMapBabelPlugin', () => { + it('exports a Babel plugin to be used during transformation', () => { + const code = 'export default function foo(bar){}'; + const result = transformFromAstSync( + getAst(code), + code, + { + filename: 'file.js', + cwd: '/my/root', + plugins: [functionMapBabelPlugin], + }, + ); + expect(result.metadata.metro?.functionMap).toEqual({ + mappings: 'AAA,eC', + names: ['', 'foo'], + }); + }); + + it('omits parent class name when it matches filename', () => { + const ast = getAst('class FooBar { baz() {} }'); + expect( + transformFromAstSync(ast, '', { + plugins: [functionMapBabelPlugin], + filename: 'FooBar.ios.js', + }).metadata.metro?.functionMap, + ).toMatchInlineSnapshot(` + Object { + "mappings": "AAA,eC,QD", + "names": Array [ + "FooBar", + "baz", + ], + } + `); + }); + }); + describe('@babel/traverse path cache workaround (babel#6437)', () => { /* These tests exist due to the need to work around a Babel issue: https://github.com/babel/babel/issues/6437 diff --git a/packages/metro-source-map/src/generateFunctionMap.js b/packages/metro-source-map/src/generateFunctionMap.js index cd7385b1e1..9e773fcd07 100644 --- a/packages/metro-source-map/src/generateFunctionMap.js +++ b/packages/metro-source-map/src/generateFunctionMap.js @@ -11,7 +11,9 @@ 'use strict'; +import type {MetroBabelFileMetadata} from 'metro-babel-transformer'; import type {FBSourceFunctionMap} from './source-map'; +import type {PluginObj} from '@babel/core'; import type {NodePath} from '@babel/traverse'; import type {Node} from '@babel/types'; @@ -42,6 +44,7 @@ import { const B64Builder = require('./B64Builder'); const t = require('@babel/types'); +const invariant = require('invariant'); const nullthrows = require('nullthrows'); const fsPath = require('path'); @@ -55,7 +58,21 @@ type RangeMapping = { start: Position, ... }; -export type Context = {filename?: string, ...}; +type FunctionMapVisitor = { + enter: ( + path: + | NodePath + | NodePath + | NodePath, + ) => void, + exit: ( + path: + | NodePath + | NodePath + | NodePath, + ) => void, +}; +export type Context = {filename?: ?string, ...}; /** * Generate a map of source positions to function names. The names are meant to @@ -91,15 +108,52 @@ function generateFunctionMappingsArray( return mappings; } -/** - * Traverses a Babel AST and calls the supplied callback with function name - * mappings, one at a time. - */ -function forEachMapping( - ast: BabelNode, +function functionMapBabelPlugin(): PluginObj<> { + return { + // Eagerly traverse the tree on `pre`, before any visitors have run, so + // that regardless of plugin order we're dealing with the AST before any + // mutations. + visitor: {}, + pre: ({path, metadata, opts}) => { + const {filename} = nullthrows(opts); + const encoder = new MappingEncoder(); + const visitor = getFunctionMapVisitor({filename}, mapping => + encoder.push(mapping), + ); + invariant( + path && t.isProgram(path.node), + 'path missing or not a program node', + ); + // $FlowFixMe[prop-missing] checked above + // $FlowFixMe[incompatible-type-arg] checked above + const programPath: NodePath = path; + + visitor.enter(programPath); + programPath.traverse({ + Function: visitor, + Class: visitor, + }); + visitor.exit(programPath); + + // $FlowFixMe[prop-missing] Babel `File` is not generically typed + const metroMetadata: MetroBabelFileMetadata = metadata; + + const functionMap = encoder.getResult(); + + // Set the result on a metadata property + if (!metroMetadata.metro) { + metroMetadata.metro = {functionMap}; + } else { + metroMetadata.metro.functionMap = functionMap; + } + }, + }; +} + +function getFunctionMapVisitor( context: ?Context, pushMapping: RangeMapping => void, -) { +): FunctionMapVisitor { const nameStack: Array<{loc: BabelNodeSourceLocation, name: string}> = []; let tailPos = {line: 1, column: 0}; let tailName = null; @@ -140,30 +194,31 @@ function forEachMapping( ? fsPath.basename(context.filename).replace(/\..+$/, '') : null; - const visitor = { - enter( - path: - | NodePath - | NodePath - | NodePath, - ) { + return { + enter(path) { let name = getNameForPath(path); if (basename) { name = removeNamePrefix(name, basename); } - pushFrame(name, nullthrows(path.node.loc)); }, - exit( - path: - | NodePath - | NodePath - | NodePath, - ): void { + exit(path) { popFrame(); }, }; +} + +/** + * Traverses a Babel AST and calls the supplied callback with function name + * mappings, one at a time. + */ +function forEachMapping( + ast: BabelNode, + context: ?Context, + pushMapping: RangeMapping => void, +) { + const visitor = getFunctionMapVisitor(context, pushMapping); // Traversing populates/pollutes the path cache (`traverse.cache.path`) with // values missing the `hub` property needed by Babel transformation, so we @@ -525,4 +580,8 @@ function positionGreater(x: Position, y: Position) { return x.line > y.line || (x.line === y.line && x.column > y.column); } -module.exports = {generateFunctionMap, generateFunctionMappingsArray}; +module.exports = { + functionMapBabelPlugin, + generateFunctionMap, + generateFunctionMappingsArray, +}; diff --git a/packages/metro-source-map/src/source-map.js b/packages/metro-source-map/src/source-map.js index a4e5481be0..eb691f02c4 100644 --- a/packages/metro-source-map/src/source-map.js +++ b/packages/metro-source-map/src/source-map.js @@ -19,7 +19,10 @@ const composeSourceMaps = require('./composeSourceMaps'); const Consumer = require('./Consumer'); // We need to export this for `metro-symbolicate` const normalizeSourcePath = require('./Consumer/normalizeSourcePath'); -const {generateFunctionMap} = require('./generateFunctionMap'); +const { + functionMapBabelPlugin, + generateFunctionMap, +} = require('./generateFunctionMap'); const Generator = require('./Generator'); // $FlowFixMe[untyped-import] - source-map const SourceMap = require('source-map'); @@ -338,6 +341,7 @@ module.exports = { generateFunctionMap, fromRawMappings, fromRawMappingsNonBlocking, + functionMapBabelPlugin, normalizeSourcePath, toBabelSegments, toSegmentTuple, diff --git a/packages/metro-transform-worker/src/index.js b/packages/metro-transform-worker/src/index.js index d627c6bd0b..0693731f30 100644 --- a/packages/metro-transform-worker/src/index.js +++ b/packages/metro-transform-worker/src/index.js @@ -11,7 +11,7 @@ 'use strict'; -import type {PluginEntry} from '@babel/core'; +import type {PluginEntry, Plugins} from '@babel/core'; import type { BabelTransformer, BabelTransformerArgs, @@ -39,6 +39,7 @@ const {stableHash} = require('metro-cache'); const getCacheKey = require('metro-cache-key'); const { fromRawMappings, + functionMapBabelPlugin, toBabelSegments, toSegmentTuple, } = require('metro-source-map'); @@ -498,13 +499,18 @@ async function transformJSWithBabel( const transformer: BabelTransformer = require(babelTransformerPath); const transformResult = await transformer.transform( - getBabelTransformArgs(file, context), + // functionMapBabelPlugin populates metadata.metro.functionMap + getBabelTransformArgs(file, context, [functionMapBabelPlugin]), ); const jsFile: JSFile = { ...file, ast: transformResult.ast, - functionMap: transformResult.functionMap ?? null, + functionMap: + transformResult.metadata?.metro?.functionMap ?? + // Fallback to deprecated explicitly-generated `functionMap` + transformResult.functionMap ?? + null, }; return await transformJS(jsFile, context); @@ -563,6 +569,7 @@ async function transformJSON( function getBabelTransformArgs( file: $ReadOnly<{filename: Path, code: string, ...}>, {options, config, projectRoot}: TransformationContext, + plugins?: Plugins = [], ): BabelTransformerArgs { return { filename: file.filename, @@ -580,7 +587,7 @@ function getBabelTransformArgs( projectRoot, publicPath: config.publicPath, }, - plugins: [], + plugins, src: file.code, }; }