diff --git a/README.md b/README.md index bd82eec40..84e6ac4fd 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,39 @@ will treat each file contained within it as an entry point. typedoc package1/index.ts package2/index.ts ``` +### Monorepos / Workspaces + +If your codebase is comprised of one or more npm packages, you can pass the paths to these +packages and TypeDoc will attempt to determine entry points from your `package.json`'s `main` +property (or its default value `index.js`). +If any of the packages given are the root of an [npm Workspace](https://docs.npmjs.com/cli/v7/using-npm/workspaces) +or a [Yarn Workspace](https://classic.yarnpkg.com/en/docs/workspaces/) TypeDoc will find all +the `workpsaces` defined in the `package.json`. +This mode requires sourcemaps in your JS entry points, in order to find the TS entry points. +Supports wildcard paths in the same fashion as those found in npm or Yarn workspaces. + +#### Single npm module + +```text +typedoc --packages . +``` + +#### Monorepo with npm/Yarn workspace at the root + +```text +typedoc --packages . +``` + +#### Monorepo with manually specified sub-packages to document + +This can be useful if you do not want all your workspaces to be processed. +Accepts the same paths as would go in the `package.json`'s workspaces + +```text +# Note the single quotes prevent shell widcard expansion, allowing typedoc to do the expansion +typedoc --packages a-package --packages 'some-more-packages/*' --packages 'some-other-packages/*' +``` + ### Arguments For a complete list of the command line arguments run `typedoc --help` or visit @@ -47,6 +80,9 @@ For a complete list of the command line arguments run `typedoc --help` or visit - `--options`
Specify a json option file that should be loaded. If not specified TypeDoc will look for 'typedoc.json' in the current directory. +- `--packages `
+ Specify one or more sub packages, or the root of a monorepo with workspaces. + Supports wildcard paths in the same fashion as those found in npm or Yarn workspaces. - `--tsconfig `
Specify a typescript config file that should be loaded. If not specified TypeDoc will look for 'tsconfig.json' in the current directory. diff --git a/bin/typedoc b/bin/typedoc index 0e4b09cb5..fd6be79ca 100755 --- a/bin/typedoc +++ b/bin/typedoc @@ -59,8 +59,11 @@ async function run(app) { return ExitCodes.OptionError; } - if (app.options.getValue("entryPoints").length === 0) { - app.logger.error("No entry points provided"); + if ( + app.options.getValue("entryPoints").length === 0 && + app.options.getValue("packages").length === 0 + ) { + app.logger.error("No entry points or packages provided"); return ExitCodes.NoEntryPoints; } diff --git a/examples/basic/.gitignore b/examples/basic/.gitignore new file mode 100644 index 000000000..5c457d797 --- /dev/null +++ b/examples/basic/.gitignore @@ -0,0 +1 @@ +docs \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1dfce6b98..3afb647c6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -435,6 +435,16 @@ "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", "dev": true }, + "@types/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==", + "dev": true, + "requires": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, "@types/json-schema": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.7.tgz", @@ -1266,8 +1276,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "2.3.2", @@ -1304,7 +1313,6 @@ "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1435,7 +1443,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1444,8 +1451,7 @@ "inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "is-binary-path": { "version": "2.1.0", @@ -2119,7 +2125,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1" } @@ -2209,8 +2214,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-key": { "version": "3.1.1", @@ -2829,8 +2833,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "write-file-atomic": { "version": "3.0.3", diff --git a/package.json b/package.json index 2cfaf4fe8..688c912a2 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "node": ">= 12.20.0" }, "dependencies": { + "glob": "^7.1.6", "handlebars": "^4.7.7", "lodash": "^4.17.21", "lunr": "^2.3.9", @@ -33,6 +34,7 @@ "typescript": "4.0.x || 4.1.x || 4.2.x || 4.3.x" }, "devDependencies": { + "@types/glob": "^7.1.3", "@types/lodash": "^4.14.170", "@types/lunr": "^2.3.3", "@types/marked": "^2.0.3", @@ -57,7 +59,7 @@ ], "scripts": { "pretest": "node scripts/copy_test_files.js", - "test": "nyc --reporter=html --reporter=text-summary mocha --timeout=10000 'dist/test/**/*.test.js'", + "test": "nyc --reporter=html --reporter=text-summary mocha --timeout=10000 'dist/test/**/*.test.js' --exclude 'dist/test/packages/**'", "prerebuild_specs": "npm run pretest", "rebuild_specs": "node scripts/rebuild_specs.js", "build": "tsc --project .", diff --git a/scripts/copy_test_files.js b/scripts/copy_test_files.js index 0c6c797b3..62b8fa97d 100644 --- a/scripts/copy_test_files.js +++ b/scripts/copy_test_files.js @@ -2,12 +2,128 @@ const { remove, copy } = require("../dist/lib/utils/fs"); const { join } = require("path"); +const { spawn } = require("child_process"); + +function promiseFromChildProcess(childProcess) { + return new Promise(function (resolve, reject) { + childProcess.on("error", function (error) { + reject( + new Error( + childProcess.spawnargs.join(" ") + " : " + error.message + ) + ); + }); + childProcess.on("exit", function (code) { + if (code !== 0) { + reject( + new Error( + childProcess.spawnargs.join(" ") + + " : exited with code " + + code + ) + ); + } else { + resolve(); + } + }); + }); +} + +const isWindows = process.platform === "win32"; +const npmCommand = isWindows ? "npm.cmd" : "npm"; + +function ensureNpmVersion() { + return Promise.resolve().then(() => { + const npmProc = spawn(npmCommand, ["--version"], { + stdio: ["ignore", "pipe", "inherit"], + }); + let npmVersion = ""; + npmProc.stdout.on("data", (data) => { + npmVersion += data; + }); + return promiseFromChildProcess(npmProc).then(() => { + npmVersion = npmVersion.trim(); + let firstDot = npmVersion.indexOf("."); + const npmMajorVer = parseInt( + npmVersion.slice(0, npmVersion.indexOf(".")) + ); + if (npmMajorVer < 7) { + throw new Error( + "npm version must be at least 7, version installed is " + + npmVersion + ); + } + }); + }); +} + +function prepareMonorepoFolder() { + return Promise.resolve() + .then(() => { + return promiseFromChildProcess( + spawn( + "git", + ["clone", "https://github.com/efokschaner/ts-monorepo.git"], + { + cwd: join(__dirname, "../dist/test/packages"), + stdio: "inherit", + } + ) + ); + }) + .then(() => { + return promiseFromChildProcess( + spawn( + "git", + ["checkout", "73bdd4c6458ad4cc3de35498e65d55a1a44a8499"], + { + cwd: join( + __dirname, + "../dist/test/packages/ts-monorepo" + ), + stdio: "inherit", + } + ) + ); + }) + .then(() => { + return promiseFromChildProcess( + spawn(npmCommand, ["install"], { + cwd: join(__dirname, "../dist/test/packages/ts-monorepo"), + stdio: "inherit", + }) + ); + }) + .then(() => { + return promiseFromChildProcess( + spawn(npmCommand, ["run", "build"], { + cwd: join(__dirname, "../dist/test/packages/ts-monorepo"), + stdio: "inherit", + }) + ); + }); +} + +function prepareSinglePackageExample() { + return Promise.resolve().then(() => { + return promiseFromChildProcess( + spawn(npmCommand, ["run", "build"], { + cwd: join( + __dirname, + "../dist/test/packages/typedoc-single-package-example" + ), + stdio: "inherit", + }) + ); + }); +} const toCopy = [ "test/converter", "test/converter2", "test/renderer", "test/module", + "test/packages", "test/utils/options/readers/data", ]; @@ -18,7 +134,12 @@ const copies = toCopy.map(async (dir) => { await copy(source, target); }); -Promise.all(copies).catch((reason) => { - console.error(reason); - process.exit(1); -}); +Promise.all(copies) + .then(ensureNpmVersion) + .then(() => + Promise.all([prepareMonorepoFolder(), prepareSinglePackageExample()]) + ) + .catch((reason) => { + console.error(reason); + process.exit(1); + }); diff --git a/scripts/rebuild_specs.js b/scripts/rebuild_specs.js index c0b1a0e23..aff5c80fa 100644 --- a/scripts/rebuild_specs.js +++ b/scripts/rebuild_specs.js @@ -69,14 +69,14 @@ function rebuildConverterTests(dirs) { for (const fullPath of dirs) { console.log(fullPath); - const src = app.expandInputFiles([fullPath]); - for (const [file, before, after] of conversions) { const out = path.join(fullPath, `${file}.json`); if (fs.existsSync(out)) { TypeDoc.resetReflectionID(); before(); - const result = app.converter.convert(src, program); + const result = app.converter.convert( + app.getEntrypointsForPaths([fullPath]) + ); const serialized = app.serializer.toObject(result); const data = JSON.stringify(serialized, null, " ") @@ -105,7 +105,7 @@ async function rebuildRendererTest() { externalPattern: ["**/node_modules/**"], }); - app.options.setValue("entryPoints", app.expandInputFiles([src])); + app.options.setValue("entryPoints", [src]); const project = app.convert(); await app.generateDocs(project, out); await app.generateJson(project, path.join(out, "specs.json")); diff --git a/src/lib/application.ts b/src/lib/application.ts index d7f5ab4af..bb4b8f3b6 100644 --- a/src/lib/application.ts +++ b/src/lib/application.ts @@ -2,10 +2,11 @@ import * as Path from "path"; import * as FS from "fs"; import * as ts from "typescript"; -import { Converter } from "./converter/index"; +import { Converter, DocumentationEntryPoint } from "./converter/index"; import { Renderer } from "./output/renderer"; import { Serializer } from "./serialization"; import { ProjectReflection } from "./models/index"; +import { getCommonDirectory } from "./utils/fs"; import { Logger, ConsoleLogger, @@ -27,7 +28,13 @@ import { import { Options, BindOption } from "./utils"; import { TypeDocOptions } from "./utils/options/declaration"; import { flatMap } from "./utils/array"; -import { basename } from "path"; +import { basename, resolve } from "path"; +import { + expandPackages, + getTsEntryPointForPackage, + ignorePackage, + loadPackageManifest, +} from "./utils/package-manifest"; // eslint-disable-next-line @typescript-eslint/no-var-requires const packageInfo = require("../../package.json") as { @@ -39,6 +46,100 @@ const supportedVersionMajorMinor = packageInfo.peerDependencies.typescript .split("||") .map((version) => version.replace(/^\s*|\.x\s*$/g, "")); +/** + * Expand the provided packages configuration paths, determining the entry points + * and creating the ts.Programs for any which are found. + * @param logger + * @param packageGlobPaths + * @returns The information about the discovered programs, undefined if an error occurs. + */ +function getEntryPointsForPackages( + logger: Logger, + packageGlobPaths: string[] +): DocumentationEntryPoint[] | undefined { + const results = new Array(); + // --packages arguments are workspace tree roots, or glob patterns + // This expands them to leave only leaf packages + const expandedPackages = expandPackages(logger, ".", packageGlobPaths); + for (const packagePath of expandedPackages) { + const packageJsonPath = resolve(packagePath, "package.json"); + const packageJson = loadPackageManifest(logger, packageJsonPath); + if (packageJson === undefined) { + logger.error(`Could not load package manifest ${packageJsonPath}`); + return; + } + const packageEntryPoint = getTsEntryPointForPackage( + logger, + packageJsonPath, + packageJson + ); + if (packageEntryPoint === undefined) { + logger.error( + `Could not determine TS entry point for package ${packageJsonPath}` + ); + return; + } + if (packageEntryPoint === ignorePackage) { + continue; + } + const tsconfigFile = ts.findConfigFile( + packageEntryPoint, + ts.sys.fileExists + ); + if (tsconfigFile === undefined) { + logger.error( + `Could not determine tsconfig.json for source file ${packageEntryPoint} (it must be on an ancestor path)` + ); + return; + } + // Consider deduplicating this with similar code in src/lib/utils/options/readers/tsconfig.ts + let fatalError = false; + const parsedCommandLine = ts.getParsedCommandLineOfConfigFile( + tsconfigFile, + {}, + { + ...ts.sys, + onUnRecoverableConfigFileDiagnostic: (error) => { + logger.diagnostic(error); + fatalError = true; + }, + } + ); + if (!parsedCommandLine) { + return; + } + logger.diagnostics(parsedCommandLine.errors); + if (fatalError) { + return; + } + const program = ts.createProgram({ + rootNames: parsedCommandLine.fileNames, + options: parsedCommandLine.options, + }); + const sourceFile = program.getSourceFile(packageEntryPoint); + if (sourceFile === undefined) { + logger.error( + `Entry point "${packageEntryPoint}" does not appear to be built by the tsconfig found at "${tsconfigFile}"` + ); + return; + } + results.push({ + displayName: packageJson.name as string, + path: packageEntryPoint, + program, + sourceFile, + }); + } + return results; +} + +function getModuleName(fileName: string, baseDir: string) { + return normalizePath(Path.relative(baseDir, fileName)).replace( + /(\/index)?(\.d)?\.[tj]sx?$/, + "" + ); +} + /** * The default TypeDoc main application class. * @@ -181,7 +282,6 @@ export class Application extends ChildableComponent< /** * Run the converter for the given set of files and return the generated reflections. * - * @param src A list of source that should be compiled and converted. * @returns An instance of ProjectReflection on success, undefined otherwise. */ public convert(): ProjectReflection | undefined { @@ -201,41 +301,30 @@ export class Application extends ChildableComponent< ); } - if (Object.keys(this.options.getCompilerOptions()).length === 0) { + if ( + Object.keys(this.options.getCompilerOptions()).length === 0 && + this.options.getValue("packages").length === 0 + ) { this.logger.warn( `No compiler options set. This likely means that TypeDoc did not find your tsconfig.json. Generated documentation will probably be empty.` ); } - const programs = [ - ts.createProgram({ - rootNames: this.options.getFileNames(), - options: this.options.getCompilerOptions(), - projectReferences: this.options.getProjectReferences(), - }), - ]; - - // This might be a solution style tsconfig, in which case we need to add a program for each - // reference so that the converter can look through each of these. - if (programs[0].getRootFileNames().length === 0) { - this.logger.verbose( - "tsconfig appears to be a solution style tsconfig - creating programs for references" - ); - const resolvedReferences = - programs[0].getResolvedProjectReferences(); - for (const ref of resolvedReferences ?? []) { - if (!ref) continue; // This indicates bad configuration... will be reported later. - - programs.push( - ts.createProgram({ - options: ref.commandLine.options, - rootNames: ref.commandLine.fileNames, - projectReferences: ref.commandLine.projectReferences, - }) - ); - } + const packages = this.options + .getValue("packages") + .map(normalizePath); + const entryPoints = getEntryPointsForPackages(this.logger, packages); + if (entryPoints === undefined) { + return; + } + if (entryPoints.length === 0) { + // No package entry points were specified. Try to process the file-oriented entry points. + // The reason this is skipped when using --packages is that this approach currently assumes a global + // tsconfig compilation setup which is not likely to exist when using --packages. + entryPoints.push(...this.getEntryPointsForPaths(this.entryPoints)); } + const programs = entryPoints.map((e) => e.program); this.logger.verbose(`Converting with ${programs.length} programs`); const errors = flatMap(programs, ts.getPreEmitDiagnostics); @@ -250,10 +339,7 @@ export class Application extends ChildableComponent< } } - return this.converter.convert( - this.expandInputFiles(this.entryPoints), - programs - ); + return this.converter.convert(entryPoints); } public convertAndWatch( @@ -297,6 +383,14 @@ export class Application extends ChildableComponent< return; } + // Support for packages mode is currently unimplemented + if (this.options.getValue("packages").length !== 0) { + this.logger.error( + 'Running with "--packages" is not supported in watch mode.' + ); + return; + } + // Matches the behavior of the tsconfig option reader. let tsconfigFile = this.options.getValue("tsconfig"); tsconfigFile = @@ -344,10 +438,25 @@ export class Application extends ChildableComponent< if (successFinished) { this.logger.resetErrors(); - const project = this.converter.convert( - this.expandInputFiles(this.entryPoints), - currentProgram - ); + const inputFiles = this.expandInputFiles(this.entryPoints); + const baseDir = getCommonDirectory(inputFiles); + const entryPoints = new Array(); + for (const file of inputFiles.map(normalizePath)) { + const sourceFile = currentProgram.getSourceFile(file); + if (sourceFile) { + entryPoints.push({ + displayName: getModuleName(resolve(file), baseDir), + path: file, + sourceFile, + program: currentProgram, + }); + } else { + this.logger.warn( + `Unable to locate entry point: ${file} within the program defined by ${tsconfigFile}` + ); + } + } + const project = this.converter.convert(entryPoints); currentProgram = undefined; successFinished = false; void success(project).then(() => { @@ -477,6 +586,66 @@ export class Application extends ChildableComponent< return files; } + /** + * Converts a list of file-oriented paths in to DocumentationEntryPoints for conversion. + * This is in contrast with the package-oriented `getEntryPointsForPackages` + * + * @param entryPointPaths The list of filepaths that should be expanded. + * @returns The DocumentationEntryPoints corresponding to all the found entry points + * @internal + */ + public getEntryPointsForPaths( + entryPointPaths: string[] + ): DocumentationEntryPoint[] { + const rootProgram = ts.createProgram({ + rootNames: this.options.getFileNames(), + options: this.options.getCompilerOptions(), + projectReferences: this.options.getProjectReferences(), + }); + const programs = new Array(); + programs.push(rootProgram); + // This might be a solution style tsconfig, in which case we need to add a program for each + // reference so that the converter can look through each of these. + if (rootProgram.getRootFileNames().length === 0) { + this.logger.verbose( + "tsconfig appears to be a solution style tsconfig - creating programs for references" + ); + const resolvedReferences = rootProgram.getResolvedProjectReferences(); + for (const ref of resolvedReferences ?? []) { + if (!ref) continue; // This indicates bad configuration... will be reported later. + + programs.push( + ts.createProgram({ + options: ref.commandLine.options, + rootNames: ref.commandLine.fileNames, + projectReferences: ref.commandLine.projectReferences, + }) + ); + } + } + const inputFiles = this.expandInputFiles(entryPointPaths); + const baseDir = getCommonDirectory(inputFiles); + const entryPoints = new Array(); + entryLoop: for (const file of inputFiles.map(normalizePath)) { + for (const program of programs) { + const sourceFile = program.getSourceFile(file); + if (sourceFile) { + entryPoints.push({ + displayName: getModuleName(resolve(file), baseDir), + path: file, + sourceFile, + program, + }); + continue entryLoop; + } + } + this.logger.warn( + `Unable to locate entry point: ${file}` + ); + } + return entryPoints; + } + /** * Print the version number. */ diff --git a/src/lib/converter/context.ts b/src/lib/converter/context.ts index 837499b48..67acb933b 100644 --- a/src/lib/converter/context.ts +++ b/src/lib/converter/context.ts @@ -92,8 +92,6 @@ export class Context { * Create a new Context instance. * * @param converter The converter instance that has created the context. - * @param entryPoints A list of all entry points for this project. - * @param checker The TypeChecker instance returned by the TypeScript compiler. * @internal */ constructor( diff --git a/src/lib/converter/converter.ts b/src/lib/converter/converter.ts index 4fa87bcc8..85dff2bc7 100644 --- a/src/lib/converter/converter.ts +++ b/src/lib/converter/converter.ts @@ -1,24 +1,28 @@ import * as ts from "typescript"; import * as _ from "lodash"; import * as assert from "assert"; -import { resolve } from "path"; import { Application } from "../application"; import { Type, ProjectReflection, ReflectionKind } from "../models/index"; import { Context } from "./context"; import { ConverterComponent } from "./components"; import { Component, ChildableComponent } from "../utils/component"; -import { BindOption, normalizePath } from "../utils"; +import { BindOption } from "../utils"; import { convertType } from "./types"; import { ConverterEvents } from "./converter-events"; import { convertSymbol } from "./symbols"; -import { relative } from "path"; -import { getCommonDirectory } from "../utils/fs"; import { createMinimatch } from "../utils/paths"; import { IMinimatch } from "minimatch"; import { hasAllFlags, hasAnyFlag } from "../utils/enum"; import { resolveAliasedSymbol } from "./utils/symbols"; +export interface DocumentationEntryPoint { + displayName: string; + path: string; + program: ts.Program; + sourceFile: ts.SourceFile; +} + /** * Compiles source files using TypeScript and converts compiler symbols to reflections. */ @@ -135,13 +139,13 @@ export class Converter extends ChildableComponent< * Compile the given source files and create a project reflection for them. * * @param entryPoints the entry points of this program. - * @param program the program to document that has already been type checked. + * @param programs the programs to document, that have already been type checked. + * @param packages an array of packages (used in --packages mode) */ convert( - entryPoints: readonly string[], - programs: ts.Program | readonly ts.Program[] + entryPoints: readonly DocumentationEntryPoint[] ): ProjectReflection { - programs = programs instanceof Array ? programs : [programs]; + const programs = entryPoints.map((e) => e.program); this.externalPatternCache = void 0; const project = new ProjectReflection(this.name); @@ -258,57 +262,43 @@ export class Converter extends ChildableComponent< * @param context The context object describing the current state the converter is in. * @returns An array containing all errors generated by the TypeScript compiler. */ - private compile(entryPoints: readonly string[], context: Context) { - const baseDir = getCommonDirectory(entryPoints); - const entries: { - file: string; - sourceFile: ts.SourceFile; - program: ts.Program; - context?: Context; - }[] = []; - - entryLoop: for (const file of entryPoints.map(normalizePath)) { - for (const program of context.programs) { - const sourceFile = program.getSourceFile(file); - if (sourceFile) { - entries.push({ file, sourceFile, program }); - continue entryLoop; - } - } - this.application.logger.warn( - `Unable to locate entry point: ${file}` - ); - } - - for (const entry of entries) { - context.setActiveProgram(entry.program); - entry.context = this.convertExports( + private compile( + entryPoints: readonly DocumentationEntryPoint[], + context: Context + ) { + const entries = entryPoints.map((e) => { + return { + entryPoint: e, + context: undefined as Context | undefined, + }; + }); + entries.forEach((e) => { + context.setActiveProgram(e.entryPoint.program); + e.context = this.convertExports( context, - entry.sourceFile, - entryPoints, - getModuleName(resolve(entry.file), baseDir) + e.entryPoint.sourceFile, + entries.length === 1, + e.entryPoint.displayName ); - } - - for (const { sourceFile, context } of entries) { + }); + for (const { entryPoint, context } of entries) { // active program is already set on context assert(context); - this.convertReExports(context, sourceFile); + this.convertReExports(context, entryPoint.sourceFile); } - context.setActiveProgram(undefined); } private convertExports( context: Context, node: ts.SourceFile, - entryPoints: readonly string[], + singleEntryPoint: boolean, entryName: string ) { const symbol = getSymbolForModuleLike(context, node); let moduleContext: Context; - if (entryPoints.length === 1) { + if (singleEntryPoint) { // Special case for when we're giving a single entry point, we don't need to // create modules for each entry. Register the project as this module. context.project.registerReflection(context.project, symbol); @@ -430,13 +420,6 @@ export class Converter extends ChildableComponent< } } -function getModuleName(fileName: string, baseDir: string) { - return normalizePath(relative(baseDir, fileName)).replace( - /(\/index)?(\.d)?\.[tj]sx?$/, - "" - ); -} - function getSymbolForModuleLike( context: Context, node: ts.SourceFile | ts.ModuleBlock diff --git a/src/lib/converter/index.ts b/src/lib/converter/index.ts index aa7c15d25..6344e7f6d 100644 --- a/src/lib/converter/index.ts +++ b/src/lib/converter/index.ts @@ -1,5 +1,6 @@ export { Context } from "./context"; export { Converter } from "./converter"; +export type { DocumentationEntryPoint } from "./converter"; export { convertDefaultValue, convertExpression } from "./convert-expression"; diff --git a/src/lib/output/themes/DefaultTheme.ts b/src/lib/output/themes/DefaultTheme.ts index 6df3c01a5..0a6762d91 100644 --- a/src/lib/output/themes/DefaultTheme.ts +++ b/src/lib/output/themes/DefaultTheme.ts @@ -157,10 +157,12 @@ export class DefaultTheme extends Theme { * @returns The root navigation item. */ getNavigation(project: ProjectReflection): NavigationItem { + const multipleEntryPoints = + project.getChildrenByKind(ReflectionKind.Module).length !== 0; const builder = new NavigationBuilder( project, project, - this.application.options.getValue("entryPoints").length > 1 + multipleEntryPoints ); return builder.build( this.application.options.getValue("readme") !== "none" diff --git a/src/lib/utils/options/declaration.ts b/src/lib/utils/options/declaration.ts index 054320134..beea1dd9e 100644 --- a/src/lib/utils/options/declaration.ts +++ b/src/lib/utils/options/declaration.ts @@ -37,6 +37,7 @@ export type TypeDocOptionValues = { export interface TypeDocOptionMap { options: string; tsconfig: string; + packages: string[]; entryPoints: string[]; exclude: string[]; diff --git a/src/lib/utils/options/readers/typedoc.ts b/src/lib/utils/options/readers/typedoc.ts index bd06521c7..178c8fc51 100644 --- a/src/lib/utils/options/readers/typedoc.ts +++ b/src/lib/utils/options/readers/typedoc.ts @@ -86,8 +86,12 @@ export class TypeDocReader implements OptionsReader { delete data["extends"]; } - for (const [key, val] of Object.entries(data)) { + for (let [key, val] of Object.entries(data)) { try { + // The "packages" option is an array of paths and should be interpreted as relative to the typedoc.json + if (key === "packages" && Array.isArray(val)) { + val = val.map((e) => Path.resolve(file, "..", e)); + } container.setValue(key, val); } catch (error) { logger.error(error.message); diff --git a/src/lib/utils/options/sources/typedoc.ts b/src/lib/utils/options/sources/typedoc.ts index 2f8b74a51..9c45fcaab 100644 --- a/src/lib/utils/options/sources/typedoc.ts +++ b/src/lib/utils/options/sources/typedoc.ts @@ -17,6 +17,13 @@ export function addTypeDocOptions(options: Pick) { hint: ParameterHint.File, defaultValue: process.cwd(), }); + options.addDeclaration({ + name: "packages", + help: + "Specify one or more package folders from which a package.json file should be loaded to determine the entry points. Your JS files must have sourcemaps for this to work. If the root of an npm or Yarn workspace is given, the packages specified in `workpaces` will be loaded.", + type: ParameterType.Array, + defaultValue: [], + }); options.addDeclaration({ name: "entryPoints", diff --git a/src/lib/utils/package-manifest.ts b/src/lib/utils/package-manifest.ts new file mode 100644 index 000000000..591e620fe --- /dev/null +++ b/src/lib/utils/package-manifest.ts @@ -0,0 +1,214 @@ +// Utilities to support the inspection of node package "manifests" (package.json's) + +import glob = require("glob"); +import { dirname, join, resolve } from "path"; +import { flatMap } from "./array"; + +import { readFile } from "./fs"; +import { Logger } from "./loggers"; + +/** + * Helper for the TS type system to understand hasOwnProperty + * and narrow a type appropriately. + * @param obj the receiver of the hasOwnProperty method call + * @param prop the property to test for + */ +function hasOwnProperty( + obj: object, + prop: K +): obj is Record { + return Object.prototype.hasOwnProperty.call(obj, prop); +} + +/** + * Loads a package.json and validates that it is a JSON Object + */ +export function loadPackageManifest( + logger: Logger, + packageJsonPath: string +): Record | undefined { + const packageJson: unknown = JSON.parse(readFile(packageJsonPath)); + if (typeof packageJson !== "object" || !packageJson) { + logger.error(`The file ${packageJsonPath} is not an object.`); + return undefined; + } + return packageJson as Record; +} + +/** + * Load the paths to packages specified in a Yarn workspace package JSON + * Returns undefined if packageJSON does not define a Yarn workspace + * @param packageJSON the package json object + */ +function getPackagePaths( + packageJSON: Record +): string[] | undefined { + if ( + Array.isArray(packageJSON.workspaces) && + packageJSON.workspaces.every((i) => typeof i === "string") + ) { + return packageJSON.workspaces; + } + if ( + typeof packageJSON.workspaces === "object" && + packageJSON.workspaces != null && + hasOwnProperty(packageJSON.workspaces, "packages") && + Array.isArray(packageJSON.workspaces.packages) && + packageJSON.workspaces.packages.every((i) => typeof i === "string") + ) { + return packageJSON.workspaces.packages; + } + return undefined; +} + +/** + * Should produce the same results as the equivalent code in Yarn + * https://github.com/yarnpkg/yarn/blob/a4708b29ac74df97bac45365cba4f1d62537ceb7/src/config.js#L799 + */ +function globPackages(workspacePath: string, packageJsonDir: string): string[] { + return glob.sync(resolve(packageJsonDir, workspacePath, "package.json"), { + ignore: resolve(packageJsonDir, workspacePath, "node_modules"), + }); +} + +/** + * Given a list of (potentially wildcarded) package paths, + * return all the actual package folders found. + */ +export function expandPackages( + logger: Logger, + packageJsonDir: string, + workspaces: string[] +): string[] { + // Technnically npm and Yarn workspaces don't support recursive nesting, + // however we support the passing of paths to either packages or + // to the root of a workspace tree in our params and so we could here + // be dealing with either a root or a leaf. So let's do this recursively, + // as it actually is simpler from an implementation perspective anyway. + return flatMap(workspaces, (workspace) => { + const globbedPackageJsonPaths = globPackages(workspace, packageJsonDir); + return flatMap(globbedPackageJsonPaths, (packageJsonPath) => { + const packageJson = loadPackageManifest(logger, packageJsonPath); + if (packageJson === undefined) { + logger.error(`Failed to load ${packageJsonPath}`); + return []; + } + const packagePaths = getPackagePaths(packageJson); + if (packagePaths === undefined) { + // Assume this is a single package repo + return [dirname(packageJsonPath)]; + } + // This is a workpace root package, recurse + return expandPackages( + logger, + dirname(packageJsonPath), + packagePaths + ); + }); + }); +} + +/** + * Finds the corresponding TS file from a transpiled JS file. + * The JS must be built with sourcemaps. + */ +function getTsSourceFromJsSource( + logger: Logger, + jsPath: string +): string | undefined { + const contents = readFile(jsPath); + const sourceMapPrefix = "\n//# sourceMappingURL="; + const indexOfSourceMapPrefix = contents.indexOf(sourceMapPrefix); + if (indexOfSourceMapPrefix === -1) { + logger.error(`The file ${jsPath} does not contain a sourceMappingURL`); + return; + } + const endOfSourceMapPrefix = + indexOfSourceMapPrefix + sourceMapPrefix.length; + const newLineIndex = contents.indexOf("\n", endOfSourceMapPrefix); + const sourceMapURL = contents.slice( + endOfSourceMapPrefix, + newLineIndex === -1 ? undefined : newLineIndex + ); + const resolvedSourceMapURL = resolve(jsPath, "..", sourceMapURL); + const sourceMap: unknown = JSON.parse(readFile(resolvedSourceMapURL)); + if (typeof sourceMap !== "object" || !sourceMap) { + logger.error( + `The source map file ${resolvedSourceMapURL} is not an object.` + ); + return undefined; + } + if ( + !hasOwnProperty(sourceMap, "sources") || + !Array.isArray(sourceMap.sources) + ) { + logger.error( + `The source map ${resolvedSourceMapURL} does not contain "sources".` + ); + return undefined; + } + let sourceRoot: string | undefined; + if ( + hasOwnProperty(sourceMap, "sourceRoot") && + typeof sourceMap.sourceRoot === "string" + ) { + sourceRoot = sourceMap.sourceRoot; + } + // There's a pretty large assumption in here that we only have + // 1 source file per js file. This is a pretty standard typescript approach, + // but people might do interesting things with transpilation that could break this. + let source = sourceMap.sources[0]; + // If we have a sourceRoot, trim any leading slash from the source, and join them + // Similar to how it's done at https://github.com/mozilla/source-map/blob/58819f09018d56ef84dc41ba9c93f554e0645169/lib/util.js#L412 + if (sourceRoot !== undefined) { + source = source.replace(/^\//, ""); + source = join(sourceRoot, source); + } + const sourcePath = resolve(resolvedSourceMapURL, "..", source); + return sourcePath; +} + +// A Symbol used to communicate that this package should be ignored +export const ignorePackage = Symbol("ignorePackage"); + +/** + * Given a package.json, attempt to find the TS file that defines its entry point + * The JS must be built with sourcemaps. + * + * When the TS file cannot be determined, the intention is to + * - Ignore things which don't appear to be `require`-able node packages. + * - Fail on things which appear to be `require`-able node packages but are missing + * the necessary metadata for us to document. + */ +export function getTsEntryPointForPackage( + logger: Logger, + packageJsonPath: string, + packageJson: Record +): string | undefined | typeof ignorePackage { + let packageMain = "index.js"; // The default, per the npm docs. + if ( + hasOwnProperty(packageJson, "main") && + typeof packageJson.main == "string" + ) { + packageMain = packageJson.main; + } + let jsEntryPointPath = resolve(packageJsonPath, "..", packageMain); + // The jsEntryPointPath from the package manifest can be like a require path. + // It could end with .js, or it could end without .js, or it could be a folder containing an index.js + // We can use require.resolve to let node do its magic. + // Pass an empty `paths` as node_modules locations do not need to be examined + try { + jsEntryPointPath = require.resolve(jsEntryPointPath, { paths: [] }); + } catch (e) { + if (e.code !== "MODULE_NOT_FOUND") { + throw e; + } else { + logger.warn( + `Could not determine the JS entry point for "${packageJsonPath}". Package will be ignored.` + ); + logger.verbose(e.message); + return ignorePackage; + } + } + return getTsSourceFromJsSource(logger, jsEntryPointPath); +} diff --git a/src/test/converter.test.ts b/src/test/converter.test.ts index aef5a84b3..e36595408 100644 --- a/src/test/converter.test.ts +++ b/src/test/converter.test.ts @@ -76,8 +76,7 @@ describe("Converter", function () { before(); resetReflectionID(); result = app.converter.convert( - app.expandInputFiles([path]), - program + app.getEntryPointsForPaths([path]) ); after(); ok( diff --git a/src/test/converter2.test.ts b/src/test/converter2.test.ts index 8ccbfe397..a672fbbf6 100644 --- a/src/test/converter2.test.ts +++ b/src/test/converter2.test.ts @@ -252,7 +252,9 @@ describe("Converter2", () => { ok(entryPoint, `No entry point found for ${entry}`); - const project = app.converter.convert([entryPoint], program); + const project = app.converter.convert( + app.getEntryPointsForPaths([entryPoint]) + ); check(project); }); } diff --git a/src/test/packages.test.ts b/src/test/packages.test.ts new file mode 100644 index 000000000..4f0477394 --- /dev/null +++ b/src/test/packages.test.ts @@ -0,0 +1,62 @@ +import { ok, strictEqual } from "assert"; +import * as Path from "path"; + +import * as td from ".."; +import { Logger } from "../lib/utils"; +import { expandPackages } from "../lib/utils/package-manifest"; + +describe("Packages support", () => { + it("handles monorepos", () => { + const base = Path.join(__dirname, "packages", "ts-monorepo"); + const app = new td.Application(); + app.options.addReader(new td.TypeDocReader()); + app.bootstrap({ + options: Path.join(base, "typedoc.json"), + }); + const project = app.convert(); + ok(project, "Failed to convert"); + const result = app.serializer.projectToObject(project); + ok(result.children !== undefined); + strictEqual( + result.children.length, + 4, + "incorrect number of packages processed" + ); + }); + + it("handles single packages", () => { + const base = Path.join( + __dirname, + "packages", + "typedoc-single-package-example" + ); + const app = new td.Application(); + app.options.addReader(new td.TypeDocReader()); + app.bootstrap({ + options: Path.join(base, "typedoc.json"), + }); + const project = app.convert(); + ok(project, "Failed to convert"); + const result = app.serializer.projectToObject(project); + ok(result.children !== undefined); + strictEqual( + result.children.length, + 1, + "incorrect number of packages processed" + ); + }); + + describe("expandPackages", () => { + it("handles a glob", () => { + const base = Path.join(__dirname, "packages", "ts-monorepo"); + const expandedPackages = expandPackages(new Logger(), base, [ + "packages/*", + ]); + strictEqual( + expandedPackages.length, + 3, + "Found an unexpected number of packages" + ); + }); + }); +}); diff --git a/src/test/packages/README.md b/src/test/packages/README.md new file mode 100644 index 000000000..16499d0c0 --- /dev/null +++ b/src/test/packages/README.md @@ -0,0 +1,5 @@ +# Example repos for "--packages" mode + +This folder contains examples for the testing of the "--packages" mode. + +The codebase https://github.com/efokschaner/ts-monorepo/tree/typedoc is also pulled dynamically in to the `dist` copy of this folder during tests. diff --git a/src/test/packages/typedoc-single-package-example/package.json b/src/test/packages/typedoc-single-package-example/package.json new file mode 100644 index 000000000..ce63db770 --- /dev/null +++ b/src/test/packages/typedoc-single-package-example/package.json @@ -0,0 +1,28 @@ +{ + "name": "typedoc-single-package-example", + "version": "1.0.0", + "description": "An example of using typedoc with a single package", + "private": true, + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "test": "node ." + }, + "devDependencies": { + "typescript": "^4.2.4" + }, + "repository": { + "type": "git", + "url": "git://github.com/TypeStrong/TypeDoc.git" + }, + "keywords": [ + "typedoc", + "example" + ], + "author": "", + "license": "Apache-2.0", + "bugs": { + "url": "https://github.com/TypeStrong/TypeDoc/issues" + }, + "homepage": "https://github.com/TypeStrong/TypeDoc#readme" +} diff --git a/src/test/packages/typedoc-single-package-example/src/index.ts b/src/test/packages/typedoc-single-package-example/src/index.ts new file mode 100644 index 000000000..e91428dc9 --- /dev/null +++ b/src/test/packages/typedoc-single-package-example/src/index.ts @@ -0,0 +1,7 @@ +export function helloWorld() { + return "Hello World!"; +} + +if (require.main === module) { + console.log(helloWorld()); +} diff --git a/src/test/packages/typedoc-single-package-example/tsconfig.json b/src/test/packages/typedoc-single-package-example/tsconfig.json new file mode 100644 index 000000000..868fdc44b --- /dev/null +++ b/src/test/packages/typedoc-single-package-example/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": false, + "declaration": true, + "declarationMap": true, + "incremental": true, + "module": "commonjs", + "moduleResolution": "node", + "noUnusedLocals": true, + "noUnusedParameters": true, + "outDir": "dist", + "preserveConstEnums": true, + "removeComments": false, + "sourceMap": true, + "strict": true, + "target": "es2015" + }, + "include": ["src/**/*"] +} diff --git a/src/test/packages/typedoc-single-package-example/typedoc.json b/src/test/packages/typedoc-single-package-example/typedoc.json new file mode 100644 index 000000000..f6320cfae --- /dev/null +++ b/src/test/packages/typedoc-single-package-example/typedoc.json @@ -0,0 +1,7 @@ +{ + "gitRevision": "master", + "name": "typedoc-single-package-example", + "out": "docs", + "packages": ["."], + "plugin": [] +} diff --git a/src/test/renderer.test.ts b/src/test/renderer.test.ts index 8ccaa2c5c..8610787d3 100644 --- a/src/test/renderer.test.ts +++ b/src/test/renderer.test.ts @@ -76,7 +76,7 @@ describe("Renderer", function () { tsconfig: Path.join(src, "..", "tsconfig.json"), plugin: [], }); - app.options.setValue("entryPoints", app.expandInputFiles([src])); + app.options.setValue("entryPoints", [src]); }); it("converts basic example", function () { diff --git a/tsconfig.json b/tsconfig.json index 977ba12ee..9ab061da6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -33,6 +33,7 @@ "src/test/converter2/**/*.ts", "src/test/renderer/specs", "src/test/.dot/**/*.ts", - "src/test/module/**/*.ts" + "src/test/module/**/*.ts", + "src/test/packages/**/*.ts" ] }