diff --git a/.changeset/famous-ants-lie.md b/.changeset/famous-ants-lie.md new file mode 100644 index 00000000000..8d73819ba98 --- /dev/null +++ b/.changeset/famous-ants-lie.md @@ -0,0 +1,5 @@ +--- +"@pnpm/text.comments-parser": major +--- + +Initial release. diff --git a/.changeset/strong-houses-visit.md b/.changeset/strong-houses-visit.md new file mode 100644 index 00000000000..7744e53f6ee --- /dev/null +++ b/.changeset/strong-houses-visit.md @@ -0,0 +1,7 @@ +--- +"@pnpm/read-project-manifest": minor +"@pnpm/write-project-manifest": minor +"pnpm": patch +--- + +Comments in `package.json5` are preserver [#2008](https://github.com/pnpm/pnpm/issues/2008). diff --git a/__typings__/typed.d.ts b/__typings__/typed.d.ts index a05a3425a70..c033485b80b 100644 --- a/__typings__/typed.d.ts +++ b/__typings__/typed.d.ts @@ -45,6 +45,29 @@ declare module 'split-cmd' { export function splitToObject (cmd: string): { command: string, args: string[] } } +declare module 'strip-comments-strings' { + export interface CodeItem { + // What feature of the code has been found: + type: string, + // The indices of the feature in the original code string: + index: number, + indexEnd: number, + // The test of the feature + content: string + } + export interface CodeAttributes { + // The remaining code text after all features have been stripped: + text: string, + // The items found: + comments: CodeItem[], + regexes: CodeItem[], + strings: CodeItem[] + } + export function parseString (str: string): CodeAttributes; + export type CodeItemReplacer = (item: CodeItem) => string; + export function stripComments ( + str: string, replacer?: CodeItemReplacer): string; +} declare module 'bin-links/lib/fix-bin' { function fixBin (path: string, execMode: number): Promise; diff --git a/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/modified.json5 b/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/modified.json5 new file mode 100644 index 00000000000..6aaa08ea229 --- /dev/null +++ b/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/modified.json5 @@ -0,0 +1,9 @@ +/* This is an example of a package.json5 file with comments. */ +{ + /* pnpm should keep comments at the same indentation level */ + name: 'foo', + version: '1.0.0', // it should keep in-line comments on the same line + // It should allow in-line comments with no other content + type: 'commonjs', +} +/* And it should preserve comments at the end of the file. Note no newline. */ \ No newline at end of file diff --git a/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/package.json5 b/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/package.json5 new file mode 100644 index 00000000000..c6b65fc12af --- /dev/null +++ b/pkg-manifest/read-project-manifest/fixtures/commented-package-json5/package.json5 @@ -0,0 +1,9 @@ +/* This is an example of a package.json5 file with comments. */ +{ + /* pnpm should keep comments at the same indentation level */ + name: 'foo', + version: '1.0.0', // it should keep in-line comments on the same line + // It should allow in-line comments with no other content + type: 'module', +} +/* And it should preserve comments at the end of the file. Note no newline. */ \ No newline at end of file diff --git a/pkg-manifest/read-project-manifest/package.json b/pkg-manifest/read-project-manifest/package.json index 0f216386e03..698a78ad67f 100644 --- a/pkg-manifest/read-project-manifest/package.json +++ b/pkg-manifest/read-project-manifest/package.json @@ -32,6 +32,7 @@ "@gwhitney/detect-indent": "7.0.1", "@pnpm/error": "workspace:*", "@pnpm/graceful-fs": "workspace:*", + "@pnpm/text.comments-parser": "workspace:*", "@pnpm/types": "workspace:*", "@pnpm/write-project-manifest": "workspace:*", "fast-deep-equal": "^3.1.3", diff --git a/pkg-manifest/read-project-manifest/src/index.ts b/pkg-manifest/read-project-manifest/src/index.ts index 74530244129..5ecab2090d5 100644 --- a/pkg-manifest/read-project-manifest/src/index.ts +++ b/pkg-manifest/read-project-manifest/src/index.ts @@ -2,9 +2,9 @@ import { promises as fs, Stats } from 'fs' import path from 'path' import { PnpmError } from '@pnpm/error' import { ProjectManifest } from '@pnpm/types' +import { extractComments, CommentSpecifier } from '@pnpm/text.comments-parser' import { writeProjectManifest } from '@pnpm/write-project-manifest' import readYamlFile from 'read-yaml-file' - import detectIndent from '@gwhitney/detect-indent' import equal from 'fast-deep-equal' import isWindows from 'is-windows' @@ -76,7 +76,7 @@ export async function tryReadProjectManifest (projectDir: string): Promise<{ fileName: 'package.json5', manifest: data, writeProjectManifest: createManifestWriter({ - ...detectFileFormatting(text), + ...detectFileFormattingAndComments(text), initialManifest: data, manifestPath, }), @@ -117,6 +117,15 @@ export async function tryReadProjectManifest (projectDir: string): Promise<{ } } +function detectFileFormattingAndComments (text: string) { + const { comments, text: newText, hasFinalNewline } = extractComments(text) + return { + comments, + indent: detectIndent(newText).indent, + insertFinalNewline: hasFinalNewline, + } +} + function detectFileFormatting (text: string) { return { indent: detectIndent(text).indent, @@ -143,7 +152,7 @@ export async function readExactProjectManifest (manifestPath: string) { return { manifest: data, writeProjectManifest: createManifestWriter({ - ...detectFileFormatting(text), + ...detectFileFormattingAndComments(text), initialManifest: data, manifestPath, }), @@ -174,6 +183,7 @@ async function readPackageYaml (filePath: string) { function createManifestWriter ( opts: { initialManifest: ProjectManifest + comments?: CommentSpecifier[] indent?: string | number | undefined insertFinalNewline?: boolean manifestPath: string @@ -184,6 +194,7 @@ function createManifestWriter ( updatedManifest = normalize(updatedManifest) if (force === true || !equal(initialManifest, updatedManifest)) { await writeProjectManifest(opts.manifestPath, updatedManifest, { + comments: opts.comments, indent: opts.indent, insertFinalNewline: opts.insertFinalNewline, }) diff --git a/pkg-manifest/read-project-manifest/test/index.ts b/pkg-manifest/read-project-manifest/test/index.ts index 00177026256..bba57a54554 100644 --- a/pkg-manifest/read-project-manifest/test/index.ts +++ b/pkg-manifest/read-project-manifest/test/index.ts @@ -82,6 +82,26 @@ test('preserve space indentation in json5 file', async () => { expect(rawManifest).toBe("{\n name: 'foo',\n dependencies: {\n bar: '1.0.0',\n },\n}\n") }) +test('preserve comments in json5 file', async () => { + const originalManifest = await fs.readFile( + path.join(fixtures, 'commented-package-json5/package.json5'), 'utf8') + const modifiedManifest = await fs.readFile( + path.join(fixtures, 'commented-package-json5/modified.json5'), 'utf8') + + process.chdir(tempy.directory()) + await fs.writeFile('package.json5', originalManifest, 'utf8') + + const { manifest, writeProjectManifest } = await readProjectManifest(process.cwd()) + + // Have to make a change to get it to write anything: + const newManifest = Object.assign({}, manifest, { type: 'commonjs' }) + + await writeProjectManifest(newManifest) + + const resultingManifest = await fs.readFile('package.json5', 'utf8') + expect(resultingManifest).toBe(modifiedManifest) +}) + test('do not save manifest if it had no changes', async () => { process.chdir(tempy.directory()) diff --git a/pkg-manifest/read-project-manifest/tsconfig.json b/pkg-manifest/read-project-manifest/tsconfig.json index 8a40c8f49c0..cd3c5ef954f 100644 --- a/pkg-manifest/read-project-manifest/tsconfig.json +++ b/pkg-manifest/read-project-manifest/tsconfig.json @@ -18,6 +18,9 @@ { "path": "../../packages/types" }, + { + "path": "../../text/comments-parser" + }, { "path": "../write-project-manifest" } diff --git a/pkg-manifest/write-project-manifest/package.json b/pkg-manifest/write-project-manifest/package.json index 89f15d0b3f4..1ad9036e1bb 100644 --- a/pkg-manifest/write-project-manifest/package.json +++ b/pkg-manifest/write-project-manifest/package.json @@ -29,6 +29,7 @@ }, "homepage": "https://github.com/pnpm/pnpm/blob/main/pkg-manifest/write-project-manifest#readme", "dependencies": { + "@pnpm/text.comments-parser": "workspace:*", "@pnpm/types": "workspace:*", "json5": "^2.2.1", "write-file-atomic": "^4.0.2", diff --git a/pkg-manifest/write-project-manifest/src/index.ts b/pkg-manifest/write-project-manifest/src/index.ts index f220231d34d..ba4f1684e50 100644 --- a/pkg-manifest/write-project-manifest/src/index.ts +++ b/pkg-manifest/write-project-manifest/src/index.ts @@ -1,5 +1,6 @@ import { promises as fs } from 'fs' import path from 'path' +import { insertComments, CommentSpecifier } from '@pnpm/text.comments-parser' import { ProjectManifest } from '@pnpm/types' import JSON5 from 'json5' import writeFileAtomic from 'write-file-atomic' @@ -14,6 +15,7 @@ export async function writeProjectManifest ( filePath: string, manifest: ProjectManifest, opts?: { + comments?: CommentSpecifier[] indent?: string | number | undefined insertFinalNewline?: boolean } @@ -25,9 +27,21 @@ export async function writeProjectManifest ( await fs.mkdir(path.dirname(filePath), { recursive: true }) const trailingNewline = opts?.insertFinalNewline === false ? '' : '\n' + const indent = opts?.indent ?? '\t' - const json = (fileType === 'json5' ? JSON5 : JSON) - .stringify(manifest, undefined, opts?.indent ?? '\t') + const json = ( + fileType === 'json5' + ? stringifyJson5(manifest, indent, opts?.comments) + : JSON.stringify(manifest, undefined, indent) + ) return writeFileAtomic(filePath, `${json}${trailingNewline}`) } + +function stringifyJson5 (obj: object, indent: string | number, comments?: CommentSpecifier[]) { + const json5 = JSON5.stringify(obj, undefined, indent) + if (comments) { + return insertComments(json5, comments) + } + return json5 +} diff --git a/pkg-manifest/write-project-manifest/tsconfig.json b/pkg-manifest/write-project-manifest/tsconfig.json index d37dd6eefe3..2692318a479 100644 --- a/pkg-manifest/write-project-manifest/tsconfig.json +++ b/pkg-manifest/write-project-manifest/tsconfig.json @@ -11,6 +11,9 @@ "references": [ { "path": "../../packages/types" + }, + { + "path": "../../text/comments-parser" } ] } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ca665da9169..2f06207024c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -3787,6 +3787,9 @@ importers: '@pnpm/graceful-fs': specifier: workspace:* version: link:../../fs/graceful-fs + '@pnpm/text.comments-parser': + specifier: workspace:* + version: link:../../text/comments-parser '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -3830,6 +3833,9 @@ importers: pkg-manifest/write-project-manifest: dependencies: + '@pnpm/text.comments-parser': + specifier: workspace:* + version: link:../../text/comments-parser '@pnpm/types': specifier: workspace:* version: link:../../packages/types @@ -5577,6 +5583,16 @@ importers: specifier: ^3.0.2 version: 3.0.2 + text/comments-parser: + dependencies: + strip-comments-strings: + specifier: 1.2.0 + version: 1.2.0 + devDependencies: + '@pnpm/text.comments-parser': + specifier: workspace:* + version: 'link:' + workspace/filter-workspace-packages: dependencies: '@pnpm/error': @@ -16100,6 +16116,10 @@ packages: resolution: {integrity: sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==} engines: {node: '>=8'} + /strip-comments-strings/1.2.0: + resolution: {integrity: sha512-zwF4bmnyEjZwRhaak9jUWNxc0DoeKBJ7lwSN/LEc8dQXZcUFG6auaaTQJokQWXopLdM3iTx01nQT8E4aL29DAQ==} + dev: false + /strip-final-newline/2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -17589,6 +17609,7 @@ time: /string.prototype.replaceall/1.0.6: '2021-10-05T01:00:00.092Z' /strip-ansi/6.0.1: '2021-09-23T16:34:41.798Z' /strip-bom/4.0.0: '2019-04-28T04:40:47.887Z' + /strip-comments-strings/1.2.0: '2022-06-12T23:34:53.852Z' /symlink-dir/5.1.0: '2022-11-20T16:48:15.918Z' /syncpack/8.3.9: '2022-10-28T16:58:42.312Z' /tar-stream/2.2.0: '2020-12-29T10:22:57.508Z' diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0dc2fad8042..4c39c34b7b4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -20,6 +20,7 @@ packages: - resolving/* - reviewing/* - store/* + - text/* - workspace/* - "!**/example/**" - "!**/test/**" diff --git a/text/comments-parser/README.md b/text/comments-parser/README.md new file mode 100644 index 00000000000..40966468699 --- /dev/null +++ b/text/comments-parser/README.md @@ -0,0 +1,14 @@ +# @pnpm/text.comments-parser + +> Extracts and inserts comments from/to text + +## Installation + +``` +pnpm i @pnpm/text.comments-parser + +``` + +## License + +[MIT](LICENSE) diff --git a/text/comments-parser/jest.config.js b/text/comments-parser/jest.config.js new file mode 100644 index 00000000000..45c039e43c6 --- /dev/null +++ b/text/comments-parser/jest.config.js @@ -0,0 +1 @@ +module.exports = require('../../jest.config.js') \ No newline at end of file diff --git a/text/comments-parser/package.json b/text/comments-parser/package.json new file mode 100644 index 00000000000..1b34c66cb14 --- /dev/null +++ b/text/comments-parser/package.json @@ -0,0 +1,41 @@ +{ + "name": "@pnpm/text.comments-parser", + "description": "Extracts and inserts comments from/to text", + "version": "0.0.0", + "bugs": { + "url": "https://github.com/pnpm/pnpm/issues" + }, + "main": "lib/index.js", + "types": "lib/index.d.ts", + "files": [ + "lib", + "!*.map" + ], + "keywords": [ + "pnpm7" + ], + "license": "MIT", + "engines": { + "node": ">=14.6" + }, + "repository": "https://github.com/pnpm/pnpm/blob/main/text/comments-parser", + "scripts": { + "start": "tsc --watch", + "_test": "jest", + "test": "pnpm run compile && pnpm run _test", + "lint": "eslint src/**/*.ts test/**/*.ts", + "prepublishOnly": "pnpm run compile", + "compile": "tsc --build && pnpm run lint --fix" + }, + "dependencies": { + "strip-comments-strings": "1.2.0" + }, + "homepage": "https://github.com/pnpm/pnpm/blob/main/text/comments-parser#readme", + "funding": "https://opencollective.com/pnpm", + "devDependencies": { + "@pnpm/text.comments-parser": "workspace:*" + }, + "exports": { + ".": "./lib/index.js" + } +} diff --git a/text/comments-parser/src/CommentSpecifier.ts b/text/comments-parser/src/CommentSpecifier.ts new file mode 100644 index 00000000000..40c2d6fe485 --- /dev/null +++ b/text/comments-parser/src/CommentSpecifier.ts @@ -0,0 +1,9 @@ +export interface CommentSpecifier { + type: string + content: string + lineNumber: number + after?: string + on: string + whitespace: string + before?: string +} diff --git a/text/comments-parser/src/extractComments.ts b/text/comments-parser/src/extractComments.ts new file mode 100644 index 00000000000..91734f9d37b --- /dev/null +++ b/text/comments-parser/src/extractComments.ts @@ -0,0 +1,72 @@ +import { parseString, stripComments } from 'strip-comments-strings' +import { CommentSpecifier } from './CommentSpecifier' + +export function extractComments (text: string) { + const hasFinalNewline = text.endsWith('\n') + if (!hasFinalNewline) { + /* For the sake of the comment parser, which otherwise loses the + * final character of a final comment + */ + text += '\n' + } + const { comments: rawComments } = parseString(text) + const comments: CommentSpecifier[] = [] + let stripped = stripComments(text) + if (!hasFinalNewline) { + stripped = stripped.slice(0, -1) + } + let offset = 0 // accumulates difference of indices from text to stripped + for (const comment of rawComments) { + /* Extract much more context for the comment needed to restore it later */ + // Unfortunately, JavaScript lastIndexOf does not have an end parameter: + const preamble: string = stripped.slice(0, comment.index - offset) + const lineStart = Math.max(preamble.lastIndexOf('\n'), 0) + const priorLines = preamble.split('\n') + let lineNumber = priorLines.length + let after = '' + let hasAfter = false + if (lineNumber === 1) { + if (preamble.trim().length === 0) { + lineNumber = 0 + } + } else { + after = priorLines[lineNumber - 2] + hasAfter = true + if (priorLines[0].trim().length === 0) { + /* JSON5.stringify will not have a whitespace-only line at the start */ + lineNumber -= 1 + } + } + let lineEnd = stripped.indexOf( + '\n', (lineStart === 0) ? 0 : lineStart + 1) + if (lineEnd < 0) { + lineEnd = stripped.length + } + const whitespaceMatch = stripped + .slice(lineStart, comment.index - offset) + .match(/^\s*/) + + const newComment: CommentSpecifier = { + type: comment.type, + content: comment.content, + lineNumber, + on: stripped.slice(lineStart, lineEnd), + whitespace: whitespaceMatch ? whitespaceMatch[0] : '', + } + + if (hasAfter) { + newComment.after = after + } + const nextLineEnd = stripped.indexOf('\n', lineEnd + 1) + if (nextLineEnd >= 0) { + newComment.before = stripped.slice(lineEnd, nextLineEnd) + } + comments.push(newComment) + offset += comment.indexEnd - comment.index + } + return { + text: stripped, + comments: comments.length ? comments : undefined, + hasFinalNewline, + } +} diff --git a/text/comments-parser/src/index.ts b/text/comments-parser/src/index.ts new file mode 100644 index 00000000000..65dcada8f38 --- /dev/null +++ b/text/comments-parser/src/index.ts @@ -0,0 +1,3 @@ +export * from './extractComments' +export * from './insertComments' +export * from './CommentSpecifier' diff --git a/text/comments-parser/src/insertComments.ts b/text/comments-parser/src/insertComments.ts new file mode 100644 index 00000000000..6e444eb86cc --- /dev/null +++ b/text/comments-parser/src/insertComments.ts @@ -0,0 +1,83 @@ +import { CommentSpecifier } from './CommentSpecifier' + +export function insertComments (json: string, comments: CommentSpecifier[]) { + // We need to reintroduce the comments. So create an index of + // the lines of the manifest so we can try to match them up. + // We eliminate whitespace and quotes in the index entries, + // because pnpm may have changed them. + const jsonLines = json.split('\n') + const index = {} + const canonicalizer = /[\s'"]/g + for (let i = 0; i < jsonLines.length; ++i) { + const key = jsonLines[i].replace(canonicalizer, '') + if (key in index) { + index[key] = -1 // Mark this line as occurring twice + } else { + index[key] = i + } + } + + // A place to put comments that come _before_ the lines they are + // anchored to: + const jsonPrefix: Record = {} + for (const comment of comments) { + // First if we can find the line the comment was on, that is + // the most reliable locator: + let key = comment.on.replace(canonicalizer, '') + if (key && index[key] !== undefined && index[key] >= 0) { + jsonLines[index[key]] += ' ' + comment.content + continue + } + // Next, if it's not before anything, it must have been at the very end: + if (comment.before === undefined) { + jsonLines[jsonLines.length - 1] += comment.whitespace + comment.content + continue + } + // Next, try to put it before something; note the comment extractor + // used the convention that position 0 is before the first line: + let location = (comment.lineNumber === 0) ? 0 : -1 + if (location < 0) { + key = comment.before.replace(canonicalizer, '') + if (key && index[key] !== undefined) { + location = index[key] + } + } + if (location >= 0) { + if (jsonPrefix[location]) { + jsonPrefix[location] += ' ' + comment.content + } else { + const inlineWhitespace = comment.whitespace.startsWith('\n') + ? comment.whitespace.slice(1) + : comment.whitespace + jsonPrefix[location] = inlineWhitespace + comment.content + } + continue + } + // The last definite indicator we can use is that it is after something: + if (comment.after) { + key = comment.after.replace(canonicalizer, '') + if (key && index[key] !== undefined && index[key] >= 0) { + jsonLines[index[key]] += comment.whitespace + comment.content + continue + } + } + // Finally, try to get it in the right general location by using the + // line number, but warn the user the comment may have been relocated: + location = comment.lineNumber - 1 // 0 was handled above + let separator = ' ' + if (location >= jsonLines.length) { + location = jsonLines.length - 1 + separator = '\n' + } + jsonLines[location] += separator + comment.content + + ' /* [comment possibly relocated by pnpm] */' + } + // Insert the accumulated prefixes: + for (let i = 0; i < jsonLines.length; ++i) { + if (jsonPrefix[i]) { + jsonLines[i] = jsonPrefix[i] + '\n' + jsonLines[i] + } + } + // And reassemble the manifest: + return jsonLines.join('\n') +} diff --git a/text/comments-parser/test/index.ts b/text/comments-parser/test/index.ts new file mode 100644 index 00000000000..c6608d613b1 --- /dev/null +++ b/text/comments-parser/test/index.ts @@ -0,0 +1,19 @@ +import { extractComments, insertComments } from '@pnpm/text.comments-parser' + +test('extract and insert JSON5 comments', () => { + const json5WithComments = `/* This is an example of a package.json5 file with comments. */ +{ + /* pnpm should keep comments at the same indentation level */ + name: 'foo', + version: '1.0.0', // it should keep in-line comments on the same line + // It should allow in-line comments with no other content + type: 'commonjs', +} +/* And it should preserve comments at the end of the file. Note no newline. */` + const { comments } = extractComments(json5WithComments) + expect(insertComments(`{ + name: 'foo', + version: '1.0.0', + type: 'commonjs', +}`, comments!)) +}) diff --git a/text/comments-parser/tsconfig.json b/text/comments-parser/tsconfig.json new file mode 100644 index 00000000000..c6f0399f60e --- /dev/null +++ b/text/comments-parser/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "@pnpm/tsconfig", + "compilerOptions": { + "outDir": "lib", + "rootDir": "src" + }, + "include": [ + "src/**/*.ts", + "../../__typings__/**/*.d.ts" + ], + "references": [] +} diff --git a/text/comments-parser/tsconfig.lint.json b/text/comments-parser/tsconfig.lint.json new file mode 100644 index 00000000000..1bbe711971a --- /dev/null +++ b/text/comments-parser/tsconfig.lint.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "../../__typings__/**/*.d.ts" + ] +}