From 1df70ff312bba1ae8bc5b0036cb84c058e5dcca1 Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 11 Jul 2022 14:37:18 -0400 Subject: [PATCH 1/4] fix(file-reporter): output logs incrementally --- packages/file-reporter/src/index.ts | 51 +++++++------------- packages/file-reporter/test/fixture/1/stdout | 14 +++--- 2 files changed, 26 insertions(+), 39 deletions(-) diff --git a/packages/file-reporter/src/index.ts b/packages/file-reporter/src/index.ts index fd7dc4b25db..2236bfbc5fc 100644 --- a/packages/file-reporter/src/index.ts +++ b/packages/file-reporter/src/index.ts @@ -4,49 +4,34 @@ import fs from 'graceful-fs' const LOG_FILENAME = 'node_modules/.pnpm-debug.log' export default function (streamParser: Object) { - const logs: Object[] = [] + let logNum = 0 - streamParser['on']('data', function (logObj: Object) { - if (isUsefulLog(logObj)) { - logs.push(logObj) - } - }) + // Clean up previous log files + if (global['writeDebugLogFile'] !== false) { + if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) + if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) + } - process.on('exit', (code: number) => { - if (code === 0 || global['writeDebugLogFile'] === false) { - // it might not exist, so it is OK if it fails - try { - fs.unlinkSync(LOG_FILENAME) - } catch (err) {} - return + streamParser['on']('data', function (logObj: Object) { + if (isUsefulLog(logObj) && global['writeDebugLogFile'] !== false) { + const msgobj = getMessageObj(logObj) + const prettyLogs = prettify(msgobj) + const jsonLogs = JSON.stringify(prettyLogs, null, 2) + '\n' + const dest = fs.existsSync(path.dirname(LOG_FILENAME)) ? LOG_FILENAME : path.basename(LOG_FILENAME) + fs.appendFileSync(dest, jsonLogs) + logNum++ } - - const prettyLogs = getPrettyLogs() - const jsonLogs = JSON.stringify(prettyLogs, null, 2) - // Don't create a node_modules directory if it does not exist - const dest = fs.existsSync(path.dirname(LOG_FILENAME)) - ? LOG_FILENAME - : path.basename(LOG_FILENAME) - fs.writeFileSync(dest, jsonLogs, 'utf-8') }) - function getPrettyLogs () { - const prettyLogs = {} - logs.forEach((logObj, i) => { - // eslint-disable-next-line - const key = `${i} ${logObj['level']} ${logObj['name']}` - const msgobj = getMessageObj(logObj) - prettyLogs[key] = prettify(msgobj) - }) - return prettyLogs - } - function getMessageObj (logobj: Object): Object { - const msgobj = {} + const msgobj: { [key: string]: string } = {} for (const key in logobj) { if (['time', 'hostname', 'pid', 'level', 'name'].includes(key)) continue msgobj[key] = logobj[key] } + const logLevel: string = logobj['level'] + const logName: string = logobj['name'] + msgobj.key = `${logNum} ${logLevel} ${logName}` return msgobj } diff --git a/packages/file-reporter/test/fixture/1/stdout b/packages/file-reporter/test/fixture/1/stdout index d3d9966885a..c9f8815d4d6 100644 --- a/packages/file-reporter/test/fixture/1/stdout +++ b/packages/file-reporter/test/fixture/1/stdout @@ -1,7 +1,9 @@ { - "0 info pnpm": "foo", - "1 warn pnpm": { - "foo": 1, - "bar": 2 - } -} \ No newline at end of file + "message": "foo", + "key": "0 info pnpm" +} +{ + "foo": 1, + "bar": 2, + "key": "1 warn pnpm" +} From ff727f2efa2103f613f57830835ae8b6c2848663 Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 11 Jul 2022 16:39:42 -0400 Subject: [PATCH 2/4] fix(file-reporter): remove file if successful --- packages/file-reporter/src/index.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/file-reporter/src/index.ts b/packages/file-reporter/src/index.ts index 2236bfbc5fc..0c8d599324d 100644 --- a/packages/file-reporter/src/index.ts +++ b/packages/file-reporter/src/index.ts @@ -7,10 +7,8 @@ export default function (streamParser: Object) { let logNum = 0 // Clean up previous log files - if (global['writeDebugLogFile'] !== false) { - if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) - if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) - } + if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) + if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) streamParser['on']('data', function (logObj: Object) { if (isUsefulLog(logObj) && global['writeDebugLogFile'] !== false) { @@ -23,6 +21,16 @@ export default function (streamParser: Object) { } }) + process.on('exit', (code: number) => { + if (code === 0 || global['writeDebugLogFile'] === false) { + // it might not exist, so it is OK if it fails + try { + if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) + if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) + } catch (err) {} + } + }) + function getMessageObj (logobj: Object): Object { const msgobj: { [key: string]: string } = {} for (const key in logobj) { From ba20445a869429c5397b08905da3039e81874647 Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 11 Jul 2022 16:41:07 -0400 Subject: [PATCH 3/4] chore: clean up code --- packages/file-reporter/src/index.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/file-reporter/src/index.ts b/packages/file-reporter/src/index.ts index 0c8d599324d..589600d2041 100644 --- a/packages/file-reporter/src/index.ts +++ b/packages/file-reporter/src/index.ts @@ -24,10 +24,8 @@ export default function (streamParser: Object) { process.on('exit', (code: number) => { if (code === 0 || global['writeDebugLogFile'] === false) { // it might not exist, so it is OK if it fails - try { - if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) - if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) - } catch (err) {} + if (fs.existsSync(LOG_FILENAME)) fs.unlinkSync(LOG_FILENAME) + if (fs.existsSync(path.basename(LOG_FILENAME))) fs.unlinkSync(path.basename(LOG_FILENAME)) } }) From 62151efda78c6619973c05aa0be646702348f08b Mon Sep 17 00:00:00 2001 From: William Huang Date: Mon, 11 Jul 2022 16:56:29 -0400 Subject: [PATCH 4/4] chore: add changeset --- .changeset/wise-cameras-yawn.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/wise-cameras-yawn.md diff --git a/.changeset/wise-cameras-yawn.md b/.changeset/wise-cameras-yawn.md new file mode 100644 index 00000000000..8543a436cab --- /dev/null +++ b/.changeset/wise-cameras-yawn.md @@ -0,0 +1,5 @@ +--- +"@pnpm/file-reporter": minor +--- + +Logs will be added as they are created instead of all at once at the end. Prevents memory exceeded errors if there are too many logs.