Added extra newlines in pretty tokenstream

tests/helper/token-stream-transformer.js

@@ -6,6 +6,8 @@ module.exports = {
 	 * @typedef TokenStreamItem
 	 * @property {string} type
 	 * @property {string | TokenStreamItem | Array<string|TokenStreamItem>} content
+	 *
+	 * @typedef {Array<string | [string, string | Array]>} SimplifiedTokenStream
 	 */
 
 	/**
@@ -15,29 +17,46 @@ module.exports = {
 	 * * In arrays each value is transformed individually
 	 * * Values that are empty (empty arrays or strings only containing whitespace)
 	 *
-	 * @param {string | TokenStreamItem | Array<string|TokenStreamItem>} tokenStream
-	 * @returns {Array<string|Array<string|any[]>>}
+	 * @param {Array<string|TokenStreamItem>} tokenStream
+	 * @returns {SimplifiedTokenStream}
 	 */
-	simplify(tokenStream) {
-		if (Array.isArray(tokenStream)) {
-			return tokenStream
-				.map(value => this.simplify(value))
-				.filter(value => {
-					return !(Array.isArray(value) && !value.length) && !(typeof value === "string" && !value.trim().length);
-				});
-		}
-		else if (typeof tokenStream === "object") {
-			return [tokenStream.type, this.simplify(tokenStream.content)];
-		}
-		else {
-			return tokenStream;
+	simplify: function simplify(tokenStream) {
+		return tokenStream
+			.map(innerSimple)
+			.filter((value, i, arr) => {
+				if (typeof value === "string" && !value.trim().length) {
+					// string contains only spaces
+					if (i > 0 && i < arr.length - 1 && value.split(/\r\n?|\n/g).length > 2) {
+						// in a valid token stream there are no adjacent strings, so we know that the previous
+						// element is a (simplified) token
+						arr[i - 1]['newline-after'] = true;
+					}
+					return false;
+				}
+				return true;
+			});
+
+		/**
+		 * @param {string | TokenStreamItem} value
+		 * @returns {string | [string, string | Array]}
+		 */
+		function innerSimple(value) {
+			if (typeof value === "object") {
+				if (Array.isArray(value.content)) {
+					return [value.type, simplify(value.content)];
+				} else {
+					return [value.type, innerSimple(value.content)];
+				}
+			} else {
+				return value;
+			}
 		}
 	},
 
 	/**
 	 *
-	 * @param {ReadonlyArray<string|ReadonlyArray<string|any[]>>} tokenStream
-	 * @param {number} [indentationLevel=0]
+	 * @param {Readonly<SimplifiedTokenStream>} tokenStream
+	 * @param {number} [indentationLevel]
 	 */
 	prettyprint(tokenStream, indentationLevel = 1) {
 		const indentChar = '    ';
@@ -49,6 +68,7 @@ module.exports = {
 		out += "[\n"
 		tokenStream.forEach((item, i) => {
 			out += indentation;
+			let extraNewline = false;
 
 			if (typeof item === 'string') {
 				out += JSON.stringify(item);
@@ -65,10 +85,16 @@ module.exports = {
 				}
 
 				out += ']';
+
+				extraNewline = !!item['newline-after'];
 			}
 
 			const lineEnd = (i === tokenStream.length - 1) ? '\n' : ',\n';
 			out += lineEnd;
+
+			if (extraNewline) {
+				out += '\n';
+			}
 		})
 		out += indentation.substr(indentChar.length) + ']'
 		return out;