Tests: Added .html.test files for replace .js language tests (#3148)

package.json

@@ -61,6 +61,7 @@
 		"mocha": "^6.2.0",
 		"node-fetch": "^2.6.0",
 		"npm-run-all": "^4.1.5",
+		"prettier": "^2.4.1",
 		"pump": "^3.0.0",
 		"refa": "^0.9.1",
 		"regexp-ast-analysis": "^0.2.4",

test-suite.html

@@ -165,14 +165,21 @@ <h2 id="writing-tests-explaining-the-simplified-token-stream">Explaining the sim
 <section id="writing-specific-tests">
 	<h1>Writing specific tests</h1>
 
-	<p>Sometimes, using the token stream tests is not powerful enough. By creating a test file with the file extension <code>.js</code> instead of <code>.test</code>, you can make Prism highlight arbitrary pieces of code and check their HTML results.</p>
+	<p>Sometimes, using the token stream tests is not powerful enough. By creating a test file with the file extension <code>.html.test</code> instead of <code>.test</code>, you can make Prism highlight arbitrary pieces of code and check their HTML results.</p>
 	<p>The language is determined by the folder containing the test file lies, as explained in the previous section.</p>
 	<p>The structure of your test file will look like this, for example:</p>
-	<pre><code>module.exports = {
-	'&amp;#x278a;': '&lt;span class="token entity" title="&amp;#x278a;">&amp;amp;#x278a;&lt;/span>',
-	'&amp;#182;': '&lt;span class="token entity" title="&amp;#182;">&amp;amp;#182;&lt;/span>',
-};</code></pre>
-	<p>The keys are the codes which will be highlighted by Prism. The values are the expected results, as HTML.</p>
+	<pre><code class="language-html">&amp;amp;
+&amp;#x41;
+
+----------------------------------------------------
+
+&lt;span class="token entity named-entity" title="&amp;amp;">&amp;amp;amp;&lt;/span>
+&lt;span class="token entity" title="&amp;#x41;">&amp;amp;#x41;&lt;/span>
+
+----------------------------------------------------
+
+This is a comment explaining this test case.
+</code></pre>
 </section>
 
 <section id="test-runner-tests">

tests/helper/test-case.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const { assert } = require('chai');
+const Prettier = require('prettier');
 const PrismLoader = require('./prism-loader');
 const TokenStreamTransformer = require('./token-stream-transformer');
 
@@ -217,6 +218,81 @@ class TokenizeJSONRunner {
 	}
 }
 
+/**
+ * @implements {Runner<string>}
+ */
+class HighlightHTMLRunner {
+	/**
+	 * @param {Prism} Prism
+	 * @param {string} code
+	 * @param {string} language
+	 * @returns {string}
+	 */
+	run(Prism, code, language) {
+		const env = {
+			element: {},
+			language,
+			grammar: Prism.languages[language],
+			code,
+		};
+
+		Prism.hooks.run('before-highlight', env);
+		env.highlightedCode = Prism.highlight(env.code, env.grammar, env.language);
+		Prism.hooks.run('before-insert', env);
+		env.element.innerHTML = env.highlightedCode;
+		Prism.hooks.run('after-highlight', env);
+		Prism.hooks.run('complete', env);
+
+		return env.highlightedCode;
+	}
+	/**
+	 * @param {string} actual
+	 * @returns {string}
+	 */
+	print(actual) {
+		return Prettier.format(actual, {
+			printWidth: 100,
+			tabWidth: 4,
+			useTabs: true,
+			htmlWhitespaceSensitivity: 'ignore',
+			filepath: 'fake.html',
+		});
+	}
+	/**
+	 * @param {string} actual
+	 * @param {string} expected
+	 * @returns {boolean}
+	 */
+	isEqual(actual, expected) {
+		return this.normalize(actual) === this.normalize(expected);
+	}
+	/**
+	 * @param {string} actual
+	 * @param {string} expected
+	 * @param {(firstDifference: number) => string} message
+	 * @returns {void}
+	 */
+	assertEqual(actual, expected, message) {
+		// We don't calculate the index of the first difference because it's difficult.
+		assert.deepEqual(this.normalize(actual), this.normalize(expected), message(0));
+	}
+
+	/**
+	 * Normalizes the given HTML by removing all leading spaces and trailing spaces. Line breaks will also be normalized
+	 * to enable good diffing.
+	 *
+	 * @param {string} html
+	 * @returns {string}
+	 */
+	normalize(html) {
+		return html
+			.replace(/</g, '\n<')
+			.replace(/>/g, '>\n')
+			.replace(/[ \t]*[\r\n]\s*/g, '\n')
+			.trim();
+	}
+}
+
 
 module.exports = {
 	TestCaseFile,
@@ -238,7 +314,11 @@ module.exports = {
 	 * @param {"none" | "insert" | "update"} updateMode
 	 */
 	runTestCase(languageIdentifier, filePath, updateMode) {
-		this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new TokenizeJSONRunner());
+		if (/\.html\.test$/i.test(filePath)) {
+			this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new HighlightHTMLRunner());
+		} else {
+			this.runTestCaseWithRunner(languageIdentifier, filePath, updateMode, new TokenizeJSONRunner());
+		}
 	},
 
 	/**
@@ -347,38 +427,6 @@ module.exports = {
 			mainLanguage: mainLanguage
 		};
 	},
-
-	/**
-	 * Runs the given pieces of codes and asserts their result.
-	 *
-	 * Code is provided as the key and expected result as the value.
-	 *
-	 * @param {string} languageIdentifier
-	 * @param {object} codes
-	 */
-	runTestsWithHooks(languageIdentifier, codes) {
-		const usedLanguages = this.parseLanguageNames(languageIdentifier);
-		const Prism = PrismLoader.createInstance(usedLanguages.languages);
-		// the first language is the main language to highlight
-
-		for (const code in codes) {
-			if (codes.hasOwnProperty(code)) {
-				const env = {
-					element: {},
-					language: usedLanguages.mainLanguage,
-					grammar: Prism.languages[usedLanguages.mainLanguage],
-					code: code
-				};
-				Prism.hooks.run('before-highlight', env);
-				env.highlightedCode = Prism.highlight(env.code, env.grammar, env.language);
-				Prism.hooks.run('before-insert', env);
-				env.element.innerHTML = env.highlightedCode;
-				Prism.hooks.run('after-highlight', env);
-				Prism.hooks.run('complete', env);
-				assert.equal(env.highlightedCode, codes[code]);
-			}
-		}
-	}
 };
 
 /**

tests/helper/test-discovery.js

@@ -3,8 +3,6 @@
 const fs = require('fs');
 const path = require('path');
 
-const SUPPORTED_TEST_FILE_EXT = new Set(['.js', '.test']);
-
 module.exports = {
 
 	/**
@@ -95,7 +93,7 @@ module.exports = {
 	getAllFiles(src) {
 		return fs.readdirSync(src)
 			.filter(fileName => {
-				return SUPPORTED_TEST_FILE_EXT.has(path.extname(fileName))
+				return path.extname(fileName) === '.test'
 					&& fs.statSync(path.join(src, fileName)).isFile();
 			})
 			.map(fileName => {

tests/languages/asciidoc/entity_feature.html.test

@@ -0,0 +1,7 @@
+➊
+¶
+
+----------------------------------------------------
+
+<span class="token entity" title="&#x278a;">&amp;#x278a;</span>
+<span class="token entity" title="&#182;">&amp;#182;</span>

tests/languages/groovy/issue1049.html.test

@@ -0,0 +1,15 @@
+"&"
+"&&"
+"<"
+"<<"
+"<"
+">"
+
+----------------------------------------------------
+
+<span class="token string gstring">"&amp;amp;"</span>
+<span class="token string gstring">"&amp;amp;&amp;amp;"</span>
+<span class="token string gstring">"&amp;lt;"</span>
+<span class="token string gstring">"&amp;lt;&amp;lt;"</span>
+<span class="token string gstring">"&amp;amp;lt;"</span>
+<span class="token string gstring">"&amp;gt;"</span>

tests/languages/groovy/string-interpolation_feature.html.test

@@ -0,0 +1,133 @@
+// Double quoted: interpolation
+"$foo"
+"${42}"
+
+// Triple double quoted: interpolation
+"""$foo"""
+"""${42}"""
+
+// Slashy string: interpolation
+/$foo/
+/${42}/
+
+// Dollar slashy string: interpolation
+$/$foo/$
+$/${42}/$
+
+// Double quoted: no interpolation (escaped)
+"\$foo \${42}"
+
+// Triple double quoted: no interpolation (escaped)
+"""\$foo \${42}"""
+
+// Slashy string: no interpolation (escaped)
+/\$foo \${42}/
+
+// Dollar slashy string: no interpolation (escaped)
+$/$$foo $${42}/$
+
+// Single quoted string: no interpolation
+'$foo ${42}'
+
+// Triple single quoted string: no interpolation
+'''$foo ${42}'''
+
+----------------------------------------------------
+
+<span class="token comment">// Double quoted: interpolation</span>
+<span class="token string gstring">
+	"
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		foo
+	</span>
+	"
+</span>
+<span class="token string gstring">
+	"
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		<span class="token punctuation">{</span>
+		<span class="token number">42</span>
+		<span class="token punctuation">}</span>
+	</span>
+	"
+</span>
+
+<span class="token comment">// Triple double quoted: interpolation</span>
+<span class="token string gstring">
+	"""
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		foo
+	</span>
+	"""
+</span>
+<span class="token string gstring">
+	"""
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		<span class="token punctuation">{</span>
+		<span class="token number">42</span>
+		<span class="token punctuation">}</span>
+	</span>
+	"""
+</span>
+
+<span class="token comment">// Slashy string: interpolation</span>
+<span class="token string regex">
+	/
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		foo
+	</span>
+	/
+</span>
+<span class="token string regex">
+	/
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		<span class="token punctuation">{</span>
+		<span class="token number">42</span>
+		<span class="token punctuation">}</span>
+	</span>
+	/
+</span>
+
+<span class="token comment">// Dollar slashy string: interpolation</span>
+<span class="token string gstring">
+	$/
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		foo
+	</span>
+	/$
+</span>
+<span class="token string gstring">
+	$/
+	<span class="token expression">
+		<span class="token punctuation">$</span>
+		<span class="token punctuation">{</span>
+		<span class="token number">42</span>
+		<span class="token punctuation">}</span>
+	</span>
+	/$
+</span>
+
+<span class="token comment">// Double quoted: no interpolation (escaped)</span>
+<span class="token string gstring">"\$foo \${42}"</span>
+
+<span class="token comment">// Triple double quoted: no interpolation (escaped)</span>
+<span class="token string gstring">"""\$foo \${42}"""</span>
+
+<span class="token comment">// Slashy string: no interpolation (escaped)</span>
+<span class="token string regex">/\$foo \${42}/</span>
+
+<span class="token comment">// Dollar slashy string: no interpolation (escaped)</span>
+<span class="token string gstring">$/$$foo $${42}/$</span>
+
+<span class="token comment">// Single quoted string: no interpolation</span>
+<span class="token string">'$foo ${42}'</span>
+
+<span class="token comment">// Triple single quoted string: no interpolation</span>
+<span class="token string">'''$foo ${42}'''</span>