HTTP: Added support for special header value tokenization (#3275)

components.json

@@ -559,7 +559,10 @@
 		"http": {
 			"title": "HTTP",
 			"optional": [
+				"csp",
 				"css",
+				"hpkp",
+				"hsts",
 				"javascript",
 				"json",
 				"markup",

components/prism-http.js

@@ -1,4 +1,13 @@
 (function (Prism) {
+
+	/**
+	 * @param {string} name
+	 * @returns {RegExp}
+	 */
+	function headerValueOf(name) {
+		return RegExp('(^(?:' + name + '):[ \t]*(?![ \t]))[^]+', 'i');
+	}
+
 	Prism.languages.http = {
 		'request-line': {
 			pattern: /^(?:CONNECT|DELETE|GET|HEAD|OPTIONS|PATCH|POST|PRI|PUT|SEARCH|TRACE)\s(?:https?:\/\/|\/)\S*\sHTTP\/[\d.]+/m,
@@ -45,10 +54,39 @@
 				}
 			}
 		},
-		// HTTP header name
-		'header-name': {
-			pattern: /^[\w-]+:(?=.)/m,
-			alias: 'keyword'
+		'header': {
+			pattern: /^[\w-]+:.+(?:(?:\r\n?|\n)[ \t].+)*/m,
+			inside: {
+				'header-value': [
+					{
+						pattern: headerValueOf(/Content-Security-Policy/.source),
+						lookbehind: true,
+						alias: ['csp', 'languages-csp'],
+						inside: Prism.languages.csp
+					},
+					{
+						pattern: headerValueOf(/Public-Key-Pins(?:-Report-Only)?/.source),
+						lookbehind: true,
+						alias: ['hpkp', 'languages-hpkp'],
+						inside: Prism.languages.hpkp
+					},
+					{
+						pattern: headerValueOf(/Strict-Transport-Security/.source),
+						lookbehind: true,
+						alias: ['hsts', 'languages-hsts'],
+						inside: Prism.languages.hsts
+					},
+					{
+						pattern: headerValueOf(/[^:]+/.source),
+						lookbehind: true
+					}
+				],
+				'header-name': {
+					pattern: /^[^:]+/,
+					alias: 'keyword'
+				},
+				'punctuation': /^:/
+			}
 		}
 	};
 
@@ -60,7 +98,8 @@
 		'application/xml': langs.xml,
 		'text/xml': langs.xml,
 		'text/html': langs.html,
-		'text/css': langs.css
+		'text/css': langs.css,
+		'text/plain': langs.plain
 	};
 
 	// Declare which types can also be suffixes
@@ -97,7 +136,7 @@
 					// However, when writing code by hand (e.g. to display on a website) people can forget about this,
 					// so we want to be liberal here. We will allow the empty line to be omitted if the first line of
 					// the body does not start with a [\w-] character (as headers do).
-					/[^\w-][\s\S]*/.source,
+					/[^ \t\w-][\s\S]*/.source,
 					'i'
 				),
 				lookbehind: true,
@@ -106,7 +145,7 @@
 		}
 	}
 	if (options) {
-		Prism.languages.insertBefore('http', 'header-name', options);
+		Prism.languages.insertBefore('http', 'header', options);
 	}
 
 }(Prism));

tests/languages/css+http/css_inclusion.test

@@ -7,8 +7,11 @@ a.link:hover {
 ----------------------------------------------------
 
 [
-	["header-name", "Content-type:"],
-	" text/css\r\n",
+	["header", [
+		["header-name", "Content-type"],
+		["punctuation", ":"],
+		["header-value", "text/css"]
+	]],
 
 	["text-css", [
 		["selector", "a.link:hover"],

tests/languages/http!+csp/inclusion.test

@@ -0,0 +1,20 @@
+Content-Security-Policy: default-src 'none'; style-src cdn.example.com; report-uri /_/csp-reports
+
+----------------------------------------------------
+
+[
+	["header", [
+		["header-name", "Content-Security-Policy"],
+		["punctuation", ":"],
+		["header-value", [
+			["directive", "default-src"],
+			["none", "'none'"],
+			["punctuation", ";"],
+			["directive", "style-src"],
+			["host", ["cdn.example.com"]],
+			["punctuation", ";"],
+			["directive", "report-uri"],
+			" /_/csp-reports"
+		]]
+	]]
+]

tests/languages/http!+hpkp/inclusion.test

@@ -0,0 +1,31 @@
+Public-Key-Pins: max-age=3000;
+       pin-sha256="d6qzRu9zOECb90Uez27xWltNsj0e1Md7GkYYkVoZWmM=";
+       pin-sha256="E9CZ9INDbd+2eRQozYqqbQ2yXLVKB9+xcprMF+44U1g="
+
+----------------------------------------------------
+
+[
+	["header", [
+		["header-name", "Public-Key-Pins"],
+		["punctuation", ":"],
+		["header-value", [
+			["directive", "max-age"],
+			["operator", "="],
+			"3000",
+			["punctuation", ";"],
+
+			["directive", "pin-sha256"],
+			["operator", "="],
+			"\"d6qzRu9zOECb90Uez27xWltNsj0e1Md7GkYYkVoZWmM",
+			["operator", "="],
+			"\"",
+			["punctuation", ";"],
+
+			["directive", "pin-sha256"],
+			["operator", "="],
+			"\"E9CZ9INDbd+2eRQozYqqbQ2yXLVKB9+xcprMF+44U1g",
+			["operator", "="],
+			"\""
+		]]
+	]]
+]

tests/languages/http!+hsts/inclusion.test

@@ -0,0 +1,15 @@
+Strict-Transport-Security: max-age=31536000
+
+----------------------------------------------------
+
+[
+	["header", [
+		["header-name", "Strict-Transport-Security"],
+		["punctuation", ":"],
+		["header-value", [
+			["directive", "max-age"],
+			["operator", "="],
+			"31536000"
+		]]
+	]]
+]

tests/languages/http/header_feature.test

@@ -0,0 +1,59 @@
+Accept-Language: fr,fr-fr;q=0.8,en-us;q=0.5,en;q=0.3
+Accept-Encoding: gzip, deflate
+Server: GitHub.com
+Date: Mon, 22 Dec 2014 18:25:30 GMT
+Content-Type: text/html; charset=utf-8
+Content-Security-Policy: default-src 'none'; style-src cdn.example.com; report-uri /_/csp-reports
+Public-Key-Pins: max-age=3000;
+       pin-sha256="d6qzRu9zOECb90Uez27xWltNsj0e1Md7GkYYkVoZWmM=";
+       pin-sha256="E9CZ9INDbd+2eRQozYqqbQ2yXLVKB9+xcprMF+44U1g="
+Strict-Transport-Security: max-age=31536000
+
+----------------------------------------------------
+
+[
+	["header", [
+		["header-name", "Accept-Language"],
+		["punctuation", ":"],
+		["header-value", "fr,fr-fr;q=0.8,en-us;q=0.5,en;q=0.3"]
+	]],
+	["header", [
+		["header-name", "Accept-Encoding"],
+		["punctuation", ":"],
+		["header-value", "gzip, deflate"]
+	]],
+	["header", [
+		["header-name", "Server"],
+		["punctuation", ":"],
+		["header-value", "GitHub.com"]
+	]],
+	["header", [
+		["header-name", "Date"],
+		["punctuation", ":"],
+		["header-value", "Mon, 22 Dec 2014 18:25:30 GMT"]
+	]],
+	["header", [
+		["header-name", "Content-Type"],
+		["punctuation", ":"],
+		["header-value", "text/html; charset=utf-8"]
+	]],
+	["header", [
+		["header-name", "Content-Security-Policy"],
+		["punctuation", ":"],
+		["header-value", "default-src 'none'; style-src cdn.example.com; report-uri /_/csp-reports"]
+	]],
+	["header", [
+		["header-name", "Public-Key-Pins"],
+		["punctuation", ":"],
+		["header-value", "max-age=3000;\r\n       pin-sha256=\"d6qzRu9zOECb90Uez27xWltNsj0e1Md7GkYYkVoZWmM=\";\r\n       pin-sha256=\"E9CZ9INDbd+2eRQozYqqbQ2yXLVKB9+xcprMF+44U1g=\""]
+	]],
+	["header", [
+		["header-name", "Strict-Transport-Security"],
+		["punctuation", ":"],
+		["header-value", "max-age=31536000"]
+	]]
+]
+
+----------------------------------------------------
+
+Checks for header names.

tests/languages/http/text-plain_feature.test

@@ -0,0 +1,14 @@
+Content-Type: text/plain
+
+Hello World!
+
+----------------------------------------------------
+
+[
+	["header", [
+		["header-name", "Content-Type"],
+		["punctuation", ":"],
+		["header-value", "text/plain"]
+	]],
+	["text-plain", ["\r\nHello World!"]]
+]

tests/languages/javascript+http/issue2733.test

@@ -30,20 +30,35 @@ transfer-encoding: chunked
 		["reason-phrase", "OK"]
 	]],
 
-	["header-name", "connection:"],
-	" keep-alive\r\n",
+	["header", [
+		["header-name", "connection"],
+		["punctuation", ":"],
+		["header-value", "keep-alive"]
+	]],
 
-	["header-name", "content-type:"],
-	" application/json\r\n",
+	["header", [
+		["header-name", "content-type"],
+		["punctuation", ":"],
+		["header-value", "application/json"]
+	]],
 
-	["header-name", "date:"],
-	" Sat, 23 Jan 2021 20:36:14 GMT\r\n",
+	["header", [
+		["header-name", "date"],
+		["punctuation", ":"],
+		["header-value", "Sat, 23 Jan 2021 20:36:14 GMT"]
+	]],
 
-	["header-name", "keep-alive:"],
-	" timeout=60\r\n",
+	["header", [
+		["header-name", "keep-alive"],
+		["punctuation", ":"],
+		["header-value", "timeout=60"]
+	]],
 
-	["header-name", "transfer-encoding:"],
-	" chunked\r\n",
+	["header", [
+		["header-name", "transfer-encoding"],
+		["punctuation", ":"],
+		["header-value", "chunked"]
+	]],
 
 	["application-json", [
 		["punctuation", "{"],

tests/languages/javascript+http/javascript_inclusion.test

@@ -5,9 +5,11 @@ var a = true;
 ----------------------------------------------------
 
 [
-	["header-name", "Content-type:"],
-	" application/javascript\r\n",
-
+	["header", [
+		["header-name", "Content-type"],
+		["punctuation", ":"],
+		["header-value", "application/javascript"]
+	]],
 	["application-javascript", [
 		["keyword", "var"],
 		" a ",

tests/languages/json+http/issue2733.test

@@ -30,20 +30,35 @@ transfer-encoding: chunked
 		["reason-phrase", "OK"]
 	]],
 
-	["header-name", "connection:"],
-	" keep-alive\r\n",
+	["header", [
+		["header-name", "connection"],
+		["punctuation", ":"],
+		["header-value", "keep-alive"]
+	]],
 
-	["header-name", "content-type:"],
-	" application/json\r\n",
+	["header", [
+		["header-name", "content-type"],
+		["punctuation", ":"],
+		["header-value", "application/json"]
+	]],
 
-	["header-name", "date:"],
-	" Sat, 23 Jan 2021 20:36:14 GMT\r\n",
+	["header", [
+		["header-name", "date"],
+		["punctuation", ":"],
+		["header-value", "Sat, 23 Jan 2021 20:36:14 GMT"]
+	]],
 
-	["header-name", "keep-alive:"],
-	" timeout=60\r\n",
+	["header", [
+		["header-name", "keep-alive"],
+		["punctuation", ":"],
+		["header-value", "timeout=60"]
+	]],
 
-	["header-name", "transfer-encoding:"],
-	" chunked\r\n",
+	["header", [
+		["header-name", "transfer-encoding"],
+		["punctuation", ":"],
+		["header-value", "chunked"]
+	]],
 
 	["application-json", [
 		["punctuation", "{"],