From b661fc94fa968ab7bd550924ab77ded9198c57ed Mon Sep 17 00:00:00 2001 From: Wei Ting Date: Sun, 3 Oct 2021 21:56:34 +0800 Subject: [PATCH 01/18] Document standard tokens --- tokens.html | 313 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 tokens.html diff --git a/tokens.html b/tokens.html new file mode 100644 index 0000000000..7ab5562ba7 --- /dev/null +++ b/tokens.html @@ -0,0 +1,313 @@ + + + + + + +Prism Tokens ▲ Prism + + + + + + + + + + + + +
+
+ +

Prism tokens

+

Prism identifies tokens in your code, which are in turn styled by CSS to produce the syntax highlighting. This page provides an overview of the standard tokens and corresponding examples.

+
+ +
+

Standard tokens

+ +

When defining a new language, you will need to provide token names for each 'type' of code, such as keywords and operators, so Prism can highlight code accordingly. It is recommended to make use of the following standard tokens to ensure that highlight code will be highlighted, as Prism's themes (both official and non-official) only guarantee coverage for these standard tokens.

+ +
+
keyword
+
Pre-defined and reserved words. + 
for (let x of y) {
+	// Code here
+	return z;
+}
+ +
builtin
+
Functions/Methods that are available out of the box. + 
pi = round(float('3.14159'), 2)
+ +
class-name
+
The name of a class. + 
class Rectangle extends Square {
+	constructor(length, breadth) {
+		super(length);
+		this.breadth = breadth;
+	}
+}
+ +
function
+
The name of a function. + 
function isEven(number) {
+	return Number(number) % 2 === 0;
+}
+
+function isOdd(number) {
+	return !isEven(number);
+}
+ +
boolean
+
True and false, and pairs with similar concepts. + 
console.log(true === false); // prints false
+console.log(true === !false); // prints true
+ +
number
+
A numerical value, regardless of base and order, and no matter real or imaginary. + 
7
+2147483647
+0o177
+0b100110111
+3
+79228162514264337593543950336
+0o377
+0x100000000
+0xdeadbeef
+3.14
+10.
+.001
+1e100
+3.14e-10
+0e0
+3.14j
+10.j
+10j
+.001j
+1e100j
+3.14e-10j
+ +
string
+
Literal text, including numbers and symbols and maybe even more special characters. + 
let greeting = 'Hello World!';
+ +
char
+
A string that can comprise only a single character, enforced by the language. + 
'A'
+'z'
+'0'
+'-'
+'\t'
+'\u{2728}'
+ +
symbol
+
A primitive data type found in some languages, can be thought of as an identifier. + 
#myFirstSymbol "#myFirstSymbol is a symbol in Smalltalk and is the same object as all other #myFirstSymbol symbols"
+ +
regex
+
A regular expression. + 
let regexPattern = /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/;
+ +
url
+
A link to another page or resource. + 
body {
+	background: url(foo.png);
+}
+ 
[Prism](https://prismjs.com) is a cool syntax highlighter.
+ +
operator
+
A symbol that represents an action or process, whether it's a mathematical operation, logical operation, and so on. + 
x = y
+x *= y
+x !== y
+x++
+-y
+x >>> y
+x || y
+z ? x : y
+ +
variable
+
An object that is assigned and holds a specific value. + 
@nice-blue: #5B83AD;
+@light-blue: lighten(@nice-blue, 20%);
+ +
constant
+
The name of a a variable that cannot be modified later on. + 
const PI = 3.14159;
+ +
property
+
An attribute/characteristic. + 
body {
+	color: red;
+	line-height: normal;
+}
+ 
{
+	"data": {
+		"labels": [
+			"foo",
+			"bar"
+		],
+		"series": [
+			[ 0, 1, 2, 3 ],
+			[ 0, -4, -8, -12 ]
+		]
+	},
+	// we even support comments
+	"error": null,
+	"status": "Ok"
+}
+ +
punctuation
+
Punctuation that is not contained within a string, such as brackets. + 
def median(pool):
+	'''Statistical median to demonstrate doctest.
+	>>> median([2, 9, 9, 7, 9, 2, 4, 5, 8])
+	7
+	'''
+	copy = sorted(pool)
+	size = len(copy)
+	if size % 2 == 1:
+		return copy[(size - 1) / 2]
+	else:
+		return (copy[size/2 - 1] + copy[size/2]) / 2
+ +
important
+
Anything that is important. + 
body {
+	color: red !important;
+}
+ 
# This is a heading. Headings are important.
+ +
comment
+
Code comments. + 
<!-- Here's a comment -->
+<style>
+	/* Here's another comment */
+</style>
+<script>
+// Here's yet another comment
+</script>
+ +
inserted
+
Added or modified line, mainly for diffs. In general, also the idea of something being increased. + 
+		// Calculate the area of the circle
+		let area = PI * (radius ** 2);
+ +
deleted
+
Deleted line, mainly for diffs. In general, also the idea of something being decreased/removed. + 
-		// TODO: Remove this comment
+		let foo = 'bar';
+ +
bold
+
Bolded text. Mostly found in document-markup languages. + 
**I am bolded text!**
+ +
italic
+
Italicised text. Mostly found in document-markup languages. + 
*I am italicised text!*
+ +
tag
+
The markup version of a keyword, to indicate when an element begins and ends. + 
<p>Hello World!</p>
+ +
attr-name
+
Kind of like a property of a markup tag. + 
<video allowfullscreen controls>
+	<source src="hello_world.mp4" type="video/mp4" />
+</video>
+ +
attr-value
+
The value or argument that is passed to a property in a markup tag. + 
<p id="greeting">Hello World!</p>
+ +
namespace
+
Used to provide uniquely named elements and attributes in XML documents. + 
<html:p foo:bar="baz" foo:weee></html:p>
+ +
prolog
+
The first part of an XML document. + 
<?xml version="1.0" encoding="utf-8"?>
+<svg></svg>
+ +
doctype
+
Document type declaration, specific to markup languages. + 
<!DOCTYPE html>
+<html></html>
+ +
cdata
+
Character data, specific to markup languages. Deprecated as of DOM4. + 
<ns1:description><![CDATA[
+  CDATA is <not> magical.
+]]></ns1:description>
+ +
entity
+
Code used to display reserved characters in markup languages. + 
&amp; &#x2665; &#160; &#x152;
+ +
atrule
+
Literally @ rules (statements) in stylesheets. + 
@font-family {
+	font-family: Questrial;
+	src: url(questrial.otf);
+}
+
+@media screen and (min-width: 768px) {
+	/* rules here */
+}
+ +
selector
+
Code that identifies or picks something out of a group to operate on, such as the names of HTML elements in stylesheets. + 
section h1,
+#features li strong,
+header h2,
+footer p {
+	/* styles here */
+}
+
+
+ +
+

Embedded languages

+ +

In addition to the standard tokens above, Prism also has a token for languages that are embedded in another language, such as CSS in HTML, JS in HTML, Bash in Shell-session, and CSS in JS, allowing Prism to highlight the tokens in the embedded languages more accurately. All embedded languages are wrapped in their own special token, which includes a CSS class language-xxxx corresponding to the embedded language.

+ +

Open your browser's developer tools and check out the example below to see it in action!

+ + 
<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="utf-8" />
+	<title>I can haz embedded CSS and JS</title>
+	<style>
+		@media print {
+			p { color: red !important; }
+		}
+	</style>
+</head>
+<body>
+	<h1>I can haz embedded CSS and JS</h1>
+	<script>
+	if (true) {
+		console.log('foo');
+	}
+	</script>
+
+</body>
+</html>
+
+ + + + + + + + + + + + + + From cf1e850765412d61021ff470f1ec2acb7cb2b06f Mon Sep 17 00:00:00 2001 From: Wei Ting <59229084+hoonweiting@users.noreply.github.com> Date: Mon, 4 Oct 2021 00:16:24 +0800 Subject: [PATCH 02/18] Apply suggestions from code review Co-authored-by: Michael Schmidt --- tokens.html | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/tokens.html b/tokens.html index 7ab5562ba7..5a6c17c12c 100644 --- a/tokens.html +++ b/tokens.html @@ -42,7 +42,7 @@

Standard tokens

pi = round(float('3.14159'), 2)
class-name
-
The name of a class. +
The name of a class, interface, trait, or type. 
class Rectangle extends Square {
 	constructor(length, breadth) {
 		super(length);
@@ -51,7 +51,7 @@ 
Standard tokens

 }
function
-
The name of a function. +
The name of a function or method. 
function isEven(number) {
 	return Number(number) % 2 === 0;
 }
@@ -61,7 +61,7 @@ 
Standard tokens

 }
boolean
-
True and false, and pairs with similar concepts. +
True and false, and pairs with similar concepts (e.g. yes and no). 
console.log(true === false); // prints false
 console.log(true === !false); // prints true
@@ -129,32 +129,24 @@

Standard tokens

z ? x : y
variable
-
An object that is assigned and holds a specific value. +
The name of a variable. This token is intended to be used sparingly. It's generally used on special variables (e.g. Less or Bash), not general variables from imperative and procedural programming languages (e.g. C, JavaScript, Python). 
@nice-blue: #5B83AD;
 @light-blue: lighten(@nice-blue, 20%);
constant
-
The name of a a variable that cannot be modified later on. +
The name of a constant. 
const PI = 3.14159;
property
-
An attribute/characteristic. +
An attribute/characteristic or object/map key. 
body {
 	color: red;
 	line-height: normal;
 }
{
 	"data": {
-		"labels": [
-			"foo",
-			"bar"
-		],
-		"series": [
-			[ 0, 1, 2, 3 ],
-			[ 0, -4, -8, -12 ]
-		]
+		"labels": ["foo", "bar"],
 	},
-	// we even support comments
 	"error": null,
 	"status": "Ok"
 }
@@ -174,7 +166,7 @@

Standard tokens

return (copy[size/2 - 1] + copy[size/2]) / 2
important
-
Anything that is important. +
Anything that is important and needs special highlighting. 
body {
 	color: red !important;
 }
@@ -209,7 +201,7 @@

Standard tokens

*I am italicised text!*
tag
-
The markup version of a keyword, to indicate when an element begins and ends. +
A markup tag (e.g. HTML and XML tags). 
<p>Hello World!</p>
attr-name
@@ -223,7 +215,7 @@

Standard tokens

<p id="greeting">Hello World!</p>
namespace
-
Used to provide uniquely named elements and attributes in XML documents. +
Used to provide uniquely named elements and attributes in XML documents. Outside of markup languages, it is used to tokenize the package/namespace part of identifiers. 
<html:p foo:bar="baz" foo:weee></html:p>
prolog
@@ -237,7 +229,7 @@

Standard tokens

<html></html>
cdata
-
Character data, specific to markup languages. Deprecated as of DOM4. +
Character data, specific to markup languages. 
<ns1:description><![CDATA[
   CDATA is <not> magical.
 ]]></ns1:description>
From d6c34832945653f33bc4c5cb4aa8236b58ad5cfe Mon Sep 17 00:00:00 2001 From: Wei Ting Date: Mon, 4 Oct 2021 01:00:58 +0800 Subject: [PATCH 03/18] Collapse inserted/deleted and attr-name/attr-value pairs --- tokens.html | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/tokens.html b/tokens.html index 5a6c17c12c..d104dbeaab 100644 --- a/tokens.html +++ b/tokens.html @@ -182,15 +182,23 @@

Standard tokens

// Here's yet another comment </script> -
inserted
-
Added or modified line, mainly for diffs. In general, also the idea of something being increased. - 
+		// Calculate the area of the circle
-		let area = PI * (radius ** 2);
- -
deleted
-
Deleted line, mainly for diffs. In general, also the idea of something being decreased/removed. - 
-		// TODO: Remove this comment
-		let foo = 'bar';
+
inserted and deleted
+
Added or modified line and deleted line respectively, mainly for diffs. In general, also the idea of something being increased and decreased/removed respectively. + 
7c7
+< qt: core
+---
+> qt: core quick
+ 
--- qcli.yml	2014-12-16 11:43:41.000000000 +0800
++++ /Users/uranusjr/Desktop/qcli.yml	2014-12-31 11:28:08.000000000 +0800
+@@ -4,5 +4,5 @@
+project:
+	sources: "src/*.cpp"
+	headers: "src/*.h"
+-    qt: core
++    qt: core gui
+public_headers: "src/*.h"
+ 
- Some deleted line
++ Some added line
bold
Bolded text. Mostly found in document-markup languages. @@ -204,16 +212,13 @@

Standard tokens

A markup tag (e.g. HTML and XML tags). 
<p>Hello World!</p>
-
attr-name
-
Kind of like a property of a markup tag. - 
<video allowfullscreen controls>
+		
attr-name and attr-value

+		
Kind of like a property of a markup tag and its value/argument respectively.
+		
<p id="greeting">Hello World!</p>
+<video width="1280" height="720" allowfullscreen controls>
 	<source src="hello_world.mp4" type="video/mp4" />
 </video>

 
-		
attr-value

-		
The value or argument that is passed to a property in a markup tag.
-		
<p id="greeting">Hello World!</p>

-
 		
namespace

 		
Used to provide uniquely named elements and attributes in XML documents. Outside of markup languages, it is used to tokenize the package/namespace part of identifiers.
 		
<html:p foo:bar="baz" foo:weee></html:p>


From 2b593f7954abf699d06b6906e28c450d705fbdc3 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Mon, 4 Oct 2021 01:04:45 +0800
Subject: [PATCH 04/18] Improve definitions for builtin and punctuation

Co-Authored-By: Michael Schmidt 
---
 tokens.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tokens.html b/tokens.html
index d104dbeaab..a53ab950f0 100644
--- a/tokens.html
+++ b/tokens.html
@@ -38,7 +38,7 @@ 
Standard tokens

 }
builtin
-
Functions/Methods that are available out of the box. +
Functions/Methods/Classes/Types that are available out of the box. 
pi = round(float('3.14159'), 2)
class-name
@@ -152,7 +152,7 @@

Standard tokens

}
punctuation
-
Punctuation that is not contained within a string, such as brackets. +
Punctuation such as brackets, parentheses, commas, and more. 
def median(pool):
 	'''Statistical median to demonstrate doctest.
 	>>> median([2, 9, 9, 7, 9, 2, 4, 5, 8])

From 4fcd9699c39a23ad07db5f62f211556c23481965 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Mon, 4 Oct 2021 01:08:12 +0800
Subject: [PATCH 05/18] Reduce number of numbers examples

---
 tokens.html | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/tokens.html b/tokens.html
index a53ab950f0..d1886f8c50 100644
--- a/tokens.html
+++ b/tokens.html
@@ -67,27 +67,11 @@ 
Standard tokens

 
 		
number

 		
A numerical value, regardless of base and order, and no matter real or imaginary.
-		
7
-2147483647
-0o177
-0b100110111
-3
-79228162514264337593543950336
-0o377
-0x100000000
+		
3.14159
+42
 0xdeadbeef
-3.14
-10.
-.001
 1e100
-3.14e-10
-0e0
-3.14j
-10.j
-10j
-.001j
-1e100j
-3.14e-10j

+.001j
string
Literal text, including numbers and symbols and maybe even more special characters. From 1d44f7059489fa63ebe2628bcca3f1e9105ec802 Mon Sep 17 00:00:00 2001 From: Wei Ting Date: Tue, 5 Oct 2021 01:24:10 +0800 Subject: [PATCH 06/18] Add more examples --- tokens.html | 69 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 15 deletions(-) diff --git a/tokens.html b/tokens.html index d1886f8c50..d39d58eee6 100644 --- a/tokens.html +++ b/tokens.html @@ -32,14 +32,24 @@

Standard tokens

keyword
Pre-defined and reserved words. - 
for (let x of y) {
-	// Code here
-	return z;
-}
+ 
for (const foo of bar) {
+	console.log(foo);
+	if (foo === 'foobar') {
+		break;
+	}
+}
+ 
if [ -d $directory ]; then
+	echo "Directory exists"
+else
+	echo "Directory does not exists"
+fi
builtin
Functions/Methods/Classes/Types that are available out of the box. - 
pi = round(float('3.14159'), 2)
+ 
pi = round(float('3.14159'), 2)
+ 
interface SearchFunc {
+	(source: string, subString: string): boolean;
+}
class-name
The name of a class, interface, trait, or type. @@ -48,6 +58,10 @@

Standard tokens

super(length); this.breadth = breadth; } +} + 
public class CameraController : MonoBehaviour
+{
+	// TODO: Control camera
 }
function
@@ -115,11 +129,16 @@

Standard tokens

variable
The name of a variable. This token is intended to be used sparingly. It's generally used on special variables (e.g. Less or Bash), not general variables from imperative and procedural programming languages (e.g. C, JavaScript, Python). 
@nice-blue: #5B83AD;
-@light-blue: lighten(@nice-blue, 20%);
+@light-blue: lighten(@nice-blue, 20%); + 
echo $STRING
+args=("$@")
+echo ${args[0]} ${args[1]} ${args[2]}
constant
The name of a constant. - 
const PI = 3.14159;
+ 
const PI = 3.14159;
+ 
const THING: u32 = 0xABAD1DEA;
+ 
fprintf( stdout, "hello world\n" );
property
An attribute/characteristic or object/map key. @@ -168,10 +187,6 @@

Standard tokens

inserted and deleted
Added or modified line and deleted line respectively, mainly for diffs. In general, also the idea of something being increased and decreased/removed respectively. - 
7c7
-< qt: core
----
-> qt: core quick
--- qcli.yml	2014-12-16 11:43:41.000000000 +0800
 +++ /Users/uranusjr/Desktop/qcli.yml	2014-12-31 11:28:08.000000000 +0800
 @@ -4,5 +4,5 @@
@@ -180,9 +195,7 @@ 
Standard tokens

 	headers: "src/*.h"
 -    qt: core
 +    qt: core gui
-public_headers: "src/*.h"
- 
- Some deleted line
-+ Some added line
+public_headers: "src/*.h"
bold
Bolded text. Mostly found in document-markup languages. @@ -205,7 +218,33 @@

Standard tokens

namespace
Used to provide uniquely named elements and attributes in XML documents. Outside of markup languages, it is used to tokenize the package/namespace part of identifiers. - 
<html:p foo:bar="baz" foo:weee></html:p>
+ 
<html:p foo:bar="baz" foo:weee></html:p>
+ 
namespace Foo.Bar {}
+using Foo.Bar;
+ 
class Foo extends foo.bar.Foo {
+	java.util.List bar(foo.bar.Baz bat) throws java.lang.IOException {
+		throw new java.lang.UnsupportedOperationException("Not implemented");
+	}
+}
+ 
use std::collections::HashMap;
+use std::fmt;
+use std::iter;
+use std::result;
+use std::sync::Arc;
+
+use regex_syntax::hir::{self, Hir};
+use regex_syntax::is_word_byte;
+use regex_syntax::utf8::{Utf8Range, Utf8Sequence, Utf8Sequences};
+
+use crate::prog::{
+		EmptyLook, Inst, InstBytes, InstChar, InstEmptyLook, InstPtr, InstRanges,
+		InstSave, InstSplit, Program,
+};
+
+use crate::Error;
+
+type Result = result::Result;
+type ResultOrEmpty = result::Result, Error>;
prolog
The first part of an XML document. From fe59d1281e8c85a4bcd6633762f28e781b8f823f Mon Sep 17 00:00:00 2001 From: Wei Ting Date: Tue, 5 Oct 2021 01:29:37 +0800 Subject: [PATCH 07/18] Edit introductory blurb --- tokens.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tokens.html b/tokens.html index d39d58eee6..881c1a5249 100644 --- a/tokens.html +++ b/tokens.html @@ -4,7 +4,7 @@ -Prism Tokens ▲ Prism +Prism tokens ▲ Prism @@ -29,6 +29,8 @@

Standard tokens

When defining a new language, you will need to provide token names for each 'type' of code, such as keywords and operators, so Prism can highlight code accordingly. It is recommended to make use of the following standard tokens to ensure that highlight code will be highlighted, as Prism's themes (both official and non-official) only guarantee coverage for these standard tokens.

+

Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar in meaning, and will ensure that Prism's themes will highlight it.

+
keyword
Pre-defined and reserved words. From 5bad8624e24b0542f74d6c753b9455349106cc0d Mon Sep 17 00:00:00 2001 From: Wei Ting <59229084+hoonweiting@users.noreply.github.com> Date: Tue, 5 Oct 2021 02:54:30 +0800 Subject: [PATCH 08/18] Apply suggestions from code review Co-authored-by: Michael Schmidt --- tokens.html | 22 +++------------------- 1 file changed, 3 insertions(+), 19 deletions(-) diff --git a/tokens.html b/tokens.html index 881c1a5249..3733a2888b 100644 --- a/tokens.html +++ b/tokens.html @@ -27,7 +27,7 @@

Prism tokens

Standard tokens

-

When defining a new language, you will need to provide token names for each 'type' of code, such as keywords and operators, so Prism can highlight code accordingly. It is recommended to make use of the following standard tokens to ensure that highlight code will be highlighted, as Prism's themes (both official and non-official) only guarantee coverage for these standard tokens.

+

When defining a new language, you will need to provide token names for each 'type' of code, such as keywords and operators, so Prism's themes can assign colors (and other styles) accordingly. Prism's themes (both official and non-official) only guarantee coverage for these standard tokens, so it is recommended to make use of the following standard tokens to ensure that code will be highlighted.

Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar in meaning, and will ensure that Prism's themes will highlight it.

@@ -224,29 +224,13 @@

Standard tokens

namespace Foo.Bar {}
 using Foo.Bar;
class Foo extends foo.bar.Foo {
-	java.util.List bar(foo.bar.Baz bat) throws java.lang.IOException {
+	java.util.List<foo.bar.Foo.Bar> bar(foo.bar.Baz bat) throws java.lang.IOException {
 		throw new java.lang.UnsupportedOperationException("Not implemented");
 	}
 }
use std::collections::HashMap;
 use std::fmt;
-use std::iter;
-use std::result;
-use std::sync::Arc;
-
-use regex_syntax::hir::{self, Hir};
-use regex_syntax::is_word_byte;
-use regex_syntax::utf8::{Utf8Range, Utf8Sequence, Utf8Sequences};
-
-use crate::prog::{
-		EmptyLook, Inst, InstBytes, InstChar, InstEmptyLook, InstPtr, InstRanges,
-		InstSave, InstSplit, Program,
-};
-
-use crate::Error;
-
-type Result = result::Result;
-type ResultOrEmpty = result::Result, Error>;
+use std::sync::Arc;
prolog
The first part of an XML document. From c8035a8be939dadb9910387ed98ec1911a76ff87 Mon Sep 17 00:00:00 2001 From: Wei Ting Date: Tue, 5 Oct 2021 03:00:31 +0800 Subject: [PATCH 09/18] Add missing `` tag --- tokens.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokens.html b/tokens.html index 3733a2888b..c112300d07 100644 --- a/tokens.html +++ b/tokens.html @@ -224,13 +224,13 @@

Standard tokens

namespace Foo.Bar {}
 using Foo.Bar;
class Foo extends foo.bar.Foo {
-	java.util.List<foo.bar.Foo.Bar> bar(foo.bar.Baz bat) throws java.lang.IOException {
+	java.util.List<foo.bar.Foo.Bar> bar(foo.bar.Baz bat) throws java.lang.IOException {
 		throw new java.lang.UnsupportedOperationException("Not implemented");
 	}
 }
use std::collections::HashMap;
 use std::fmt;
-use std::sync::Arc;
+use std::sync::Arc;
prolog
The first part of an XML document. From 871ce6458f1e7d6822adada62213de4dd3dc64af Mon Sep 17 00:00:00 2001 From: RunDevelopment Date: Mon, 4 Oct 2021 21:45:51 +0200 Subject: [PATCH 10/18] Made the token list a table --- tokens.html | 524 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 317 insertions(+), 207 deletions(-) diff --git a/tokens.html b/tokens.html index c112300d07..c518abaee9 100644 --- a/tokens.html +++ b/tokens.html @@ -14,6 +14,27 @@ + @@ -31,247 +52,336 @@

Standard tokens

Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar in meaning, and will ensure that Prism's themes will highlight it.

-
-
keyword
-
Pre-defined and reserved words. - 
for (const foo of bar) {
-	console.log(foo);
-	if (foo === 'foobar') {
-		break;
-	}
-}
- 
if [ -d $directory ]; then
-	echo "Directory exists"
-else
-	echo "Directory does not exists"
-fi
- -
builtin
-
Functions/Methods/Classes/Types that are available out of the box. - 
pi = round(float('3.14159'), 2)
- 
interface SearchFunc {
-	(source: string, subString: string): boolean;
-}
- -
class-name
-
The name of a class, interface, trait, or type. - 
class Rectangle extends Square {
-	constructor(length, breadth) {
-		super(length);
-		this.breadth = breadth;
-	}
+	
+		
+			
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+			
+			
+				
+				
+			
+			
+				
+				
+			
+			
+				
+			
+			
+				
+				
+			
+		
+	
General purpose
keyword
+					Pre-defined and reserved words.
+
+					
for (const foo of bar) {
+	if (foo === 'foobar') break;
+	await foo;
 }

-		
public class CameraController : MonoBehaviour
-{
-	// TODO: Control camera
-}

-
-		
function

-		
The name of a function or method.
-		
function isEven(number) {
+				
builtin
+					Functions/Methods/Classes/Types that are available out of the box.
+
+					
pi = round(float('3.14159'), 2)

+					
type SearchFunc = (source: string, subStr: string) => boolean;

+				
class-name
+					The name of a class, interface, trait, or type.
+
+					
class Rectangle extends Square { /* ... */ }

+					
public class CameraController : MonoBehaviour { /* ... */ }

+				
function
+					The name of a function or method.
+
+					
function isEven(number) {
 	return Number(number) % 2 === 0;
 }
-
-function isOdd(number) {
-	return !isEven(number);
-}

-
-		
boolean

-		
True and false, and pairs with similar concepts (e.g. yes and no).
-		
console.log(true === false); // prints false
-console.log(true === !false); // prints true

-
-		
number

-		
A numerical value, regardless of base and order, and no matter real or imaginary.
-		
3.14159
-42
-0xdeadbeef
-1e100
-.001j

-
-		
string

-		
Literal text, including numbers and symbols and maybe even more special characters.
-		
let greeting = 'Hello World!';

-
-		
char

-		
A string that can comprise only a single character, enforced by the language.
-		
'A'
-'z'
-'0'
-'-'
-'\t'
-'\u{2728}'

-
-		
symbol

-		
A primitive data type found in some languages, can be thought of as an identifier.
-		
#myFirstSymbol "#myFirstSymbol is a symbol in Smalltalk and is the same object as all other #myFirstSymbol symbols"

-
-		
regex

-		
A regular expression.
-		
let regexPattern = /((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/;

-
-		
url

-		
A link to another page or resource.
-		
body {
+const isOdd = (number) => !isEven(number);

+				
boolean
+					True and false, and pairs with similar concepts (e.g. yes and no).
+
+					
console.log(true === false); // prints false
+console.log(true === !false); // prints true

+				
number
+					A numerical value, regardless of base and order, and no matter real or imaginary.
+
+					
print(3.14159 * 42)
+print(1e100 + .001j)
+return foo & 0xdeadbeef

+				
string
+					Literal text, including numbers and symbols and maybe even more special characters.
+
+					
let greeting = 'Hello World!';

+				
char
+					A string that can comprise only a single character, enforced by the language.
+
+					
['A', 'z', '0', '-', '\t', '\u{2728}']

+				
symbol
+					A primitive data type found in some languages, can be thought of as an identifier.
+
+					
#myFirstSymbol "#myFirstSymbol is a symbol in Smalltalk"

+				
regex
+					A regular expression.
+
+					
let entity = /&#x?[\da-f]{1,8};/;

+				
url
+					A link to another page or resource.
+
+					
body {
 	background: url(foo.png);
 }

-		
[Prism](https://prismjs.com) is a cool syntax highlighter.

-
-		
operator

-		
A symbol that represents an action or process, whether it's a mathematical operation, logical operation, and so on.
-		
x = y
-x *= y
-x !== y
-x++
--y
-x >>> y
-x || y
-z ? x : y

-
-		
variable

-		
The name of a variable. This token is intended to be used sparingly. It's generally used on special variables (e.g. Less or Bash), not general variables from imperative and procedural programming languages (e.g. C, JavaScript, Python).
-		
@nice-blue: #5B83AD;
+					
[Prism](https://prismjs.com) is a cool syntax highlighter.

+				
operator
+					A symbol that represents an action or process, whether it's a mathematical operation, logical operation, and so on.
+
+					
x += (y + 4 >> -z === w) ? b ** c : ~a;

+				
variable
+					The name of a variable. This token is intended to be used sparingly. It's generally used on special variables (e.g. Less or Bash), not general variables from imperative and procedural programming languages (e.g. C, JavaScript, Python).
+
+					
@nice-blue: #5B83AD;
 @light-blue: lighten(@nice-blue, 20%);

-		
echo $STRING
+					
echo $STRING
 args=("$@")
-echo ${args[0]} ${args[1]} ${args[2]}

-
-		
constant

-		
The name of a constant.
-		
const PI = 3.14159;

-		
const THING: u32 = 0xABAD1DEA;

-		
fprintf( stdout, "hello world\n" );

-
-		
property

-		
An attribute/characteristic or object/map key.
-		
body {
+echo ${args[0]} ${args[1]} ${args[2]}

+				
constant
+					The name of a constant.
+
+					
const PI = 3.14159;

+					
const THING: u32 = 0xABAD1DEA;

+					
fprintf(stdout, "hello world\n");

+				
property
+					An attribute/characteristic or object/map key.
+
+					
body {
 	color: red;
 	line-height: normal;
 }

-		
{
-	"data": {
-		"labels": ["foo", "bar"],
-	},
+					
{
+	"data": { "labels": ["foo", "bar"], },
 	"error": null,
 	"status": "Ok"
-}

-
-		
punctuation

-		
Punctuation such as brackets, parentheses, commas, and more.
-		
def median(pool):
-	'''Statistical median to demonstrate doctest.
-	>>> median([2, 9, 9, 7, 9, 2, 4, 5, 8])
-	7
-	'''
+}

+				
punctuation
+					Punctuation such as brackets, parentheses, commas, and more.
+
+					
def median(pool):
 	copy = sorted(pool)
 	size = len(copy)
 	if size % 2 == 1:
 		return copy[(size - 1) / 2]
 	else:
-		return (copy[size/2 - 1] + copy[size/2]) / 2

-
-		
important

-		
Anything that is important and needs special highlighting.
-		
body {
+		return (copy[size/2 - 1] + copy[size/2]) / 2

+				
important
+					Anything that is important and needs special highlighting.
+
+					
body {
 	color: red !important;
 }

-		
# This is a heading. Headings are important.

-
-		
comment

-		
Code comments.
-		
<!-- Here's a comment -->
+					
# This is a heading. Headings are important.

+				
comment
+					Code comments.
+
+					
<!-- Here's a comment -->
 <style>
 	/* Here's another comment */
 </style>
 <script>
 // Here's yet another comment
-</script>

-
-		
inserted and deleted

-		
Added or modified line and deleted line respectively, mainly for diffs. In general, also the idea of something being increased and decreased/removed respectively.
-		
--- qcli.yml	2014-12-16 11:43:41.000000000 +0800
-+++ /Users/uranusjr/Desktop/qcli.yml	2014-12-31 11:28:08.000000000 +0800
-@@ -4,5 +4,5 @@
-project:
-	sources: "src/*.cpp"
-	headers: "src/*.h"
--    qt: core
-+    qt: core gui
-public_headers: "src/*.h"

-
-		
bold

-		
Bolded text. Mostly found in document-markup languages.
-		
**I am bolded text!**

-
-		
italic

-		
Italicised text. Mostly found in document-markup languages.
-		
*I am italicised text!*

-
-		
tag

-		
A markup tag (e.g. HTML and XML tags).
-		
<p>Hello World!</p>

-
-		
attr-name and attr-value

-		
Kind of like a property of a markup tag and its value/argument respectively.
-		
<p id="greeting">Hello World!</p>
+</script>

+				
Markup languages
tag
+					A markup tag (e.g. HTML and XML tags).
+
+					
<p>Hello World!</p>

+				
attr-name, attr-value
+					Kind of like a property of a markup tag and its value/argument respectively.
+
+					
<p id="greeting">Hello World!</p>
 <video width="1280" height="720" allowfullscreen controls>
 	<source src="hello_world.mp4" type="video/mp4" />
-</video>

-
-		
namespace

-		
Used to provide uniquely named elements and attributes in XML documents. Outside of markup languages, it is used to tokenize the package/namespace part of identifiers.
-		
<html:p foo:bar="baz" foo:weee></html:p>

-		
namespace Foo.Bar {}
-using Foo.Bar;

-		
class Foo extends foo.bar.Foo {
-	java.util.List<foo.bar.Foo.Bar> bar(foo.bar.Baz bat) throws java.lang.IOException {
-		throw new java.lang.UnsupportedOperationException("Not implemented");
+</video>

+				
namespace
+					Used to provide uniquely named elements and attributes in XML documents. Outside of markup languages, it is used to tokenize the package/namespace part of identifiers.
+
+					
<html:p foo:bar="baz" foo:weee></html:p>

+					
class Foo extends foo.bar.Foo {
+	java.util.List<foo.bar.Foo.Bar> bar(foo.bar.Baz bat) {
+		throw new java.lang.UnsupportedOperationException();
 	}
 }

-		
use std::collections::HashMap;
-use std::fmt;
-use std::sync::Arc;

-
-		
prolog

-		
The first part of an XML document.
-		
<?xml version="1.0" encoding="utf-8"?>
-<svg></svg>

-
-		
doctype

-		
Document type declaration, specific to markup languages.
-		
<!DOCTYPE html>
-<html></html>

-
-		
cdata

-		
Character data, specific to markup languages.
-		
<ns1:description><![CDATA[
+					
use std::sync::Arc;

+				
prolog
+					The first part of an XML document.
+
+					
<?xml version="1.0" encoding="utf-8"?>
+<svg></svg>

+				
doctype
+					Document type declaration, specific to markup languages.
+
+					
<!DOCTYPE html>
+<html></html>

+				
cdata
+					Character data, specific to markup languages.
+
+					
<ns1:description><![CDATA[
   CDATA is <not> magical.
-]]></ns1:description>

-
-		
entity

-		
Code used to display reserved characters in markup languages.
-		
&amp; &#x2665; &#160; &#x152;

-
-		
atrule

-		
Literally @ rules (statements) in stylesheets.
-		
@font-family {
+]]></ns1:description>

+				
entity
+					Code used to display reserved characters in markup languages.
+
+					
&amp; &#x2665; &#160; &#x152;

+				
Document-markup languages
bold
+					Bolded text. Mostly found in document-markup languages.
+
+					
**I am bolded text!**

+				
italic
+					Italicised text. Mostly found in document-markup languages.
+
+					
*I am italicised text!*

+				
Stylesheets
atrule
+					Literally @ rules (statements) in stylesheets.
+
+					
@font-family {
 	font-family: Questrial;
 	src: url(questrial.otf);
 }
-
-@media screen and (min-width: 768px) {
-	/* rules here */
-}

-
-		
selector

-		
Code that identifies or picks something out of a group to operate on, such as the names of HTML elements in stylesheets.
-		
section h1,
+@media screen and (min-width: 768px) { /* ... */ }

+				
selector
+					Code that identifies or picks something out of a group to operate on, such as the names of HTML elements in stylesheets.
+
+					
section h1,
 #features li strong,
 header h2,
-footer p {
-	/* styles here */
-}

-	
+footer p { /* ... */ }
+				
Diff
inserted, deleted
+					Added or modified line and deleted line respectively, mainly for diffs. In general, also the idea of something being increased and decreased/removed respectively.
+
+					
--- bar.yml	2014-12-16 11:43:41 +0800
++++ /Users/foo/Desktop/bar.yml	2014-12-31 11:28:08 +0800
+@@ -4,5 +4,5 @@
+project:
+	sources: "src/*.cpp"
+	headers: "src/*.h"
+-    qt: core
++    qt: core gui
+public_headers: "src/*.h"

+				

 
 
 


From 08c4e460bf1c9e3e6fc94fb2a883a654ab611e20 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Tue, 5 Oct 2021 21:49:30 +0800
Subject: [PATCH 11/18] Move non-standard tokens blurb into a section of its
 own

---
 tokens.html | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tokens.html b/tokens.html
index c518abaee9..98b305153a 100644
--- a/tokens.html
+++ b/tokens.html
@@ -50,8 +50,6 @@ 
Standard tokens

 
 	
When defining a new language, you will need to provide token names for each 'type' of code, such as keywords and operators, so Prism's themes can assign colors (and other styles) accordingly. Prism's themes (both official and non-official) only guarantee coverage for these standard tokens, so it is recommended to make use of the following standard tokens to ensure that code will be highlighted.

 
-	
Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar in meaning, and will ensure that Prism's themes will highlight it.

-
 	
@@ -414,6 +412,12 @@ 
Embedded languages

 </html>
 
+

+	
Non-standard tokens

+
+	
Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar, and will ensure that Prism's themes will highlight it.

+

+
 


From 0ad11a292b2c642b0c523c680a43870be8becb07 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Tue, 5 Oct 2021 22:15:47 +0800
Subject: [PATCH 12/18] Match table cell padding with that in extending.html

---
 tokens.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tokens.html b/tokens.html
index 98b305153a..e9b7468e97 100644
--- a/tokens.html
+++ b/tokens.html
@@ -29,7 +29,7 @@
 	background-color: #F8F8F8;
 }
 table.styled tr > * {
-	padding: .5em;
+	padding: .5em .75em;
 }
 table.styled tr > th {
 	text-align: left;

From 483d267e473aba45392c0c9b5d8910251ec2cb6d Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Tue, 5 Oct 2021 22:16:36 +0800
Subject: [PATCH 13/18] Change "Prism" to "Prism's themes" in the embedded
 languages section

---
 tokens.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tokens.html b/tokens.html
index e9b7468e97..90c8b29c2d 100644
--- a/tokens.html
+++ b/tokens.html
@@ -385,7 +385,7 @@ 
Standard tokens

 

 	
Embedded languages

 
-	
In addition to the standard tokens above, Prism also has a token for languages that are embedded in another language, such as CSS in HTML, JS in HTML, Bash in Shell-session, and CSS in JS, allowing Prism to highlight the tokens in the embedded languages more accurately. All embedded languages are wrapped in their own special token, which includes a CSS class language-xxxx corresponding to the embedded language.

+	
In addition to the standard tokens above, Prism also has a token for languages that are embedded in another language, such as CSS in HTML, JS in HTML, Bash in Shell-session, and CSS in JS, allowing Prism's themes to highlight the tokens in the embedded languages more accurately. All embedded languages are wrapped in their own special token, which includes a CSS class language-xxxx corresponding to the embedded language.

 
 	
Open your browser's developer tools and check out the example below to see it in action!

 

From d47f0a53a50b0a5f6d5f1743cb75eea92857866d Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Tue, 5 Oct 2021 22:44:11 +0800
Subject: [PATCH 14/18] Provide example for non-standard tokens

---
 tokens.html | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tokens.html b/tokens.html
index 90c8b29c2d..3a0362b3f5 100644
--- a/tokens.html
+++ b/tokens.html
@@ -415,7 +415,12 @@ 
Embedded languages

 

 	
Non-standard tokens

 
-	
Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar, and will ensure that Prism's themes will highlight it.

+	
Sometimes, a language might use a particular name to refer to certain pieces of code, but which is not one of Prism's standard token names, such as function-defintion. Since function-definition is not a standard token, you might want to alias it with a standard token such as function, which is semantically similar, and will ensure that Prism's themes will highlight it. Here's an example:

+
+	
fn main() {
+	println!("Hello World");
+	another_function();
+}

 

 
 


From ca765bec939f9c5f90a00a9a19f88d330f4a947c Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Thu, 7 Oct 2021 20:27:19 +0800
Subject: [PATCH 15/18] Remove link to non-existent Show Language CSS file

---
 tokens.html | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tokens.html b/tokens.html
index 3a0362b3f5..0fa13cce4f 100644
--- a/tokens.html
+++ b/tokens.html
@@ -9,7 +9,6 @@
 
 
 
-
 
 
 

From 6ac2537358cfa0af6cfe5fc375244d12df2b5c20 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Fri, 22 Oct 2021 01:20:27 +0800
Subject: [PATCH 16/18] Remove blank line at the start of the code block

---
 faq.html | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/faq.html b/faq.html
index c060119167..64c858796f 100644
--- a/faq.html
+++ b/faq.html
@@ -101,8 +101,7 @@ 
How do I know which tokens I can style for every language?

 	
How can I use different highlighting for tokens with the same name in different languages?

 	
Just use a descendant selector, that includes the language class. The default prism.css does this, to have different colors for
 	JavaScript strings (which are very common) and CSS strings (which are relatively rare). Here’s that code, simplified to illustrate the technique:
-	

-.token.string {
+	
.token.string {
 	color: #690;
 }
 

From f0846ddceba5fe8e0a3d6d6f3e5872ae178ef30d Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Fri, 22 Oct 2021 01:21:18 +0800
Subject: [PATCH 17/18] Add `language-none` class to `body` tag

---
 faq.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/faq.html b/faq.html
index 64c858796f..3c6cd32a41 100644
--- a/faq.html
+++ b/faq.html
@@ -21,7 +21,7 @@
 
 
 
-
+
 
 

 	


From a27a11c2ec2279d4010cf6f2c7006672af584210 Mon Sep 17 00:00:00 2001
From: Wei Ting 
Date: Fri, 22 Oct 2021 01:21:47 +0800
Subject: [PATCH 18/18] Add links to `tokens.html`

---
 extending.html | 4 ++--
 faq.html       | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/extending.html b/extending.html
index eb6a561d26..606c2816fb 100644
--- a/extending.html
+++ b/extending.html
@@ -60,8 +60,8 @@ 
Language definitions

 
 		
alias

 		
This option can be used to define one or more aliases for the matched token. The result will be, that
-		the styles of the token and its aliases are combined. This can be useful, to combine the styling of a well known
-		token, which is already supported by most of the themes,  with a semantically correct token name. The option
+		the styles of the token and its aliases are combined. This can be useful, to combine the styling of a standard
+		token, which is already supported by most of the themes,  with a semantically correct token name. The option
 		can be set to a string literal or an array of string literals. In the following example the token
 		name latex-equation is not supported by any theme, but it will be highlighted the same as a string.
 		
{
diff --git a/faq.html b/faq.html
index 3c6cd32a41..8dd2a57338 100644
--- a/faq.html
+++ b/faq.html
@@ -95,6 +95,8 @@ 
How do I know which tokens I can style for every language?

 
 	
Language: 

 	

+
+	
Additionally, you can find a list of standard tokens on this page.