Basic lexer for Microsoft Power Query M (#462)

alecthomas · Jun 15, 2021 · 841c391 · 841c391
1 parent d13f838
commit 841c391
Show file tree

Hide file tree

Showing 3 changed files with 205 additions and 0 deletions.
diff --git a/lexers/p/powerquery.go b/lexers/p/powerquery.go
@@ -0,0 +1,38 @@
+package p
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// PowerQuery lexer.
+var PowerQuery = internal.Register(MustNewLazyLexer(
+	&Config{
+		Name:            "PowerQuery",
+		Aliases:         []string{"powerquery", "pq"},
+		Filenames:       []string{"*.pq"},
+		MimeTypes:       []string{"text/x-powerquery"},
+		DotAll:          true,
+		CaseInsensitive: true,
+	},
+	powerqueryRules,
+))
+
+func powerqueryRules() Rules {
+	return Rules{
+		"root": {
+			{`\s+`, Text, nil},
+			{`//.*?\n`, CommentSingle, nil},
+			{`/\*.*?\*/`, CommentMultiline, nil},
+			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
+			{`(and|as|each|else|error|false|if|in|is|let|meta|not|null|or|otherwise|section|shared|then|true|try|type)\b`, Keyword, nil},
+			{`(#binary|#date|#datetime|#datetimezone|#duration|#infinity|#nan|#sections|#shared|#table|#time)\b`, KeywordType, nil},
+			{`(([a-zA-Z]|_)[\w|._]*|#"[^"]+")`, Name, nil},
+			{`0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?`, LiteralNumberHex, nil},
+			{`([0-9]+\.[0-9]+|\.[0-9]+)([eE][0-9]+)?`, LiteralNumberFloat, nil},
+			{`[0-9]+`, LiteralNumberInteger, nil},
+			{`[\(\)\[\]\{\}]`, Punctuation, nil},
+			{`\.\.|\.\.\.|=>|<=|>=|<>|[@!?,;=<>\+\-\*\/&]`, Operator, nil},
+		},
+	}
+}
diff --git a/lexers/testdata/powerquery.actual b/lexers/testdata/powerquery.actual
@@ -0,0 +1,21 @@
+let
+  GetLabel = (subject as text) as nullable table =>
+    let 
+      // Request Ntriples from the LDF Service
+      Options = [Headers = [#"Accept"="text/csv"]],
+      FixedSubject = if Text.At(subject,0) <> "<" then Text.Combine({"<",subject,">"}) else subject,
+      Query = Text.Combine({"SELECT ?label WHERE {
+        SERVICE wikibase:label {
+        bd:serviceParam wikibase:language ""en"" .
+        ",FixedSubject," <http://www.w3.org/2000/01/rdf-schema#label> ?label
+        }
+      }"}),
+      /* this is a multiline comment, see https://docs.microsoft.com/en-us/powerquery-m/comments
+      */
+      url = Text.Combine({"https://query.wikidata.org/sparql?",Uri.BuildQueryString([query=Query])}),
+      data = Csv.Document(Web.Contents(url,Options)),
+      promoted = Table.PromoteHeaders(data, [PromoteAllScalars=true])
+    in 
+      promoted 
+in
+  GetLabel 
diff --git a/lexers/testdata/powerquery.expected b/lexers/testdata/powerquery.expected
@@ -0,0 +1,146 @@
+[
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"GetLabel"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"subject"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"as"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"text"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"as"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"nullable"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"table"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" \n      "},
+  {"type":"CommentSingle","value":"// Request Ntriples from the LDF Service\n"},
+  {"type":"Text","value":"      "},
+  {"type":"Name","value":"Options"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"Headers"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"#\"Accept\""},
+  {"type":"Operator","value":"="},
+  {"type":"LiteralString","value":"\"text/csv\""},
+  {"type":"Punctuation","value":"]]"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"FixedSubject"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Text.At"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"subject"},
+  {"type":"Operator","value":","},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003c\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"\u003c\""},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"then"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Text.Combine"},
+  {"type":"Punctuation","value":"({"},
+  {"type":"LiteralString","value":"\"\u003c\""},
+  {"type":"Operator","value":","},
+  {"type":"Name","value":"subject"},
+  {"type":"Operator","value":","},
+  {"type":"LiteralString","value":"\"\u003e\""},
+  {"type":"Punctuation","value":"})"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"else"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"subject"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"Query"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Text.Combine"},
+  {"type":"Punctuation","value":"({"},
+  {"type":"LiteralString","value":"\"SELECT ?label WHERE {\n        SERVICE wikibase:label {\n        bd:serviceParam wikibase:language \"\"en\"\" .\n        \""},
+  {"type":"Operator","value":","},
+  {"type":"Name","value":"FixedSubject"},
+  {"type":"Operator","value":","},
+  {"type":"LiteralString","value":"\" \u003chttp://www.w3.org/2000/01/rdf-schema#label\u003e ?label\n        }\n      }\""},
+  {"type":"Punctuation","value":"})"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":"\n      "},
+  {"type":"CommentMultiline","value":"/* this is a multiline comment, see https://docs.microsoft.com/en-us/powerquery-m/comments\n      */"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"url"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Text.Combine"},
+  {"type":"Punctuation","value":"({"},
+  {"type":"LiteralString","value":"\"https://query.wikidata.org/sparql?\""},
+  {"type":"Operator","value":","},
+  {"type":"Name","value":"Uri.BuildQueryString"},
+  {"type":"Punctuation","value":"(["},
+  {"type":"Name","value":"query"},
+  {"type":"Operator","value":"="},
+  {"type":"Name","value":"Query"},
+  {"type":"Punctuation","value":"])})"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"data"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Csv.Document"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"Web.Contents"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"url"},
+  {"type":"Operator","value":","},
+  {"type":"Name","value":"Options"},
+  {"type":"Punctuation","value":"))"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"promoted"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Table.PromoteHeaders"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"data"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"Name","value":"PromoteAllScalars"},
+  {"type":"Operator","value":"="},
+  {"type":"Keyword","value":"true"},
+  {"type":"Punctuation","value":"])"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"in"},
+  {"type":"Text","value":" \n      "},
+  {"type":"Name","value":"promoted"},
+  {"type":"Text","value":" \n"},
+  {"type":"Keyword","value":"in"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"GetLabel"},
+  {"type":"Text","value":" "}
+]