gopls/internal: start on LSP stub generator in Go.

This is the first in a series of CLs implementing the new stub
generator. The code is intended to reproduce exactly the current
state of the generated code.

This CL has the final file layout, but primarily consists
of the parsing of the specification.

The LSP maintainers now provide a .json file describing the messages and
types used in the protocol. The new code in this CL, written in Go,
parses this file and generates Go definitions.

The tests need to be run by hand because the metaModel.json file is not
available to the presubmit tests.

Related golang/go#52969

Change-Id: Id2fc58c973a92c39ba98c936f2af03b1c40ada44
Reviewed-on: https://go-review.googlesource.com/c/tools/+/443055
Reviewed-by: Robert Findley <rfindley@google.com>
Reviewed-by: Alan Donovan <adonovan@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Run-TryBot: Peter Weinberger <pjw@google.com>

gopls/internal/lsp/protocol/generate/compare.go

@@ -0,0 +1,10 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+package main
+
+// compare the generated files in two directories

gopls/internal/lsp/protocol/generate/data.go

@@ -0,0 +1,104 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+package main
+
+// various data tables
+
+// methodNames is a map from the method to the name of the function that handles it
+var methodNames = map[string]string{
+	"$/cancelRequest":                        "CancelRequest",
+	"$/logTrace":                             "LogTrace",
+	"$/progress":                             "Progress",
+	"$/setTrace":                             "SetTrace",
+	"callHierarchy/incomingCalls":            "IncomingCalls",
+	"callHierarchy/outgoingCalls":            "OutgoingCalls",
+	"client/registerCapability":              "RegisterCapability",
+	"client/unregisterCapability":            "UnregisterCapability",
+	"codeAction/resolve":                     "ResolveCodeAction",
+	"codeLens/resolve":                       "ResolveCodeLens",
+	"completionItem/resolve":                 "ResolveCompletionItem",
+	"documentLink/resolve":                   "ResolveDocumentLink",
+	"exit":                                   "Exit",
+	"initialize":                             "Initialize",
+	"initialized":                            "Initialized",
+	"inlayHint/resolve":                      "Resolve",
+	"notebookDocument/didChange":             "DidChangeNotebookDocument",
+	"notebookDocument/didClose":              "DidCloseNotebookDocument",
+	"notebookDocument/didOpen":               "DidOpenNotebookDocument",
+	"notebookDocument/didSave":               "DidSaveNotebookDocument",
+	"shutdown":                               "Shutdown",
+	"telemetry/event":                        "Event",
+	"textDocument/codeAction":                "CodeAction",
+	"textDocument/codeLens":                  "CodeLens",
+	"textDocument/colorPresentation":         "ColorPresentation",
+	"textDocument/completion":                "Completion",
+	"textDocument/declaration":               "Declaration",
+	"textDocument/definition":                "Definition",
+	"textDocument/diagnostic":                "Diagnostic",
+	"textDocument/didChange":                 "DidChange",
+	"textDocument/didClose":                  "DidClose",
+	"textDocument/didOpen":                   "DidOpen",
+	"textDocument/didSave":                   "DidSave",
+	"textDocument/documentColor":             "DocumentColor",
+	"textDocument/documentHighlight":         "DocumentHighlight",
+	"textDocument/documentLink":              "DocumentLink",
+	"textDocument/documentSymbol":            "DocumentSymbol",
+	"textDocument/foldingRange":              "FoldingRange",
+	"textDocument/formatting":                "Formatting",
+	"textDocument/hover":                     "Hover",
+	"textDocument/implementation":            "Implementation",
+	"textDocument/inlayHint":                 "InlayHint",
+	"textDocument/inlineValue":               "InlineValue",
+	"textDocument/linkedEditingRange":        "LinkedEditingRange",
+	"textDocument/moniker":                   "Moniker",
+	"textDocument/onTypeFormatting":          "OnTypeFormatting",
+	"textDocument/prepareCallHierarchy":      "PrepareCallHierarchy",
+	"textDocument/prepareRename":             "PrepareRename",
+	"textDocument/prepareTypeHierarchy":      "PrepareTypeHierarchy",
+	"textDocument/publishDiagnostics":        "PublishDiagnostics",
+	"textDocument/rangeFormatting":           "RangeFormatting",
+	"textDocument/references":                "References",
+	"textDocument/rename":                    "Rename",
+	"textDocument/selectionRange":            "SelectionRange",
+	"textDocument/semanticTokens/full":       "SemanticTokensFull",
+	"textDocument/semanticTokens/full/delta": "SemanticTokensFullDelta",
+	"textDocument/semanticTokens/range":      "SemanticTokensRange",
+	"textDocument/signatureHelp":             "SignatureHelp",
+	"textDocument/typeDefinition":            "TypeDefinition",
+	"textDocument/willSave":                  "WillSave",
+	"textDocument/willSaveWaitUntil":         "WillSaveWaitUntil",
+	"typeHierarchy/subtypes":                 "Subtypes",
+	"typeHierarchy/supertypes":               "Supertypes",
+	"window/logMessage":                      "LogMessage",
+	"window/showDocument":                    "ShowDocument",
+	"window/showMessage":                     "ShowMessage",
+	"window/showMessageRequest":              "ShowMessageRequest",
+	"window/workDoneProgress/cancel":         "WorkDoneProgressCancel",
+	"window/workDoneProgress/create":         "WorkDoneProgressCreate",
+	"workspace/applyEdit":                    "ApplyEdit",
+	"workspace/codeLens/refresh":             "CodeLensRefresh",
+	"workspace/configuration":                "Configuration",
+	"workspace/diagnostic":                   "DiagnosticWorkspace",
+	"workspace/diagnostic/refresh":           "DiagnosticRefresh",
+	"workspace/didChangeConfiguration":       "DidChangeConfiguration",
+	"workspace/didChangeWatchedFiles":        "DidChangeWatchedFiles",
+	"workspace/didChangeWorkspaceFolders":    "DidChangeWorkspaceFolders",
+	"workspace/didCreateFiles":               "DidCreateFiles",
+	"workspace/didDeleteFiles":               "DidDeleteFiles",
+	"workspace/didRenameFiles":               "DidRenameFiles",
+	"workspace/executeCommand":               "ExecuteCommand",
+	"workspace/inlayHint/refresh":            "InlayHintRefresh",
+	"workspace/inlineValue/refresh":          "InlineValueRefresh",
+	"workspace/semanticTokens/refresh":       "SemanticTokensRefresh",
+	"workspace/symbol":                       "Symbol",
+	"workspace/willCreateFiles":              "WillCreateFiles",
+	"workspace/willDeleteFiles":              "WillDeleteFiles",
+	"workspace/willRenameFiles":              "WillRenameFiles",
+	"workspace/workspaceFolders":             "WorkspaceFolders",
+	"workspaceSymbol/resolve":                "ResolveWorkspaceSymbol",
+}

gopls/internal/lsp/protocol/generate/doc.go

@@ -0,0 +1,32 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+/*
+GenLSP  generates the files tsprotocol.go, tsclient.go,
+tsserver.go, tsjson.go that support the language server protocol
+for gopls.
+
+Usage:
+
+	go run . [flags]
+
+The flags are:
+
+	-d <directory name>
+		The directory containing the vscode-languageserver-node repository.
+		(git clone https://github.com/microsoft/vscode-languageserver-node.git).
+		If not specified, the default is $HOME/vscode-languageserver-node.
+
+	-o <directory name>
+		The directory to write the generated files to. It must exist.
+		The default is "gen".
+
+	-c <directory name>
+		Compare the generated files to the files in the specified directory.
+		If this flag is not specified, no comparison is done.
+*/
+package main

gopls/internal/lsp/protocol/generate/generate.go

@@ -0,0 +1,10 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+package main
+
+// generate the Go code

gopls/internal/lsp/protocol/generate/main.go

@@ -0,0 +1,93 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"os"
+)
+
+var (
+	// git clone https://github.com/microsoft/vscode-languageserver-node.git
+	repodir   = flag.String("d", "", "directory of vscode-languageserver-node")
+	outputdir = flag.String("o", "gen", "output directory")
+	cmpolder  = flag.String("c", "", "directory of older generated code")
+)
+
+func main() {
+	log.SetFlags(log.Lshortfile) // log file name and line number, not time
+	flag.Parse()
+
+	if *repodir == "" {
+		*repodir = fmt.Sprintf("%s/vscode-languageserver-node", os.Getenv("HOME"))
+	}
+	spec := parse(*repodir)
+
+	// index the information in the specification
+	spec.indexRPCInfo() // messages
+	spec.indexDefInfo() // named types
+
+}
+
+func (s *spec) indexRPCInfo() {
+	for _, r := range s.model.Requests {
+		r := r
+		s.byMethod[r.Method] = &r
+	}
+	for _, n := range s.model.Notifications {
+		n := n
+		if n.Method == "$/cancelRequest" {
+			// viewed as too confusing to generate
+			continue
+		}
+		s.byMethod[n.Method] = &n
+	}
+}
+
+func (sp *spec) indexDefInfo() {
+	for _, s := range sp.model.Structures {
+		s := s
+		sp.byName[s.Name] = &s
+	}
+	for _, e := range sp.model.Enumerations {
+		e := e
+		sp.byName[e.Name] = &e
+	}
+	for _, ta := range sp.model.TypeAliases {
+		ta := ta
+		sp.byName[ta.Name] = &ta
+	}
+
+	// some Structure and TypeAlias names need to be changed for Go
+	// so byName contains the name used in the .json file, and
+	// the Name field contains the Go version of the name.
+	v := sp.model.Structures
+	for i, s := range v {
+		switch s.Name {
+		case "_InitializeParams": // _ is not upper case
+			v[i].Name = "XInitializeParams"
+		case "ConfigurationParams": // gopls compatibility
+			v[i].Name = "ParamConfiguration"
+		case "InitializeParams": // gopls compatibility
+			v[i].Name = "ParamInitialize"
+		case "PreviousResultId": // Go naming convention
+			v[i].Name = "PreviousResultID"
+		case "WorkspaceFoldersServerCapabilities": // gopls compatibility
+			v[i].Name = "WorkspaceFolders5Gn"
+		}
+	}
+	w := sp.model.TypeAliases
+	for i, t := range w {
+		switch t.Name {
+		case "PrepareRenameResult": // gopls compatibility
+			w[i].Name = "PrepareRename2Gn"
+		}
+	}
+}

gopls/internal/lsp/protocol/generate/main_test.go

@@ -0,0 +1,122 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.19
+// +build go1.19
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"testing"
+)
+
+// this is not a test, but an easy way to invoke the debugger
+func TestAll(t *testing.T) {
+	t.Skip("run by hand")
+	log.SetFlags(log.Lshortfile)
+	main()
+}
+
+// this is not a test, but an easy way to invoke the debugger
+func TestCompare(t *testing.T) {
+	t.Skip("run by hand")
+	log.SetFlags(log.Lshortfile)
+	*cmpolder = "../lsp/gen" // instead use a directory containing the older generated files
+	main()
+}
+
+// check that the parsed file includes all the information
+// from the json file. This test will fail if the spec
+// introduces new fields. (one can test this test by
+// commenting out some special handling in parse.go.)
+func TestParseContents(t *testing.T) {
+	t.Skip("run by hand")
+	log.SetFlags(log.Lshortfile)
+
+	// compute our parse of the specification
+	dir := os.Getenv("HOME") + "/vscode-languageserver-node"
+	v := parse(dir)
+	out, err := json.Marshal(v.model)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var our interface{}
+	if err := json.Unmarshal(out, &our); err != nil {
+		t.Fatal(err)
+	}
+
+	// process the json file
+	fname := dir + "/protocol/metaModel.json"
+	buf, err := os.ReadFile(fname)
+	if err != nil {
+		t.Fatalf("could not read metaModel.json: %v", err)
+	}
+	var raw interface{}
+	if err := json.Unmarshal(buf, &raw); err != nil {
+		t.Fatal(err)
+	}
+
+	// convert to strings showing the fields
+	them := flatten(raw)
+	us := flatten(our)
+
+	// everything in them should be in us
+	lesser := make(sortedMap[bool])
+	for _, s := range them {
+		lesser[s] = true
+	}
+	greater := make(sortedMap[bool]) // set of fields we have
+	for _, s := range us {
+		greater[s] = true
+	}
+	for _, k := range lesser.keys() { // set if fields they have
+		if !greater[k] {
+			t.Errorf("missing %s", k)
+		}
+	}
+}
+
+// flatten(nil) = "nil"
+// flatten(v string) = fmt.Sprintf("%q", v)
+// flatten(v float64)= fmt.Sprintf("%g", v)
+// flatten(v bool) = fmt.Sprintf("%v", v)
+// flatten(v []any) = []string{"[0]"flatten(v[0]), "[1]"flatten(v[1]), ...}
+// flatten(v map[string]any) = {"key1": flatten(v["key1"]), "key2": flatten(v["key2"]), ...}
+func flatten(x any) []string {
+	switch v := x.(type) {
+	case nil:
+		return []string{"nil"}
+	case string:
+		return []string{fmt.Sprintf("%q", v)}
+	case float64:
+		return []string{fmt.Sprintf("%g", v)}
+	case bool:
+		return []string{fmt.Sprintf("%v", v)}
+	case []any:
+		var ans []string
+		for i, x := range v {
+			idx := fmt.Sprintf("[%.3d]", i)
+			for _, s := range flatten(x) {
+				ans = append(ans, idx+s)
+			}
+		}
+		return ans
+	case map[string]any:
+		var ans []string
+		for k, x := range v {
+			idx := fmt.Sprintf("%q:", k)
+			for _, s := range flatten(x) {
+				ans = append(ans, idx+s)
+			}
+		}
+		return ans
+	default:
+		log.Fatalf("unexpected type %T", x)
+		return nil
+	}
+}