Jack/fix mermaid namespaces (#63)

* Improve diagram error messages

* Add sysl_modules to gitignore

* Bump to sysl v0.228.0

* Remove unnecessary flag

* Add support for new sysl_modules

* Add instructions for SYSL_TOKENS config

* Migrate Redoc to use new sysl Retriever

* Fix tests and remove unused code

* Fix broken anchor links from apps with namespace

Clean db name

* Use working directory rather than hardcoding
Fix type error

* Speed up mermaid data diagrams

* Speed up docker builds


Add comments to dockerfile

.gitignore

@@ -1,3 +1,4 @@
 .sysl-tmp
 .vscode/*
 coverage.txt
+sysl_modules

Makefile

@@ -18,7 +18,7 @@ test-integration: # Run integration tests (against Github API. Requires SYSL_GIT
 .PHONY: demo demo-html demo-markdown
 demo: demo-html demo-markdown
 demo-html:
-	sysl-catalog --type=html --plantuml=https://plantuml.com/plantuml -o demo/html demo/demo.sysl --redoc --mermaid
+	sysl-catalog --type=html -o demo/html demo/demo.sysl --redoc --mermaid
 demo-markdown:
 	sysl-catalog -o demo/markdown demo/demo.sysl --mermaid
 demo-server:

README.md

@@ -91,6 +91,11 @@ services:
 ## How to use
 1. Set up environment
 
+
+```bash
+export SYSL_TOKENS=github.com:<YOUR_TOKEN_HERE>
+```
+
 ```bash
 export SYSL_PLANTUML=http://www.plantuml.com/plantuml`
 ```

demo/demo.sysl

@@ -1,5 +1,5 @@
-import //github.com/anz-bank/sysl-examples/demos/sizzle_restaurant/sizzle.sysl
-import //github.com/anz-bank/sysl-examples/demos/sizzle_restaurant/mastercard.yaml as MasterCard.MasterCard
+import //github.com/anz-bank/sysl-examples/demos/sizzle_restaurant/sizzle.sysl@master
+import //github.com/anz-bank/sysl-examples/demos/sizzle_restaurant/mastercard.yaml@master as MasterCard.MasterCard
 
 Sizzle[~project]:
     @contact.name = "Jimmy Smith"

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/anz-bank/mermaid-go v0.1.1
 	github.com/anz-bank/pkg v0.0.25
 	github.com/anz-bank/protoc-gen-sysl v0.0.17
-	github.com/anz-bank/sysl v0.223.0
+	github.com/anz-bank/sysl v0.228.0
 	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/getkin/kin-openapi v0.18.0 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
@@ -19,6 +19,7 @@ require (
 	github.com/hashicorp/go-retryablehttp v0.6.6
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334
+	github.com/joshcarp/gop v0.0.0-20200922043230-a225272c1746
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1

main.go

@@ -9,7 +9,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/anz-bank/sysl/pkg/loader"
+	"github.com/anz-bank/sysl/pkg/parse"
+
 	"github.com/anz-bank/sysl/pkg/sysl"
 
 	"github.com/anz-bank/sysl-catalog/pkg/catalog"
@@ -92,7 +93,8 @@ func main() {
 				m, err = parseSyslFile(".", *input, fs, logger)
 			} else {
 				var changedModule *sysl.Module
-				relativeChangedFilePath := "." + strings.TrimPrefix(i.(watch.Event).Path, "/usr")
+				wd, _ := os.Getwd()
+				relativeChangedFilePath := "." + strings.TrimPrefix(i.(watch.Event).Path, wd)
 				changedModule, err = parseSyslFile(".", relativeChangedFilePath, fs, logger)
 				if err == nil {
 					m = overwriteSyslModules(handler.RootModule, changedModule)
@@ -152,7 +154,10 @@ func setupLogger() *logrus.Logger {
 func parseSyslFile(root string, filename string, fs afero.Fs, logger *logrus.Logger) (*sysl.Module, error) {
 	logger.Info("Parsing...")
 	start := time.Now()
-	m, _, err := loader.LoadSyslModule(root, filename, fs, logger)
+	m, err := parse.NewParser().ParseFromFs(filename, fs)
+	if err != nil {
+		return nil, err
+	}
 	elapsed := time.Since(start)
 	logger.Info("Done, time elapsed: ", elapsed)
 	return m, err

mermaid.Dockerfile

@@ -1,5 +1,14 @@
 # Build sysl-catalog binary
 FROM golang:alpine AS builder
+
+# Caches go dependencies in the first layer 
+# Speeds up builds when go.mod and go.sum is unchanged
+RUN mkdir /src
+WORKDIR /src
+COPY go.mod .
+COPY go.sum .
+RUN go mod download
+
 ADD . /src
 RUN cd /src && CGO_ENABLED=0 go build -o sysl-catalog
 

pkg/catalog/diagram.go

@@ -13,6 +13,7 @@ import (
 	"text/template"
 
 	"github.com/pkg/errors"
+	"github.com/spf13/afero"
 
 	"github.com/yuin/goldmark"
 	"github.com/yuin/goldmark/extension"
@@ -145,7 +146,7 @@ func (p *Generator) CreateIntegrationDiagramPlantuml(m *sysl.Module, title strin
 func (p *Generator) CreateSequenceDiagram(appName string, endpoint *sysl.Endpoint) string {
 	defer func() {
 		if err := recover(); err != nil {
-			p.Log.Errorf("error creating sequence diagram: %s", err)
+			p.Log.Errorf("error creating sequence diagram for %s %s: %s", appName, endpoint.Name, err)
 		}
 	}()
 	if p.Mermaid {
@@ -300,7 +301,7 @@ func (p *Generator) CreateReturnDataModel(appname string, stmnt *sysl.Statement,
 func (p *Generator) CreateTypeDiagram(appName string, typeName string, t *sysl.Type, recursive bool) string {
 	defer func() {
 		if err := recover(); err != nil {
-			p.Log.Errorf("error creating type diagram: %s", err)
+			p.Log.Errorf("error creating type diagram for %s %s: %s", appName, typeName, err)
 		}
 	}()
 	if p.Mermaid {
@@ -313,14 +314,13 @@ func (p *Generator) CreateAliasedTypeDiagramMermaid(appName string, typeName str
 	if appName == "" || typeName == "" {
 		return ""
 	}
-	m := p.RootModule
 	var mermaidString string
 	var err error
 
 	if appName == "primitive" {
 		mermaidString += datamodeldiagram.GeneratePrimitive(typeName)
 	} else {
-		mermaidString, err = datamodeldiagram.GenerateDataDiagramWithAppAndType(m, appName, typeName)
+		mermaidString, err = datamodeldiagram.GenerateDataDiagramWithMapper(p.Mapper, appName, typeName)
 		if err != nil {
 			return ""
 		}
@@ -388,7 +388,7 @@ func (p *Generator) CreateRedoc(app *sysl.Application, appName string) string {
 		return ""
 	}
 
-	importPath, version, err := GetImportPathAndVersion(app)
+	importPath, version, err := GetImportPathAndVersion(app, afero.NewOsFs())
 	if err != nil {
 		p.Log.Error(err)
 		return ""

pkg/catalog/diagram_test.go

@@ -303,10 +303,9 @@ func TestCreateRedoc(t *testing.T) {
 		Redoc:              true,
 	}
 	link := gen.CreateRedoc(app, appName)
-	t.Log(gen.RedocFilesToCreate)
 	registeredFile, ok := gen.RedocFilesToCreate["myAppName/myappname.redoc.html"]
 	assert.True(t, ok)
-	assert.Equal(t, "/sysl/myfile.yaml", registeredFile)
+	assert.Equal(t, "/sysl/myfile.yaml@master", registeredFile)
 	assert.Equal(t, "myappname.redoc.html", link)
 }
 

pkg/catalog/generator.go

@@ -20,6 +20,7 @@ import (
 	"github.com/iancoleman/strcase"
 
 	"github.com/anz-bank/sysl/pkg/syslutil"
+	"github.com/anz-bank/sysl/pkg/syslwrapper"
 
 	"github.com/sirupsen/logrus"
 
@@ -75,6 +76,7 @@ type Generator struct {
 	Server     bool
 
 	MermaidGen *MermaidGenerator // This is used for mermaid diagram generation
+	Mapper     *syslwrapper.AppMapper
 
 	BasePath string // for using on another endpoint that isn't '/'
 }
@@ -267,7 +269,7 @@ func (p *Generator) Run() {
 
 	wg.Wait()
 	if p.CopySpecsToOutput {
-		err := CopySyslModCache(p.OutputDir)
+		err := CopySyslCacheDir(p.OutputDir)
 		if err != nil {
 			logrus.Warn(err)
 		}

pkg/catalog/server.go

@@ -12,6 +12,7 @@ import (
 	"sync"
 
 	"github.com/anz-bank/sysl/pkg/sysl"
+	"github.com/anz-bank/sysl/pkg/syslwrapper"
 	"github.com/spf13/afero"
 )
 
@@ -21,6 +22,9 @@ func (p *Generator) Update(m *sysl.Module, errs ...error) *Generator {
 	p.RootModule = m
 	p.GeneratedFiles = make(map[string][]byte)
 	p.GeneratedFilesMutex = &sync.RWMutex{}
+	p.Mapper = syslwrapper.MakeAppMapper(m)
+	p.Mapper.IndexTypes()
+	p.Mapper.ConvertTypes()
 	if p.RootModule != nil && len(p.ModuleAsMacroPackage(p.RootModule)) <= 1 && !p.CustomTemplate {
 		p.StartTemplateIndex = 1 // skip the MacroPackageProject
 	} else {

pkg/catalog/spec.go

@@ -2,26 +2,22 @@ package catalog
 
 import (
 	"fmt"
-	"net/url"
-	"os"
 	"os/exec"
 	"path"
 	"strings"
 
 	"github.com/anz-bank/pkg/mod"
+	"github.com/anz-bank/sysl/pkg/env"
 	"github.com/anz-bank/sysl/pkg/sysl"
+	"github.com/joshcarp/gop/gop/cli"
 	"github.com/spf13/afero"
 )
 
-// CopySyslModCache copies ALL the contents of the sysl module cache directory
+// CopySyslCacheDir copies ALL the contents of the sysl module cache directory
 // (typically found in ~/.sysl/github.com) and outputs it to the specified folder
-func CopySyslModCache(targetDir string) error {
-	homeDir, err := os.UserHomeDir()
-	if err != nil {
-		return err
-	}
-	syslCacheDir := homeDir + "/.sysl/github.com"
-	cpSpecsToOutputDir := exec.Command("cp", "-r", syslCacheDir, targetDir)
+func CopySyslCacheDir(targetDir string) error {
+	syslCacheDir := env.SYSL_CACHE.Value()
+	cpSpecsToOutputDir := exec.Command("cp", "-r", syslCacheDir+"/.", targetDir)
 	if err := cpSpecsToOutputDir.Run(); err != nil {
 		return err
 	}
@@ -37,26 +33,48 @@ func IsOpenAPIFile(filePath string) bool {
 	return false
 }
 
+func initRetriever(fs afero.Fs) (*cli.Retriever, error) {
+	tokensEnv := env.SYSL_TOKENS.Value() // Expects the token to be in form gita.com:<tokena>,gitb.com:<tokenb>
+	var hostTokens []string
+	var cache, proxy string
+	if tokensEnv != "" {
+		hostTokens = strings.Split(tokensEnv, ",")
+	}
+	tokenmap := make(map[string]string, len(hostTokens))
+	for _, e := range hostTokens {
+		arr := strings.Split(e, ":")
+		if len(arr) < 2 {
+			return nil, fmt.Errorf("SYSL_TOKENS env var is invalid, should be in form `gita.com:<tokena>,gitb.com:<tokenb>`")
+		}
+		tokenmap[arr[0]] = arr[1]
+	}
+	if moduleFlag := env.SYSL_MODULES.Value(); moduleFlag != "" && moduleFlag != "false" && moduleFlag != "off" {
+		cache = env.SYSL_CACHE.Value()
+		proxy = env.SYSL_PROXY.Value()
+	}
+	retriever := cli.Default(fs, cache, proxy, tokenmap)
+	return &retriever, nil
+}
+
 // GetImportPathAndVersion takes a Sysl Application and returns an import and version string
 // The import path is of format github.com/org/repo/myfile.yaml
 // it CAN contain version tags e.g github.com/org/repo/myfile.yaml@develop or github.com/org/repo/myfile.yaml@v1.1.0
 // The version string is of format version-12digitCommitSHA e.g v0.0.0-c63b9e92813a
-func GetImportPathAndVersion(app *sysl.Application) (importPath string, version string, err error) {
+func GetImportPathAndVersion(app *sysl.Application, fs afero.Fs) (importPath string, version string, err error) {
 	specURL, ok := app.Attrs["redoc-spec"]
 	if ok {
-		homeDir, _ := os.UserHomeDir()
-		// Fetch the OpenAPI spec file into the cached ~/.sysl directory
-		err2 := mod.Config("github", mod.GoModulesOptions{}, mod.GitHubOptions{CacheDir: homeDir + "/.sysl", AccessToken: os.Getenv("SYSL_GITHUB_TOKEN"), Fs: afero.NewOsFs()}) // Setup sysl module in Github mode
-		if err2 != nil {
-			return "", "", err2
+		retriever, err := initRetriever(fs)
+		if err != nil {
+			return "", "", err
 		}
-		name, ver := mod.ExtractVersion(specURL.GetS())
-		remoteFileMod, err := mod.Retrieve(name, ver)
+		_, _, err = retriever.Retrieve(specURL.GetS())
 		if err != nil {
 			return "", "", err
 		}
+		_, ver := mod.ExtractVersion(specURL.GetS())
+
 		importPath = specURL.GetS()
-		version = remoteFileMod.Version
+		version = ver
 	} else {
 		importPath = app.SourceContext.GetFile()
 		version = app.SourceContext.GetVersion()
@@ -67,82 +85,13 @@ func GetImportPathAndVersion(app *sysl.Application) (importPath string, version
 // BuildSpecURL takes a filepath and version and builds a URL to the cached spec file
 // It also trims . prefixes and adds a / so that the URL is relative
 func BuildSpecURL(filePath string, version string) string {
-	// Remove trailing @tag or @branchname
-	repoPath, _ := mod.ExtractVersion(filePath)
-	// Append the version tag to the repo name
-	if version != "" {
-		repoPath = AppendVersion(repoPath, version)
+	_, ver := mod.ExtractVersion(filePath)
+	if ver == "" {
+		filePath = filePath + "@master"
 	}
-	repoPath = strings.TrimPrefix(repoPath, ".")
+	repoPath := strings.TrimPrefix(filePath, ".")
 	if !strings.HasPrefix(repoPath, "/") {
 		repoPath = "/" + repoPath
 	}
 	return repoPath
 }
-
-// AppendVersion takes in a remote file import path and a version, and appends the version tag to the repo name separated by the '@' char
-// e.g for an input
-// - remoteFilePath github.com/anz-bank/sysl-examples/demos/grocerystore/grocerystore.sysl
-// - version v0.0.0-c63b9e92813a
-// it returns /github.com/anz-bank/sysl-examples@v0.0.0-c63b9e92813a/demos/grocerystore/grocerystore.sysl
-func AppendVersion(remoteFilePath string, version string) string {
-	names := strings.FieldsFunc(remoteFilePath, func(c rune) bool {
-		return c == '/'
-	})
-	if len(names) < 3 {
-		return ""
-	}
-	names[2] = names[2] + "@" + version
-	return path.Join(names...)
-}
-
-// GetRemoteFromGit gets the URL to the git remote
-// e.g github.com/myorg/somerepo/
-func GetRemoteFromGit() (string, error) {
-	cmd := exec.Command("git", "config", "--get", "remote.origin.url")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-
-		return "", fmt.Errorf("error getting git remote: is sysl-catalog running in a git repo? %w", err)
-	}
-	return StripExtension(string(out)), nil
-}
-
-// StripExtension removes spaces and suffixes
-func StripExtension(input string) string {
-	noExt := strings.TrimSuffix(input, path.Ext(input))
-	noSpace := strings.TrimSpace(noExt)
-	return noSpace
-}
-
-// BuildGithubRawURL gets the base URL for raw content hosted on github.com or Github Enterprise
-// For github.com it takes in https://github.com/anz-bank/sysl-catalog and returns https://raw.githubusercontent.com/anz-bank/sysl-catalog/master/
-// For Github Enterprise it takes in https://github.myorg.com/anz-bank/sysl-catalog and returns https://github.myorg.com/raw/anz-bank/sysl-catalog/master/
-func BuildGithubRawURL(repoURL string) (gitURL string) {
-	url, err := url.Parse(repoURL)
-	if err != nil {
-		panic(err)
-	}
-	switch url.Host {
-	case "github.com":
-		url.Host = "raw.githubusercontent.com"
-		url.Path = url.Path + "/master/"
-		gitURL = url.String()
-	default:
-		// Handles github enterprise which uses a different URL scheme for raw files
-		url.Path = "raw" + url.Path + "/master/"
-		gitURL = url.String()
-	}
-	return gitURL
-}
-
-// BuildGithubBlobURL creates a root URL for github blob
-// it will not work for non github links.
-func BuildGithubBlobURL(repoURL string) string {
-	url, err := url.Parse(repoURL)
-	if err != nil {
-		panic(err)
-	}
-	url.Path = path.Join(url.Path, "/blob/master/")
-	return url.String()
-}