From 73a62a8c2526cd5c2d0ebc3c534e2d4129c5ad7f Mon Sep 17 00:00:00 2001 From: Jagger De Leo Date: Tue, 13 Dec 2022 16:23:05 -0500 Subject: [PATCH 1/5] feat(e2): Port subo K8s commands to new e2 tool --- Makefile | 9 +- e2/README.md | 3 + e2/command/deploy.go | 287 ++++++++++++++++++ e2/command/flags.go | 9 + e2/command/status.go | 26 ++ e2/main.go | 33 ++ e2/templater/config.go | 48 +++ e2/templater/templates.go | 286 +++++++++++++++++ e2/util/cache.go | 38 +++ e2/util/exec.go | 80 +++++ e2/util/exec_test.go | 73 +++++ e2/util/input.go | 20 ++ e2/util/log.go | 53 ++++ e2/util/mkdir.go | 19 ++ e2/util/permissions.go | 16 + e2/util/token.go | 45 +++ e2core/release/version.go | 2 +- .../se2-e2core-deployment.yaml.tmpl | 57 ++++ 18 files changed, 1102 insertions(+), 2 deletions(-) create mode 100644 e2/README.md create mode 100644 e2/command/deploy.go create mode 100644 e2/command/flags.go create mode 100644 e2/command/status.go create mode 100644 e2/main.go create mode 100644 e2/templater/config.go create mode 100644 e2/templater/templates.go create mode 100644 e2/util/cache.go create mode 100644 e2/util/exec.go create mode 100644 e2/util/exec_test.go create mode 100644 e2/util/input.go create mode 100644 e2/util/log.go create mode 100644 e2/util/mkdir.go create mode 100644 e2/util/permissions.go create mode 100644 e2/util/token.go create mode 100644 templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl diff --git a/Makefile b/Makefile index cd95a0c1..23f3fc1c 100644 --- a/Makefile +++ b/Makefile @@ -44,5 +44,12 @@ lintfixer: loadtest: go run ./testingsupport/load/load-tester.go -.PHONY: build e2core e2core/docker docker/dev docker/dev/multi docker/publish docker/builder example-project test lint \ +e2: + go build -o .bin/e2 e2/main.go + +e2/install: + go install ./e2 + +.PHONY: e2core e2core/docker e2 \ + docker/dev docker/dev/multi docker/publish docker/builder example-project test lint \ lint/fix fix-imports diff --git a/e2/README.md b/e2/README.md new file mode 100644 index 00000000..6c8a2fa5 --- /dev/null +++ b/e2/README.md @@ -0,0 +1,3 @@ +# e2 + +A simple CLI tool to deploy and interact with `e2core`. It is released in lockstep with E2 Core. diff --git a/e2/command/deploy.go b/e2/command/deploy.go new file mode 100644 index 00000000..c7e35f30 --- /dev/null +++ b/e2/command/deploy.go @@ -0,0 +1,287 @@ +package command + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/suborbital/e2core/e2/templater" + "github.com/suborbital/e2core/e2/util" + "github.com/suborbital/e2core/e2core/release" +) + +type deployData struct { + E2CoreTag string + EnvToken string + BuilderDomain string + StorageClassName string +} + +const defaultRepo string = "suborbital/e2core" +const defaultBranch = "v" + release.E2CoreServerDotVersion + +// DeployCommand returns the SE2 deploy command. +func DeployCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "deploy", + Short: "Deploy E2 Core to Kubernetes", + Long: "Deploy E2 Core to Kubernetes", + RunE: func(cmd *cobra.Command, args []string) error { + shouldReset := cmd.Flags().Changed(resetFlag) + repo, _ := cmd.Flags().GetString(repoFlag) + branch, _ := cmd.Flags().GetString(branchFlag) + forceUpdateTemplates, _ := cmd.Flags().GetBool(updateTemplatesFlag) + + if err := introAcceptance(); err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return errors.Wrap(err, "🚫 failed to Getwd") + } + + workingDirectory, err := filepath.Abs(cwd) + if err != nil { + return errors.Wrap(err, "🚫 failed to get absolute path") + } + + if forceUpdateTemplates { + util.LogInfo(fmt.Sprintf("updating (forced) templates from %s (%s)", repo, branch)) + + _, err = templater.UpdateTemplates(repo, branch) + if err != nil { + return errors.Wrap(err, "🚫 failed to UpdateTemplates") + } + } + + templatesPath, err := templater.TemplatesExist(repo, branch) + // Fetch templates if they don't exist + if err != nil { + util.LogInfo(fmt.Sprintf("updating templates from %s (%s)", repo, branch)) + templatesPath, err = templater.UpdateTemplates(repo, branch) + + if err != nil { + return errors.Wrap(err, "🚫 failed to UpdateTemplates") + } + } + + // if the --reset flag was passed or there's no existing manifests + // then we need to 'build the world' from scratch. + if shouldReset || !manifestsExist(workingDirectory) { + util.LogStart("preparing deployment") + + // if there are any existing deployment manifests sitting around, let's replace them. + if err := removeExistingManifests(workingDirectory); err != nil { + return errors.Wrap(err, "🚫 failed to removeExistingManifests") + } + + _, err = util.Mkdir(workingDirectory, ".suborbital") + if err != nil { + return errors.Wrap(err, "🚫 failed to Mkdir") + } + + envToken, err := getEnvToken() + if err != nil { + return errors.Wrap(err, "🚫 failed to getEnvToken") + } + + data := deployData{ + E2CoreTag: "v" + release.E2CoreServerDotVersion, + EnvToken: envToken, + } + + templateName := "e2core-k8s" + + data.StorageClassName, err = getStorageClass() + if err != nil { + return errors.Wrap(err, "🚫 failed to getStorageClass") + } + + if err := templater.ExecTmplDir(workingDirectory, "", templatesPath, templateName, data); err != nil { + return errors.Wrap(err, "🚫 failed to ExecTmplDir") + } + + util.LogDone("ready to start installation") + } + + dryRun, _ := cmd.Flags().GetBool(dryRunFlag) + + if dryRun { + util.LogInfo("aborting due to dry-run, manifest files remain in " + workingDirectory) + return nil + } + + util.LogStart("installing...") + + if _, err := util.Command.Run("kubectl apply -f https://github.com/kedacore/keda/releases/download/v2.4.0/keda-2.4.0.yaml"); err != nil { + return errors.Wrap(err, "🚫 failed to install KEDA") + } + + // we don't care if this fails, so don't check error. + util.Command.Run("kubectl create ns suborbital") + + if err := createConfigMap(cwd); err != nil { + return errors.Wrap(err, "🚫 failed to createConfigMap") + } + + if _, err := util.Command.Run("kubectl apply -f .suborbital/"); err != nil { + return errors.Wrap(err, "🚫 failed to kubectl apply") + } + + util.LogDone("installation complete!") + + return nil + }, + } + + cmd.Flags().String(repoFlag, defaultRepo, "git repo to download templates from") + cmd.Flags().String(branchFlag, defaultBranch, "git branch to download templates from") + cmd.Flags().Bool(dryRunFlag, false, "prepare the deployment in the .suborbital directory, but do not apply it") + cmd.Flags().Bool(resetFlag, false, "reset the deployment to default (replaces Kubernetes manifests)") + cmd.Flags().Bool(updateTemplatesFlag, false, "forces templates to be updated") + + return cmd +} + +// TODO: update this +func introAcceptance() error { + fmt.Print(` +Suborbital Extension Engine (SE2) Installer + +BEFORE YOU CONTINUE: + - You must first run "subo se2 create token " to get an environment token + + - You must have kubectl installed in PATH, and it must be connected to the cluster you'd like to use + + - You must be able to set up DNS records for the builder service after this installation completes + - Choose the DNS name you'd like to use before continuing, e.g. builder.acmeco.com + + - Subo will attempt to determine the default storage class for your Kubernetes cluster, + but if is unable to do so you will need to provide one + - See the SE2 documentation for more details + + - Subo will install the KEDA autoscaler into your cluster. It will not affect any existing deployments. + +Are you ready to continue? (y/N): `) + + answer, err := util.ReadStdinString() + if err != nil { + return errors.Wrap(err, "failed to ReadStdinString") + } + + if !strings.EqualFold(answer, "y") { + return errors.New("aborting") + } + + return nil +} + +// getEnvToken gets the environment token from stdin. +func getEnvToken() (string, error) { + existing, err := util.ReadEnvironmentToken() + if err == nil { + util.LogInfo("using cached environment token") + return existing, nil + } + + fmt.Print("Enter your environment token: ") + token, err := util.ReadStdinString() + + if err != nil { + return "", errors.Wrap(err, "failed to ReadStdinString") + } + + if len(token) != 32 { + return "", errors.New("token must be 32 characters in length") + } + + if err := util.WriteEnvironmentToken(token); err != nil { + util.LogWarn(err.Error()) + return token, nil + + } else { + util.LogInfo("saved environment token to cache") + } + + return token, nil +} + +// getStorageClass gets the storage class to use. +func getStorageClass() (string, error) { + defaultClass, err := detectStorageClass() + if err != nil { + // that's fine, continue. + fmt.Println("failed to automatically detect Kubernetes storage class:", err.Error()) + } else if defaultClass != "" { + fmt.Println("using default storage class: ", defaultClass) + return defaultClass, nil + } + + fmt.Print("Enter the Kubernetes storage class to use: ") + storageClass, err := util.ReadStdinString() + if err != nil { + return "", errors.Wrap(err, "failed to ReadStdinString") + } + + if len(storageClass) == 0 { + return "", errors.New("storage class must not be empty") + } + + return storageClass, nil +} + +func detectStorageClass() (string, error) { + output, err := util.Command.Run("kubectl get storageclass --output=name") + if err != nil { + return "", errors.Wrap(err, "failed to get default storageclass") + } + + // output will look like: storageclass.storage.k8s.io/do-block-storage + // so split on the / and return the last part. + + outputParts := strings.Split(output, "/") + if len(outputParts) != 2 { + return "", errors.New("could not automatically determine storage class") + } + + return outputParts[1], nil +} + +func createConfigMap(cwd string) error { + configFilepath := filepath.Join(cwd, "config", "se2-config.yaml") + + _, err := os.Stat(configFilepath) + if err != nil { + return errors.Wrap(err, "failed to Stat se2-config.yaml") + } + + if _, err := util.Command.Run(fmt.Sprintf("kubectl create configmap se2-config --from-file=se2-config.yaml=%s -n suborbital", configFilepath)); err != nil { + return errors.Wrap(err, "failed to create configmap (you may need to run `kubectl delete configmap se2-config -n suborbital`)") + } + + return nil +} + +func manifestsExist(workingDirectory string) bool { + if _, err := os.Stat(filepath.Join(workingDirectory, ".suborbital")); err == nil { + return true + } + + return false +} + +func removeExistingManifests(workingDirectory string) error { + // start with a clean slate. + if _, err := os.Stat(filepath.Join(workingDirectory, ".suborbital")); err == nil { + if err := os.RemoveAll(filepath.Join(workingDirectory, ".suborbital")); err != nil { + return errors.Wrap(err, "failed to RemoveAll .suborbital") + } + } + + return nil +} diff --git a/e2/command/flags.go b/e2/command/flags.go new file mode 100644 index 00000000..d791707d --- /dev/null +++ b/e2/command/flags.go @@ -0,0 +1,9 @@ +package command + +const ( + branchFlag = "branch" + repoFlag = "repo" + updateTemplatesFlag = "update-templates" + dryRunFlag = "dryrun" + resetFlag = "reset" +) diff --git a/e2/command/status.go b/e2/command/status.go new file mode 100644 index 00000000..fb81fe8b --- /dev/null +++ b/e2/command/status.go @@ -0,0 +1,26 @@ +package command + +import ( + "fmt" + + "github.com/spf13/cobra" + "github.com/suborbital/e2core/e2/util" +) + +func StatusCommand() *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Get K8s deployment status", + Long: "Get Kubernetes pods and service status for your E2 Core deployment", + RunE: func(cmd *cobra.Command, args []string) error { + util.LogInfo("Pods:") + util.Command.Run("kubectl get pods -n suborbital") + + fmt.Println() + util.LogInfo("Services:") + util.Command.Run("kubectl get svc -n suborbital") + + return nil + }, + } +} diff --git a/e2/main.go b/e2/main.go new file mode 100644 index 00000000..c4691ec6 --- /dev/null +++ b/e2/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "os" + + "github.com/spf13/cobra" + "github.com/suborbital/e2core/e2/command" + "github.com/suborbital/e2core/e2/util" + "github.com/suborbital/e2core/e2core/release" +) + +func main() { + cmd := &cobra.Command{ + Use: "e2", + Short: "E2 Core CLI", + Version: release.E2CoreServerDotVersion, + Long: `e2 is a simple deployment tool to manage your E2 Core Kubernetes deployments.`, + RunE: func(cmd *cobra.Command, args []string) error { + cmd.Help() + return nil + }, + } + + cmd.SetVersionTemplate("E2 Core deployment CLI v{{.Version}}\n") + + cmd.AddCommand(command.DeployCommand()) + cmd.AddCommand(command.StatusCommand()) + + if err := cmd.Execute(); err != nil { + util.LogFail(err.Error()) + os.Exit(1) + } +} diff --git a/e2/templater/config.go b/e2/templater/config.go new file mode 100644 index 00000000..f71e81b2 --- /dev/null +++ b/e2/templater/config.go @@ -0,0 +1,48 @@ +package templater + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/suborbital/e2core/e2/util" +) + +func FullPath(repo, branch string) (string, error) { + repoParts := strings.Split(repo, "/") + if len(repoParts) != 2 { + return "", fmt.Errorf("repo is invalid, contains %d parts", len(repoParts)) + } + + repoName := repoParts[1] + + root, err := TemplateRootDir() + if err != nil { + return "", errors.Wrap(err, "failed to TemplateRootDir") + } + + return filepath.Join(root, fmt.Sprintf("%s-%s", repoName, strings.ReplaceAll(branch, "/", "-")), "templates"), nil +} + +// TemplateRootDir gets the template directory for subo and ensures it exists. +func TemplateRootDir() (string, error) { + tmplPath, err := util.CacheDir("templates") + if err != nil { + return "", errors.Wrap(err, "failed to CacheDir") + } + + if _, err = os.Stat(tmplPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + if err := os.MkdirAll(tmplPath, util.PermDirectory); err != nil { + return "", errors.Wrap(err, "failed to MkdirAll template directory") + } + } else { + return "", errors.Wrap(err, "failed to Stat template directory") + } + } + + return tmplPath, nil +} diff --git a/e2/templater/templates.go b/e2/templater/templates.go new file mode 100644 index 00000000..96733835 --- /dev/null +++ b/e2/templater/templates.go @@ -0,0 +1,286 @@ +package templater + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "text/template" + + "github.com/pkg/errors" + + "github.com/suborbital/e2core/e2/util" + "github.com/suborbital/systemspec/tenant" +) + +// ErrTemplateMissing and others are template related errors. +var ErrTemplateMissing = errors.New("template missing") + +type tmplData struct { + tenant.Module + NameCaps string + NameCamel string +} + +// GitHub strips the v in the .zip from versioned tags, which means that extracted +// template directories end up having names like `e2core-1.2.3` instead of `e2core-v1.2.3` +// as you might expect. +func normalizeBranch(branch string) string { + if match := regexp.MustCompile(`^v(\d\.\d\.\d)$`).FindStringSubmatch(branch); len(match) == 2 { + branch = match[1] + } + return branch +} + +func UpdateTemplates(repo, branch string) (string, error) { + repoParts := strings.Split(repo, "/") + if len(repoParts) != 2 { + return "", fmt.Errorf("repo is invalid, contains %d parts", len(repoParts)) + } + + repoName := repoParts[1] + + branchDirName := fmt.Sprintf("%s-%s", repoName, strings.ReplaceAll(normalizeBranch(branch), "/", "-")) + + templateRootPath, err := TemplateRootDir() + if err != nil { + return "", errors.Wrap(err, "failed to TemplateDir") + } + + filepathVar, err := downloadZip(repo, branch, templateRootPath) + if err != nil { + return "", errors.Wrap(err, "🚫 failed to downloadZip for templates") + } + + // The tmplPath may be different than the default if a custom URL was provided. + tmplPath, err := extractZip(filepathVar, templateRootPath, branchDirName) + if err != nil { + return "", errors.Wrap(err, "🚫 failed to extractZip for templates") + } + + util.LogDone("templates downloaded") + + return tmplPath, nil +} + +// TemplatesExist returns the templates directory for the provided repo and branch. +func TemplatesExist(repo, branch string) (string, error) { + branch = normalizeBranch(branch) + + repoParts := strings.Split(repo, "/") + if len(repoParts) != 2 { + return "", fmt.Errorf("repo is invalid, contains %d parts", len(repoParts)) + } + + repoName := repoParts[1] + + templateRootPath, err := TemplateRootDir() + if err != nil { + return "", errors.Wrap(err, "failed to TemplateDir") + } + + branchDirName := fmt.Sprintf("%s-%s", repoName, strings.ReplaceAll(branch, "/", "-")) + existingPath := filepath.Join(templateRootPath, branchDirName) + + tmplPath := filepath.Join(existingPath, "templates") + + if files, err := os.ReadDir(tmplPath); err != nil { + return "", errors.Wrap(err, "failed to ReadDir") + } else if len(files) == 0 { + return "", errors.New("templates directory is empty") + } + + return tmplPath, nil +} + +// ExecModuleTmplStr executes a template string with the module's data. +func ExecModuleTmplStr(templateStr string, module *tenant.Module) (string, error) { + templateData := makeTemplateData(module) + + tmpl, err := template.New("tmpl").Parse(templateStr) + if err != nil { + return "", errors.Wrap(err, "failed to parse template string") + } + + builder := &strings.Builder{} + if err := tmpl.Execute(builder, templateData); err != nil { + return "", errors.Wrap(err, "failed to Execute template") + } + + return builder.String(), nil +} + +// ExecModuleTmpl copies a template. +func ExecModuleTmpl(cwd, name, templatesPath string, module *tenant.Module) error { + templateData := makeTemplateData(module) + + return ExecTmplDir(cwd, name, templatesPath, module.Lang, templateData) +} + +// ExecTmplDir copies a generic templated directory. +func ExecTmplDir(cwd, name, templatesPath, tmplName string, templateData interface{}) error { + templatePath := filepath.Join(templatesPath, tmplName) + targetPath := filepath.Join(cwd, name) + + if _, err := os.Stat(templatePath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return ErrTemplateMissing + } + + return errors.Wrap(err, "failed to Stat template directory") + } + + var err = filepath.Walk(templatePath, func(path string, info os.FileInfo, _ error) error { + var relPath = strings.Replace(path, templatePath, "", 1) + if relPath == "" { + return nil + } + + targetRelPath := relPath + if strings.Contains(relPath, ".tmpl") { + tmpl, err := template.New("tmpl").Parse(strings.Replace(relPath, ".tmpl", "", -1)) + if err != nil { + return errors.Wrapf(err, "failed to parse template directory name %s", info.Name()) + } + + builder := &strings.Builder{} + if err := tmpl.Execute(builder, templateData); err != nil { + return errors.Wrapf(err, "failed to Execute template for %s", info.Name()) + } + + targetRelPath = builder.String() + } + + // Check if the target path is an existing file, and skip it if so. + if _, err := os.Stat(filepath.Join(targetPath, targetRelPath)); err != nil { + if os.IsNotExist(err) { + // That's fine, continue. + } else { + return errors.Wrap(err, "failed to Stat") + } + } else { + // If the target file already exists, we're going to skip the rest since we don't want to overwrite. + return nil + } + + if info.IsDir() { + if err := os.Mkdir(filepath.Join(targetPath, targetRelPath), util.PermDirectory); err != nil { + return errors.Wrap(err, "failed to Mkdir") + } + + return nil + } + + var data, err1 = ioutil.ReadFile(filepath.Join(templatePath, relPath)) + if err1 != nil { + return err1 + } + + if strings.HasSuffix(info.Name(), ".tmpl") { + tmpl, err := template.New("tmpl").Parse(string(data)) + if err != nil { + return errors.Wrapf(err, "failed to parse template file %s", info.Name()) + } + + builder := &strings.Builder{} + if err := tmpl.Execute(builder, templateData); err != nil { + return errors.Wrapf(err, "failed to Execute template for %s", info.Name()) + } + + data = []byte(builder.String()) + } + + if err := ioutil.WriteFile(filepath.Join(targetPath, targetRelPath), data, util.PermFilePrivate); err != nil { + return errors.Wrap(err, "failed to WriteFile") + } + + return nil + }) + + return err +} + +// downloadZip downloads a ZIP from a particular branch of the repo. +func downloadZip(repo, branch, targetPath string) (string, error) { + // If downloading from a tag, use full v0.0.0 format. + url := fmt.Sprintf("https://github.com/%s/archive/%s.zip", repo, branch) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return "", errors.Wrap(err, "failed to NewRequest") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", errors.Wrap(err, "failed to Do request") + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("response was non-200: %d", resp.StatusCode) + } + + filepathVar := filepath.Join(targetPath, fmt.Sprintf("e2core-%s.zip", normalizeBranch(branch))) + + // Check if the zip already exists, and delete it if it does. + if _, err := os.Stat(filepathVar); err == nil { + if err := os.Remove(filepathVar); err != nil { + return "", errors.Wrap(err, "failed to delete exising templates zip") + } + } + + if err := os.MkdirAll(targetPath, util.PermDirectory); err != nil { + return "", errors.Wrap(err, "failed to MkdirAll") + } + + file, err := os.Create(filepathVar) + if err != nil { + return "", errors.Wrap(err, "failed to Open file") + } + + defer resp.Body.Close() + if _, err := io.Copy(file, resp.Body); err != nil { + return "", errors.Wrap(err, "failed to Copy data to file") + } + + return filepathVar, nil +} + +// extractZip extracts a ZIP file. +func extractZip(filePath, destPath, branchDirName string) (string, error) { + escapedFilepath := strings.ReplaceAll(filePath, " ", "\\ ") + escapedDestPath := strings.ReplaceAll(destPath, " ", "\\ ") + string(filepath.Separator) + + existingPath := filepath.Join(destPath, branchDirName) + + if _, err := os.Stat(existingPath); err == nil { + if err := os.RemoveAll(existingPath); err != nil { + return "", errors.Wrap(err, "failed to RemoveAll old templates") + } + } + + if _, err := util.Command.Run(fmt.Sprintf("unzip -q %s -d %s", escapedFilepath, escapedDestPath)); err != nil { + return "", errors.Wrap(err, "failed to Run unzip") + } + + return filepath.Join(existingPath, "templates"), nil +} + +// makeTemplateData makes data to be used in templates. +func makeTemplateData(module *tenant.Module) tmplData { + nameCamel := "" + nameParts := strings.Split(module.Name, "-") + for _, part := range nameParts { + nameCamel += strings.ToUpper(string(part[0])) + nameCamel += string(part[1:]) + } + + return tmplData{ + Module: *module, + NameCaps: strings.ToUpper(strings.Replace(module.Name, "-", "", -1)), + NameCamel: nameCamel, + } +} diff --git a/e2/util/cache.go b/e2/util/cache.go new file mode 100644 index 00000000..ffd5d376 --- /dev/null +++ b/e2/util/cache.go @@ -0,0 +1,38 @@ +package util + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +const CacheBaseDir = "suborbital" + +// CacheDir returns the cache directory and creates it if it doesn't exist. If +// no subdirectories are passed it defaults to `suborbital/subo`. +func CacheDir(subdirectories ...string) (string, error) { + tmpPath := os.TempDir() + basePath, err := os.UserCacheDir() + + if err != nil { + // fallback if $HOME is not set. + basePath = tmpPath + } + + base := []string{basePath, CacheBaseDir} + + if len(subdirectories) == 0 { + base = append(base, "subo") + } + + targetPath := filepath.Join(append(base, subdirectories...)...) + + if _, err := os.Stat(targetPath); os.IsNotExist(err) { + if err := os.MkdirAll(targetPath, PermDirectory); err != nil { + return "", errors.Wrap(err, "failed to MkdirAll") + } + } + + return targetPath, nil +} diff --git a/e2/util/exec.go b/e2/util/exec.go new file mode 100644 index 00000000..c8a9d661 --- /dev/null +++ b/e2/util/exec.go @@ -0,0 +1,80 @@ +package util + +import ( + "bytes" + "io" + "os" + "os/exec" + + "github.com/pkg/errors" +) + +type CommandRunner interface { + Run(cmd string) (string, error) + RunInDir(cmd, dir string) (string, error) +} + +type silentOutput bool + +const ( + SilentOutput = true + NormalOutput = false +) + +type CommandLineExecutor struct { + silent silentOutput + writer io.Writer +} + +// Command is a barebones command executor. +var Command = &CommandLineExecutor{} + +// NewCommandLineExecutor creates a new CommandLineExecutor with the given configuration. +func NewCommandLineExecutor(silent silentOutput, writer io.Writer) *CommandLineExecutor { + return &CommandLineExecutor{ + silent: silent, + writer: writer, + } +} + +// Run runs a command, outputting to terminal and returning the full output and/or error. +func (d *CommandLineExecutor) Run(cmd string) (string, error) { + return run(cmd, "", d.silent, d.writer) +} + +// RunInDir runs a command in the specified directory and returns the full output or error. +func (d *CommandLineExecutor) RunInDir(cmd, dir string) (string, error) { + return run(cmd, dir, d.silent, d.writer) +} + +func run(cmd, dir string, silent silentOutput, writer io.Writer) (string, error) { + // you can uncomment this below if you want to see exactly the commands being run + // fmt.Println("▶️", cmd). + + command := exec.Command("sh", "-c", cmd) + + command.Dir = dir + + var outBuf bytes.Buffer + + if silent { + command.Stdout = &outBuf + command.Stderr = &outBuf + } else if writer != nil { + command.Stdout = io.MultiWriter(os.Stdout, &outBuf, writer) + command.Stderr = io.MultiWriter(os.Stderr, &outBuf, writer) + } else { + command.Stdout = io.MultiWriter(os.Stdout, &outBuf) + command.Stderr = io.MultiWriter(os.Stderr, &outBuf) + } + + runErr := command.Run() + + outStr := outBuf.String() + + if runErr != nil { + return outStr, errors.Wrap(runErr, "failed to Run command") + } + + return outStr, nil +} diff --git a/e2/util/exec_test.go b/e2/util/exec_test.go new file mode 100644 index 00000000..76cd422c --- /dev/null +++ b/e2/util/exec_test.go @@ -0,0 +1,73 @@ +package util + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCommandRunner_Run(t *testing.T) { + tests := []struct { + name string + cmd string + runner func() (CommandRunner, *bytes.Buffer) + want string + wantErr assert.ErrorAssertionFunc + wantBuf []byte + }{ + + { + name: "writes to a supplied io.Writer", + cmd: "echo 'the mitochondria is the powerhouse of the cell'", + runner: func() (CommandRunner, *bytes.Buffer) { + buf := new(bytes.Buffer) + return NewCommandLineExecutor(NormalOutput, buf), buf + }, + want: "the mitochondria is the powerhouse of the cell\n", + wantErr: assert.NoError, + wantBuf: []byte("the mitochondria is the powerhouse of the cell\n"), + }, + { + name: "writes nothing to a supplied io.Writer when silent", + cmd: "echo 'the mitochondria is the powerhouse of the cell'", + runner: func() (CommandRunner, *bytes.Buffer) { + buf := new(bytes.Buffer) + return NewCommandLineExecutor(SilentOutput, buf), buf + }, + want: "the mitochondria is the powerhouse of the cell\n", + wantErr: assert.NoError, + wantBuf: nil, + }, + { + name: "accepts a nil io.Writer", + cmd: "echo 'the mitochondria is the powerhouse of the cell'", + runner: func() (CommandRunner, *bytes.Buffer) { + return NewCommandLineExecutor(SilentOutput, nil), new(bytes.Buffer) + }, + want: "the mitochondria is the powerhouse of the cell\n", + wantErr: assert.NoError, + wantBuf: nil, + }, + { + name: "performs with the default executor", + cmd: "echo 'the mitochondria is the powerhouse of the cell'", + runner: func() (CommandRunner, *bytes.Buffer) { + return Command, new(bytes.Buffer) + }, + want: "the mitochondria is the powerhouse of the cell\n", + wantErr: assert.NoError, + wantBuf: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runner, buf := tt.runner() + got, err := runner.Run(tt.cmd) + + tt.wantErr(t, err) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.wantBuf, buf.Bytes()) + }) + } +} diff --git a/e2/util/input.go b/e2/util/input.go new file mode 100644 index 00000000..185bea22 --- /dev/null +++ b/e2/util/input.go @@ -0,0 +1,20 @@ +package util + +import ( + "bufio" + "os" + + "github.com/pkg/errors" +) + +// ReadStdinString reads a string from stdin. +func ReadStdinString() (string, error) { + scanner := bufio.NewScanner(os.Stdin) + scanner.Scan() + + if err := scanner.Err(); err != nil { + return "", errors.Wrap(err, "failed to scanner.Scan") + } + + return scanner.Text(), nil +} diff --git a/e2/util/log.go b/e2/util/log.go new file mode 100644 index 00000000..9ac3cf0a --- /dev/null +++ b/e2/util/log.go @@ -0,0 +1,53 @@ +package util + +import ( + "fmt" +) + +// FriendlyLogger describes a logger designed to provide friendly output for interactive CLI purposes. +type FriendlyLogger interface { + LogInfo(string) + LogStart(string) + LogDone(string) + LogFail(string) + LogWarn(string) +} + +// PrintLogger is a struct wrapper around the logging plugins used by Subo. +type PrintLogger struct{} + +func (p *PrintLogger) LogInfo(msg string) { LogInfo(msg) } +func (p *PrintLogger) LogStart(msg string) { LogStart(msg) } +func (p *PrintLogger) LogDone(msg string) { LogDone(msg) } +func (p *PrintLogger) LogFail(msg string) { LogFail(msg) } +func (p *PrintLogger) LogWarn(msg string) { LogWarn(msg) } + +// Keeping it DRY. +func log(msg string) { + fmt.Println(msg) +} + +// LogInfo logs information. +func LogInfo(msg string) { + log(fmt.Sprintf("ℹ️ %s", msg)) +} + +// LogStart logs the start of something. +func LogStart(msg string) { + log(fmt.Sprintf("⏩ START: %s", msg)) +} + +// LogDone logs the success of something. +func LogDone(msg string) { + log(fmt.Sprintf("✅ DONE: %s", msg)) +} + +// LogFail logs the failure of something. +func LogFail(msg string) { + log(fmt.Sprintf("🚫 FAILED: %s", msg)) +} + +// LogWarn logs a warning from something. +func LogWarn(msg string) { + log(fmt.Sprintf("⚠️ WARNING: %s", msg)) +} diff --git a/e2/util/mkdir.go b/e2/util/mkdir.go new file mode 100644 index 00000000..f25a65d5 --- /dev/null +++ b/e2/util/mkdir.go @@ -0,0 +1,19 @@ +package util + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Mkdir creates a new directory to contain a module. +func Mkdir(cwd, name string) (string, error) { + path := filepath.Join(cwd, name) + + if err := os.Mkdir(path, PermDirectory); err != nil { + return "", errors.Wrap(err, "failed to Mkdir") + } + + return path, nil +} diff --git a/e2/util/permissions.go b/e2/util/permissions.go new file mode 100644 index 00000000..83dd8dac --- /dev/null +++ b/e2/util/permissions.go @@ -0,0 +1,16 @@ +package util + +import ( + "io/fs" +) + +// These constants are meant to be used as reasonable default values for files and directories. +// nolint:godot +const ( + PermDirectory fs.FileMode = 0755 // rwxr-xr-x + PermDirectoryPrivate fs.FileMode = 0700 // rwx------ + PermExecutable fs.FileMode = 0755 // rwxr-xr-x + PermExecutablePrivate fs.FileMode = 0700 // rwx------ + PermFile fs.FileMode = 0644 // rw-r--r-- + PermFilePrivate fs.FileMode = 0600 // rw------- +) diff --git a/e2/util/token.go b/e2/util/token.go new file mode 100644 index 00000000..b8f30d2a --- /dev/null +++ b/e2/util/token.go @@ -0,0 +1,45 @@ +package util + +import ( + "io/ioutil" + "path/filepath" + + "github.com/pkg/errors" +) + +func getTokenPath() (string, error) { + tokenPath, err := CacheDir("compute") + if err != nil { + return "", errors.Wrap(err, `failed to CacheDir("compute")`) + } + + return filepath.Join(tokenPath, "envtoken"), nil +} + +func WriteEnvironmentToken(tokenStr string) error { + tokenPath, err := getTokenPath() + if err != nil { + return errors.Wrap(err, "failed to getTokenPath") + } + + if err := ioutil.WriteFile(tokenPath, []byte(tokenStr), PermFilePrivate); err != nil { + return errors.Wrapf(err, "failed to write %s", tokenPath) + } + + return nil +} + +func ReadEnvironmentToken() (string, error) { + tokenPath, err := getTokenPath() + if err != nil { + return "", errors.Wrap(err, "failed to getTokenPath") + } + + buf, err := ioutil.ReadFile(tokenPath) + + if err != nil { + return "", errors.Wrapf(err, "failed to read %s", tokenPath) + } + + return string(buf), nil +} diff --git a/e2core/release/version.go b/e2core/release/version.go index 268123d4..0f0bd10f 100644 --- a/e2core/release/version.go +++ b/e2core/release/version.go @@ -1,4 +1,4 @@ package release // E2CoreServerDotVersion represents the dot version for E2Core Server -var E2CoreServerDotVersion = "0.5.1" +const E2CoreServerDotVersion = "0.5.1" diff --git a/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl b/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl new file mode 100644 index 00000000..5ac49cab --- /dev/null +++ b/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment + +metadata: + name: se2-e2core-deployment + namespace: suborbital + labels: + app: se2-e2core + +spec: + replicas: 1 + + selector: + matchLabels: + app: se2-e2core + + template: + metadata: + labels: + app: se2-e2core + + spec: + containers: + - name: e2core + image: suborbital/e2core:{{ .E2CoreTag }} + command: ["e2core start"] + + ports: + - containerPort: 8080 + + env: + - name: E2CORE_HTTP_PORT + value: "8080" + + - name: E2CORE_LOG_LEVEL + value: "info" + + - name: E2CORE_CONTROL_PLANE + value: "se2-controlplane-service:8081" + + - name: E2CORE_API_FEATURES + value: "adminV1" + +--- + +apiVersion: v1 +kind: Service +metadata: + namespace: suborbital + name: se2-e2core-service +spec: + selector: + app: se2-e2core + ports: + - protocol: TCP + port: 80 + targetPort: 8080 \ No newline at end of file From 8afe547c3281780f52b69dd0d2d831d773dc770e Mon Sep 17 00:00:00 2001 From: Jagger De Leo Date: Tue, 13 Dec 2022 16:32:36 -0500 Subject: [PATCH 2/5] fix: handle slashes in branch names --- e2/templater/templates.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2/templater/templates.go b/e2/templater/templates.go index 96733835..70420a88 100644 --- a/e2/templater/templates.go +++ b/e2/templater/templates.go @@ -206,7 +206,7 @@ func ExecTmplDir(cwd, name, templatesPath, tmplName string, templateData interfa // downloadZip downloads a ZIP from a particular branch of the repo. func downloadZip(repo, branch, targetPath string) (string, error) { - // If downloading from a tag, use full v0.0.0 format. + // If downloading from a tag, use full v0.0.0 format, do not normalize. url := fmt.Sprintf("https://github.com/%s/archive/%s.zip", repo, branch) req, err := http.NewRequest(http.MethodGet, url, nil) @@ -223,7 +223,7 @@ func downloadZip(repo, branch, targetPath string) (string, error) { return "", fmt.Errorf("response was non-200: %d", resp.StatusCode) } - filepathVar := filepath.Join(targetPath, fmt.Sprintf("e2core-%s.zip", normalizeBranch(branch))) + filepathVar := filepath.Join(targetPath, fmt.Sprintf("e2core-%s.zip", strings.ReplaceAll(normalizeBranch(branch), "/", "-"))) // Check if the zip already exists, and delete it if it does. if _, err := os.Stat(filepathVar); err == nil { From efac1ccc8b97f6fb5e62a2545bbb6ac103fd8082 Mon Sep 17 00:00:00 2001 From: Jagger De Leo Date: Tue, 13 Dec 2022 16:36:05 -0500 Subject: [PATCH 3/5] chore: lint --- e2/command/status.go | 1 + e2/main.go | 1 + 2 files changed, 2 insertions(+) diff --git a/e2/command/status.go b/e2/command/status.go index fb81fe8b..5a3baf8d 100644 --- a/e2/command/status.go +++ b/e2/command/status.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/spf13/cobra" + "github.com/suborbital/e2core/e2/util" ) diff --git a/e2/main.go b/e2/main.go index c4691ec6..61833494 100644 --- a/e2/main.go +++ b/e2/main.go @@ -4,6 +4,7 @@ import ( "os" "github.com/spf13/cobra" + "github.com/suborbital/e2core/e2/command" "github.com/suborbital/e2core/e2/util" "github.com/suborbital/e2core/e2core/release" From 0682668e4498574698dd08ec3d3f2e80e7bae2a4 Mon Sep 17 00:00:00 2001 From: Jagger De Leo Date: Thu, 15 Dec 2022 12:21:15 -0500 Subject: [PATCH 4/5] remove templates dir --- .../se2-e2core-deployment.yaml.tmpl | 57 ------------------- 1 file changed, 57 deletions(-) delete mode 100644 templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl diff --git a/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl b/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl deleted file mode 100644 index 5ac49cab..00000000 --- a/templates/e2core-k8s/.suborbital/se2-e2core-deployment.yaml.tmpl +++ /dev/null @@ -1,57 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment - -metadata: - name: se2-e2core-deployment - namespace: suborbital - labels: - app: se2-e2core - -spec: - replicas: 1 - - selector: - matchLabels: - app: se2-e2core - - template: - metadata: - labels: - app: se2-e2core - - spec: - containers: - - name: e2core - image: suborbital/e2core:{{ .E2CoreTag }} - command: ["e2core start"] - - ports: - - containerPort: 8080 - - env: - - name: E2CORE_HTTP_PORT - value: "8080" - - - name: E2CORE_LOG_LEVEL - value: "info" - - - name: E2CORE_CONTROL_PLANE - value: "se2-controlplane-service:8081" - - - name: E2CORE_API_FEATURES - value: "adminV1" - ---- - -apiVersion: v1 -kind: Service -metadata: - namespace: suborbital - name: se2-e2core-service -spec: - selector: - app: se2-e2core - ports: - - protocol: TCP - port: 80 - targetPort: 8080 \ No newline at end of file From a25896c76b85af081b97ce9bc8e0b94f64f2c91b Mon Sep 17 00:00:00 2001 From: Jagger De Leo Date: Thu, 15 Dec 2022 12:36:42 -0500 Subject: [PATCH 5/5] feat: use suborbital/templates repo --- e2/command/deploy.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/e2/command/deploy.go b/e2/command/deploy.go index c7e35f30..5e0edf12 100644 --- a/e2/command/deploy.go +++ b/e2/command/deploy.go @@ -3,6 +3,7 @@ package command import ( "fmt" "os" + "path" "path/filepath" "strings" @@ -15,14 +16,17 @@ import ( ) type deployData struct { - E2CoreTag string - EnvToken string - BuilderDomain string + Identifier string + ImageName string + EnvToken string + Domain string + AppVersion string + StorageClassName string } -const defaultRepo string = "suborbital/e2core" -const defaultBranch = "v" + release.E2CoreServerDotVersion +const defaultRepo string = "suborbital/templates" +const defaultBranch = "main" // DeployCommand returns the SE2 deploy command. func DeployCommand() *cobra.Command { @@ -91,11 +95,13 @@ func DeployCommand() *cobra.Command { } data := deployData{ - E2CoreTag: "v" + release.E2CoreServerDotVersion, - EnvToken: envToken, + Identifier: path.Base(workingDirectory), + ImageName: "suborbital/e2core:v" + release.E2CoreServerDotVersion, + Domain: "prd.suborbital.network", // TODO + EnvToken: envToken, } - templateName := "e2core-k8s" + templateName := "k8s" data.StorageClassName, err = getStorageClass() if err != nil {