Replace go-bindata-assetfs build dependency with native go:embed (#11208

)

* copy over the webui

move web_ui to http

remove web ui files, add .gitkeep

updates, messing with gitkeep and ignoring web_ui

update ui scripts

gitkeep

ignore http/web_ui

Remove debugging

remove the jwt reference, that was from something else

restore old jwt plugin

move things around

Revert "move things around"

This reverts commit 2a35121.

Update ui path handling to not need the web_ui name part

add desc

move the http.FS conversion internal to assetFS

update gitignore

remove bindata dep

clean up some comments

remove asset check script that's no longer needed

Update readme

remove more bindata things

restore asset check

update packagespec

update stub

stub the assetFS method and set uiBuiltIn to false for non-ui builds

update packagespec to build ui

* fail if assets aren't found

* tidy up vendor

* go mod tidy

* updating .circleci

* restore tools.go

* re-re-re-run make packages

* re-enable arm64

* Adding change log

* Removing a file

Co-authored-by: hamid ghaf <hamid@hashicorp.com>

.gitignore

@@ -28,6 +28,9 @@ _testmain.go
 /bin/
 /pkg/
 
+# Generated Web UI goes here
+/http/web_ui/**
+
 # Vault-specific
 example.hcl
 example.vault.d
@@ -73,7 +76,6 @@ dist/*
 ui/dist
 ui/tmp
 ui/root
-http/bindata_assetfs.go
 
 # dependencies
 ui/node_modules

Makefile

@@ -8,8 +8,6 @@ EXTENDED_TEST_TIMEOUT=60m
 INTEG_TEST_TIMEOUT=120m
 VETARGS?=-asmdecl -atomic -bool -buildtags -copylocks -methods -nilfunc -printf -rangeloops -shift -structtags -unsafeptr
 EXTERNAL_TOOLS_CI=\
-	github.com/elazarl/go-bindata-assetfs/... \
-	github.com/hashicorp/go-bindata/... \
 	github.com/mitchellh/gox \
 	golang.org/x/tools/cmd/goimports
 EXTERNAL_TOOLS=\
@@ -139,13 +137,7 @@ update-plugins:
 	grep vault-plugin- go.mod | cut -d ' ' -f 1 | while read -r P; do echo "Updating $P..."; go get -v "$P"; done
 
 static-assets-dir:
-	@mkdir -p ./pkg/web_ui
-
-static-assets: static-assets-dir
-	@echo "--> Generating static assets"
-	@go-bindata-assetfs -o bindata_assetfs.go -pkg http -prefix pkg -modtime 1480000000 -tags ui ./pkg/web_ui/...
-	@mv bindata_assetfs.go http
-	@$(MAKE) -f $(THIS_FILE) fmt
+	@mkdir -p ./http/web_ui
 
 test-ember:
 	@echo "--> Installing JavaScript assets"
@@ -187,8 +179,8 @@ ember-dist-dev:
 	@echo "--> Building Ember application"
 	@cd ui && yarn run build:dev
 
-static-dist: ember-dist static-assets
-static-dist-dev: ember-dist-dev static-assets
+static-dist: ember-dist 
+static-dist-dev: ember-dist-dev 
 
 proto:
 	protoc vault/*.proto --go_out=plugins=grpc,paths=source_relative:.
@@ -257,8 +249,8 @@ ci-config:
 ci-verify:
 	@$(MAKE) -C .circleci ci-verify
 
-.PHONY: bin default prep test vet bootstrap ci-bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin static-assets ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path check-browserstack-creds test-ui-browserstack packages build build-ci
+.PHONY: bin default prep test vet bootstrap ci-bootstrap fmt fmtcheck mysql-database-plugin mysql-legacy-database-plugin cassandra-database-plugin influxdb-database-plugin postgresql-database-plugin mssql-database-plugin hana-database-plugin mongodb-database-plugin ember-dist ember-dist-dev static-dist static-dist-dev assetcheck check-vault-in-path check-browserstack-creds test-ui-browserstack packages build build-ci
 
-.NOTPARALLEL: ember-dist ember-dist-dev static-assets
+.NOTPARALLEL: ember-dist ember-dist-dev
 
 -include packagespec.mk

changelog/11208.txt

@@ -0,0 +1,3 @@
+```release-note:change
+ui: replaces Vault's use of elazarl/go-bindata-assetfs in building the UI with Go's native Embed package
+```

go.mod

@@ -42,7 +42,6 @@ require (
 	github.com/dsnet/compress v0.0.1 // indirect
 	github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74
 	github.com/dustin/go-humanize v1.0.0
-	github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f
 	github.com/fatih/color v1.11.0
 	github.com/fatih/structs v1.1.0
 	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa
@@ -60,7 +59,6 @@ require (
 	github.com/hashicorp/consul-template v0.26.0
 	github.com/hashicorp/consul/api v1.4.0
 	github.com/hashicorp/errwrap v1.1.0
-	github.com/hashicorp/go-bindata v3.0.8-0.20180209072458-bf7910af8997+incompatible
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-discover v0.0.0-20201029210230-738cb3105cd0
 	github.com/hashicorp/go-gcp-common v0.7.0
@@ -102,7 +100,7 @@ require (
 	github.com/hashicorp/vault-plugin-auth-kerberos v0.4.0
 	github.com/hashicorp/vault-plugin-auth-kubernetes v0.10.1
 	github.com/hashicorp/vault-plugin-auth-oci v0.8.0
-	github.com/hashicorp/vault-plugin-database-couchbase v0.4.1
+	github.com/hashicorp/vault-plugin-database-couchbase v0.4.0
 	github.com/hashicorp/vault-plugin-database-elasticsearch v0.8.0
 	github.com/hashicorp/vault-plugin-database-mongodbatlas v0.4.0
 	github.com/hashicorp/vault-plugin-database-snowflake v0.2.0

go.sum

@@ -326,8 +326,6 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m
 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
 github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M=
-github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f h1:AwZUiMWfYSmIiHdFJIubTSs8BFIFoMmUFbeuwBzHIPs=
-github.com/elazarl/go-bindata-assetfs v1.0.1-0.20200509193318-234c15e7648f/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -570,8 +568,6 @@ github.com/hashicorp/consul/sdk v0.4.1-0.20200910203702-bb2b5dd871ca/go.mod h1:f
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-bindata v3.0.8-0.20180209072458-bf7910af8997+incompatible h1:EDTAuh27kAIhxuyK8ef3iHQARA+8NQaXyTeDEiG3Q6o=
-github.com/hashicorp/go-bindata v3.0.8-0.20180209072458-bf7910af8997+incompatible/go.mod h1:+IrDq36jUYG0q6TsDY9uO2p77C8f8S5y+RbYHr2UI+U=
 github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@@ -717,8 +713,8 @@ github.com/hashicorp/vault-plugin-auth-kubernetes v0.10.1 h1:7c2ufXt5oXSUISNHpO0
 github.com/hashicorp/vault-plugin-auth-kubernetes v0.10.1/go.mod h1:2c/k3nsoGPKV+zpAWCiajt4e66vncEq8Li/eKLqErAc=
 github.com/hashicorp/vault-plugin-auth-oci v0.8.0 h1:qYtVYsQlVnqqlCVqZ+CAiFEXuYJqUQCuqcWQVELybZY=
 github.com/hashicorp/vault-plugin-auth-oci v0.8.0/go.mod h1:Cn5cjR279Y+snw8LTaiLTko3KGrbigRbsQPOd2D5xDw=
-github.com/hashicorp/vault-plugin-database-couchbase v0.4.1 h1:DSFwDOcmgZ+CSgTh4F5AK7p311QHoT1Jebj/z9PNi6g=
-github.com/hashicorp/vault-plugin-database-couchbase v0.4.1/go.mod h1:Seivjno/BOtkqX41d/DDYtTg6zNoxIgNaUVZ3ObZYi4=
+github.com/hashicorp/vault-plugin-database-couchbase v0.4.0 h1:N5ChjecnC88mRfT9KehoeSK6xEyzGoL6g9HJnSZKgII=
+github.com/hashicorp/vault-plugin-database-couchbase v0.4.0/go.mod h1:Seivjno/BOtkqX41d/DDYtTg6zNoxIgNaUVZ3ObZYi4=
 github.com/hashicorp/vault-plugin-database-elasticsearch v0.8.0 h1:c9/fwjJf9XjXSM8WzCKL2fco4jyAudUSM9QIY4hY+5M=
 github.com/hashicorp/vault-plugin-database-elasticsearch v0.8.0/go.mod h1:QiQnpM6tI8LqIO+XfI/5AddV7d9cT1DhhOekLV2+AKY=
 github.com/hashicorp/vault-plugin-database-mongodbatlas v0.4.0 h1:baCsn+MRffmcqkOf3p6Fh0fvw2llXl63Ts4Fl14Vn3A=

http/assets.go

@@ -0,0 +1,24 @@
+// +build ui
+
+package http
+
+import (
+	"embed"
+	"io/fs"
+	"net/http"
+)
+
+// content is our static web server content.
+//go:embed web_ui/*
+var content embed.FS
+
+// assetFS is a http Filesystem that serves the generated web UI from the
+// "ember-dist" make step
+func assetFS() http.FileSystem {
+	// sub out to web_ui, where the generated content lives
+	f, err := fs.Sub(content, "web_ui")
+	if err != nil {
+		panic(err)
+	}
+	return http.FS(f)
+}

http/stub_assets.go → http/assets_stub.go

@@ -3,14 +3,16 @@
 package http
 
 import (
-	assetfs "github.com/elazarl/go-bindata-assetfs"
+	"net/http"
 )
 
 func init() {
+	// set uiBuiltIn to false to indicate the ui is not built in. See
+	// http/handler.go
 	uiBuiltIn = false
 }
 
 // assetFS is a stub for building Vault without a UI.
-func assetFS() *assetfs.AssetFS {
+func assetFS() http.FileSystem {
 	return nil
 }

http/handler.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"mime"
 	"net"
@@ -19,7 +20,6 @@ import (
 	"time"
 
 	"github.com/NYTimes/gziphandler"
-	assetfs "github.com/elazarl/go-bindata-assetfs"
 	"github.com/hashicorp/errwrap"
 	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-secure-stdlib/parseutil"
@@ -581,17 +581,18 @@ func handleUIRedirect() http.Handler {
 }
 
 type UIAssetWrapper struct {
-	FileSystem *assetfs.AssetFS
+	FileSystem http.FileSystem
 }
 
-func (fs *UIAssetWrapper) Open(name string) (http.File, error) {
-	file, err := fs.FileSystem.Open(name)
+func (fsw *UIAssetWrapper) Open(name string) (http.File, error) {
+	file, err := fsw.FileSystem.Open(name)
 	if err == nil {
 		return file, nil
 	}
 	// serve index.html instead of 404ing
-	if err == os.ErrNotExist {
-		return fs.FileSystem.Open("index.html")
+	if errors.Is(err, fs.ErrNotExist) {
+		file, err := fsw.FileSystem.Open("index.html")
+		return file, err
 	}
 	return nil, err
 }