Merge pull request #506 from baidu/release/v0.10.0

Release/v0.10.0

CHANGELOG.md

@@ -10,6 +10,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v0.10.0] - 2020-05-25  
+### Added
+- mod_auth_request: authorize clients based on thirdparty authorization service
+- mod_trace: support tracing based on Elastic APM
+- mod_compress: support brotli algorithm
+- mod_rewrite: add HostSuffixReplace action
+- Add condition primitive: req_host_tag_in/req_host_suffix_in
+- Add static content of website based on mkdocs
+- Documents optimization
+
+### Changed
+- Refactor and simplify mod_auth_jwt
+
+### Removed
+- Remove legacy type of Layer4LoadBalancer 
+
+
 ## [v0.9.0] - 2020-04-16
 ### Added
 - Support loading dynamic modules that may be written and complied by thirdparty vendors
@@ -146,6 +163,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Flexible plugin framework to extend functionality. Based on the framework, developer can add new features rapidly
 - Detailed built-in metrics available for service status monitor
 
+[v0.10.0]: https://github.com/baidu/bfe/compare/v0.9.0...v0.10.0
 [v0.9.0]: https://github.com/baidu/bfe/compare/v0.8.0...v0.9.0
 [v0.8.0]: https://github.com/baidu/bfe/compare/v0.7.0...v0.8.0
 [v0.7.0]: https://github.com/baidu/bfe/compare/v0.6.0...v0.7.0

CONTRIBUTORS.md

@@ -6,7 +6,9 @@
 | Caiyuan Yang | jansci621 |
 | Chongmiao Liu | lcmmhcc |
 | Derek Zheng | shanhuhai5739 |
+| Haobin zhang | zhanghaobin |
 | Jie Liu | freeHackOfJeff |
+| Jie Wan | wanjiecs |
 | Jin Tong | cumirror |
 | Jiyang Zhang | scriptkids |
 | Kaiyu Zheng | kaiyuzheng | 
@@ -26,14 +28,18 @@
 | Sijie Yang | iyangsj |
 | Tianqi Zhang | NKztq |
 | Wenjie Tian | WJTian |
+| Wenlong Chen | LeroChen |
 | Wensi Yang | tianxinheihei | 
 | Xiaofei Yu | xiaofei0800 |
 | Xiaoli Liu | liuxiaoli007 |
+| Xiaonan chen | two |
 | Xiaoye Jiang | kidleaf-jiang |
 | Xin Li | lx-or-xxxl |
 | Yang Liu | dut-yangliu |
 | Yuqi Xiao | YuqiXiao |
+| Zheng Liu | liuzheng |
 | Zhichao Lin | lxiaozhic |
+| Yuchen Wang | CHneger |
 |          | 0xflotus |
 |          | calify |
 |          | MoonShining |

Makefile

@@ -38,6 +38,8 @@ endif
 
 # init bfe version
 BFE_VERSION ?= $(shell cat VERSION)
+# init git commit id
+GIT_COMMIT ?= $(shell git rev-parse HEAD)
 
 # init bfe packages
 BFE_PKGS := $(shell go list ./...)
@@ -55,7 +57,7 @@ prepare-gen:
 # make compile, go build
 compile: test build
 build:
-	$(GOBUILD) -ldflags "-X main.version=$(BFE_VERSION)" 
+	$(GOBUILD) -ldflags "-X main.version=$(BFE_VERSION) -X main.commit=$(GIT_COMMIT)"
 
 # make test, test your code
 test: test-case vet-case

README.md

@@ -26,8 +26,8 @@ BFE is an open-source layer 7 load balancer derived from proprietary Baidu Front
 - See [Build and run](docs/en_us/installation/install_from_source.md)
 
 ## Documentation
-- [English version](docs/en_us/SUMMARY.md)
-- [Chinese version](docs/zh_cn/SUMMARY.md)
+- [English version](https://www.bfe-networks.net/en_us/ABOUT/)
+- [Chinese version](https://www.bfe-networks.net/zh_cn/ABOUT/)
 
 ## Contributing
 - Please create an issue in [issue list](http://github.com/baidu/bfe/issues).
@@ -39,11 +39,9 @@ BFE is an open-source layer 7 load balancer derived from proprietary Baidu Front
 - Owners: [MAINTAINERS](MAINTAINERS.md)
 - Contributors: [CONTRIBUTORS](CONTRIBUTORS.md)
 
-## Discussion
-- Issue: https://github.com/baidu/bfe/issues
-
-## Contact
-- Email：bfe-osc@baidu.com
+## Communication
+- BFE community on Slack: [Sign up](https://join.slack.com/t/bfe-networks/shared_invite/zt-cn04xsqr-j7LDFmPkCuCZ39OLcHlMBA) and join channels on topics that interest you.
+- BFE developer group on WeChat: [Send a request mail](mailto:bfe-osc@baidu.com) with your WeChat ID and a contribution you've made to BFE(such as a PR/Issue). We will invite you right away.
 
 ## License
 BFE is under the Apache 2.0 license. See the [LICENSE](LICENSE) file for details.

VERSION

@@ -1 +1 @@
-0.9.0
+0.10.0

bfe.go

@@ -35,15 +35,17 @@ import (
 )
 
 var (
-	help     *bool   = flag.Bool("h", false, "to show help")
-	confRoot *string = flag.String("c", "./conf", "root path of configuration")
-	logPath  *string = flag.String("l", "./log", "dir path of log")
-	stdOut   *bool   = flag.Bool("s", false, "to show log in stdout")
-	showVer  *bool   = flag.Bool("v", false, "to show version of bfe")
-	debugLog *bool   = flag.Bool("d", false, "to show debug log (otherwise >= info)")
+	help        *bool   = flag.Bool("h", false, "to show help")
+	confRoot    *string = flag.String("c", "./conf", "root path of configuration")
+	logPath     *string = flag.String("l", "./log", "dir path of log")
+	stdOut      *bool   = flag.Bool("s", false, "to show log in stdout")
+	showVersion *bool   = flag.Bool("v", false, "to show version of bfe")
+	showVerbose *bool   = flag.Bool("V", false, "to show verbose information about bfe")
+	debugLog    *bool   = flag.Bool("d", false, "to show debug log (otherwise >= info)")
 )
 
 var version string
+var commit string
 
 func main() {
 	var err error
@@ -55,8 +57,14 @@ func main() {
 		flag.PrintDefaults()
 		return
 	}
-	if *showVer {
-		fmt.Printf("bfe: version %s\n", version)
+	if *showVersion {
+		fmt.Printf("bfe version: %s\n", version)
+		return
+	}
+	if *showVerbose {
+		fmt.Printf("bfe version: %s\n", version)
+		fmt.Printf("go version: %s\n", runtime.Version())
+		fmt.Printf("git commit: %s\n", commit)
 		return
 	}
 

bfe_basic/action/action.go

@@ -39,6 +39,7 @@ const (
 	// host actions
 	ActionHostSetFromPathPrefix = "HOST_SET_FROM_PATH_PREFIX" // set host from path prefix
 	ActionHostSet               = "HOST_SET"                  // set host
+	ActionHostSuffixReplace     = "HOST_SUFFIX_REPLACE"       // set host replaced suffx
 
 	// path actions
 	ActionPathSet        = "PATH_SET"         // set path
@@ -111,6 +112,8 @@ func (ac *Action) Do(req *bfe_basic.Request) error {
 		ReqHostSet(req, ac.Params[0])
 	case ActionHostSetFromPathPrefix:
 		ReqHostSetFromFirstPathSegment(req)
+	case ActionHostSuffixReplace:
+		ReqHostSuffixReplace(req, ac.Params[0], ac.Params[1])
 
 	// for path
 	case ActionPathSet:

bfe_basic/action/action_host.go

@@ -50,3 +50,14 @@ func ReqHostSetFromFirstPathSegment(req *bfe_basic.Request) {
 	req.HttpRequest.Host = segs[1]
 	req.HttpRequest.URL.Path = "/" + segs[2]
 }
+
+// ReqHostSuffixReplace replaces suffix of hostname.
+func ReqHostSuffixReplace(req *bfe_basic.Request, originSuffix, newSuffix string) {
+	hostname := req.HttpRequest.URL.Host
+	if !strings.HasSuffix(hostname, originSuffix) {
+		return
+	}
+
+	hostname = strings.TrimSuffix(hostname, originSuffix) + newSuffix
+	req.HttpRequest.Host = hostname
+}

bfe_basic/common.go

@@ -47,6 +47,19 @@ const (
 	GlobalProduct = "global"
 )
 
+// Hop-by-hop headers. These are removed when sent to the backend.
+// http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
+var HopHeaders = []string{
+	"Connection",
+	"Keep-Alive",
+	"Proxy-Authenticate",
+	"Proxy-Authorization",
+	"Te", // canonicalized version of "TE"
+	"Trailers",
+	"Transfer-Encoding",
+	"Upgrade",
+}
+
 // CreateInternalSrvErrResp returns a HTTP 500 response
 func CreateInternalSrvErrResp(request *Request) *bfe_http.Response {
 	return CreateInternalResp(request, bfe_http.StatusInternalServerError)

bfe_basic/condition/build.go

@@ -154,6 +154,13 @@ func buildPrimitive(node *parser.CallExpr) (Condition, error) {
 			fetcher: &HostFetcher{},
 			matcher: matcher,
 		}, nil
+	case "req_host_tag_in":
+		return &PrimitiveCond{
+			name:    node.Fun.Name,
+			node:    node,
+			fetcher: &HostTagFetcher{},
+			matcher: NewInMatcher(node.Args[0].Value, true),
+		}, nil
 	case "req_host_regmatch":
 		reg, err := regexp.Compile(node.Args[0].Value)
 		if err != nil {
@@ -165,6 +172,13 @@ func buildPrimitive(node *parser.CallExpr) (Condition, error) {
 			fetcher: &HostFetcher{},
 			matcher: NewRegMatcher(reg),
 		}, nil
+	case "req_host_suffix_in":
+		return &PrimitiveCond{
+			name:    node.Fun.Name,
+			node:    node,
+			fetcher: &HostFetcher{},
+			matcher: NewSuffixInMatcher(node.Args[0].Value, true),
+		}, nil
 	case "req_path_in":
 		return &PrimitiveCond{
 			name:    node.Fun.Name,

bfe_basic/condition/build_test.go

@@ -398,3 +398,22 @@ func TestBuildTlsClientCAIn(t *testing.T) {
 		t.Errorf("ca match ses_tls_client_ca_in(\"clientCa\")")
 	}
 }
+
+func TestBuildHostTagIn(t *testing.T) {
+	cond, err := Build("req_host_tag_in(\"host_tag1|host_tag2\")")
+	if err != nil {
+		t.Fatalf("should have no error")
+	}
+	req.Route.HostTag = "host_tag1"
+	if !cond.Match(&req) {
+		t.Fatalf("should match host tag %s", req.Route.HostTag)
+	}
+	req.Route.HostTag = "host_tag2"
+	if !cond.Match(&req) {
+		t.Fatalf("should match host tag %s", req.Route.HostTag)
+	}
+	req.Route.HostTag = "host_tag3"
+	if cond.Match(&req) {
+		t.Fatalf("should not match host tag %s", req.Route.HostTag)
+	}
+}

bfe_basic/condition/parser/semant.go

@@ -30,6 +30,8 @@ var funcProtos = map[string][]Token{
 	"req_proto_secure":           nil,
 	"req_host_in":                {STRING},
 	"req_host_regmatch":          {STRING},
+	"req_host_tag_in":            {STRING},
+	"req_host_suffix_in":         {STRING},
 	"req_path_in":                {STRING, BOOL},
 	"req_path_prefix_in":         {STRING, BOOL},
 	"req_path_suffix_in":         {STRING, BOOL},

bfe_basic/condition/primitive.go

@@ -90,6 +90,15 @@ func (hf *HostFetcher) Fetch(req *bfe_basic.Request) (interface{}, error) {
 	return host, nil
 }
 
+type HostTagFetcher struct{}
+
+func (hf *HostTagFetcher) Fetch(req *bfe_basic.Request) (interface{}, error) {
+	if req == nil {
+		return nil, fmt.Errorf("fetcher: nil pointer")
+	}
+	return req.Route.HostTag, nil
+}
+
 type ProtoFetcher struct{}
 
 func (pf *ProtoFetcher) Fetch(req *bfe_basic.Request) (interface{}, error) {
@@ -527,10 +536,6 @@ func suffixIn(v string, patterns []string) bool {
 	return false
 }
 
-func contains(v, pattern string) bool {
-	return strings.Contains(v, pattern)
-}
-
 type UAFetcher struct{}
 
 func (uaf *UAFetcher) Fetch(req *bfe_basic.Request) (interface{}, error) {

bfe_basic/request.go

@@ -35,8 +35,9 @@ type BackendInfo struct {
 }
 
 type RedirectInfo struct {
-	Url  string // URL
-	Code int    // HTTP status code
+	Url    string          // URL
+	Code   int             // HTTP status code
+	Header bfe_http.Header // Extra header
 }
 
 type RequestRoute struct {

bfe_config/bfe_conf/conf_basic.go

@@ -29,7 +29,6 @@ import (
 )
 
 const (
-	BalancerBgw   = "BGW"   // layer4 balancer in baidu
 	BalancerProxy = "PROXY" // layer4 balancer working in PROXY mode (eg. F5, Ctrix, ELB etc)
 	BalancerNone  = "NONE"  // layer4 balancer not used
 )
@@ -234,14 +233,12 @@ func checkLayer4LoadBalancer(cfg *ConfigBasic) error {
 	}
 
 	switch cfg.Layer4LoadBalancer {
-	case BalancerBgw:
-		return nil
 	case BalancerProxy:
 		return nil
 	case BalancerNone:
 		return nil
 	default:
-		return fmt.Errorf("Layer4LoadBalancer[%s] should be BGW/PROXY/NONE", cfg.Layer4LoadBalancer)
+		return fmt.Errorf("Layer4LoadBalancer[%s] should be PROXY/NONE", cfg.Layer4LoadBalancer)
 	}
 }
 

bfe_config/bfe_conf/conf_session_cache.go

@@ -26,7 +26,7 @@ type ConfigSessionCache struct {
 	// address for redis servers
 	Servers string
 
-	// perfix for cache key
+	// prefix for cache key
 	KeyPrefix string
 
 	// config for connection (ms)

bfe_config/bfe_tls_conf/session_ticket_key_conf/session_ticket_key_conf.go

@@ -28,7 +28,7 @@ import (
 )
 
 const (
-	RAW_SESSION_TICKET_KEY_SIZE = 48 // bytes
+	RawSessionTicketKeySize = 48 // bytes
 )
 
 // SessionTicketKeyConf is session ticket key config.
@@ -47,7 +47,7 @@ func SessionTicketKeyConfCheck(conf SessionTicketKeyConf) error {
 	if err != nil {
 		return fmt.Errorf("session ticket key %s(%s)", err.Error(), conf.SessionTicketKey)
 	}
-	if len(key) != RAW_SESSION_TICKET_KEY_SIZE {
+	if len(key) != RawSessionTicketKeySize {
 		return fmt.Errorf("session ticket key should be 96 bytes hex string (%s)", conf.SessionTicketKey)
 	}
 
@@ -63,7 +63,7 @@ func rawSessionTicketKeyLoad(filename string) (SessionTicketKeyConf, error) {
 		return config, err
 	}
 
-	if len(data) != RAW_SESSION_TICKET_KEY_SIZE {
+	if len(data) != RawSessionTicketKeySize {
 		return config, fmt.Errorf("invalid session ticket key(%d)", len(data))
 	}
 

bfe_module/bfe_callback.go

@@ -74,23 +74,23 @@ func NewBfeCallbacks() *BfeCallbacks {
 	bfeCallbacks.callbacks = make(map[int]*HandlerList)
 
 	// create handler list for each callback point
-	// for HANDLERS_ACCEPT
+	// for HandlesAccept
 	bfeCallbacks.callbacks[HandleAccept] = NewHandlerList(HandleAccept)
 	bfeCallbacks.callbacks[HandleHandshake] = NewHandlerList(HandleAccept)
 
-	// for HANDLERS_REQUEST
+	// for HandlersRequest
 	bfeCallbacks.callbacks[HandleBeforeLocation] = NewHandlerList(HandlersRequest)
 	bfeCallbacks.callbacks[HandleFoundProduct] = NewHandlerList(HandlersRequest)
 	bfeCallbacks.callbacks[HandleAfterLocation] = NewHandlerList(HandlersRequest)
 
-	// for HANDLERS_FORWARD
+	// for HandlersForward
 	bfeCallbacks.callbacks[HandleForward] = NewHandlerList(HandlersForward)
 
-	// for HANDLERS_RESPONSE
+	// for HandlersResponse
 	bfeCallbacks.callbacks[HandleReadResponse] = NewHandlerList(HandlersResponse)
 	bfeCallbacks.callbacks[HandleRequestFinish] = NewHandlerList(HandlersResponse)
 
-	// for HANDLERS_FINISH
+	// for HandlersFinish
 	bfeCallbacks.callbacks[HandleFinish] = NewHandlerList(HandlersFinish)
 
 	return bfeCallbacks