diff --git a/go.mod b/go.mod index cf6828c6e..bebc7b5a5 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/cerbos/cloud-api v0.1.7 github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash/v2 v2.2.0 + github.com/cloudflare/certinel v0.4.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/doug-martin/goqu/v9 v9.18.0 github.com/fatih/color v1.15.0 @@ -171,6 +172,7 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gdamore/encoding v1.0.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect diff --git a/go.sum b/go.sum index 9c1c38d5f..1fec9c8c2 100644 --- a/go.sum +++ b/go.sum @@ -222,6 +222,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/certinel v0.4.0 h1:gQMQlMcg38XAbHdBweB84jYX722eqDq3f+qG+B2UVks= +github.com/cloudflare/certinel v0.4.0/go.mod h1:Cyc0MG7cor0NIhmP1Xj8CpSkEub2YeCVN5A+/qPTYFc= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -308,6 +310,9 @@ github.com/fergusstrange/embedded-postgres v1.24.0/go.mod h1:wL562t1V+iuFwq0UcgM github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell/v2 v2.6.0 h1:OKbluoP9VYmJwZwq/iLb4BxwKcwGthaa1YNBJIyCySg= @@ -573,6 +578,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -848,6 +854,7 @@ github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1159,6 +1166,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1206,6 +1214,7 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1258,6 +1267,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1456,6 +1466,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo= gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A= helm.sh/helm/v3 v3.13.1 h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0= diff --git a/internal/server/server.go b/internal/server/server.go index 656a41693..c203fb4b1 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -18,6 +18,11 @@ import ( "time" "contrib.go.opencensus.io/exporter/prometheus" + svcv1 "github.com/cerbos/cerbos/api/genpb/cerbos/svc/v1" + "github.com/cerbos/cerbos/internal/audit" + "github.com/cerbos/cerbos/internal/telemetry" + "github.com/cerbos/cerbos/internal/validator" + "github.com/cloudflare/certinel/fswatcher" "github.com/gorilla/mux" grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/protovalidate" @@ -25,6 +30,7 @@ import ( "github.com/grpc-ecosystem/grpc-gateway/v2/runtime" reuseport "github.com/kavu/go_reuseport" prom "github.com/prometheus/client_golang/prometheus" + "github.com/sourcegraph/conc/pool" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/plugin/ochttp" "go.opencensus.io/stats/view" @@ -33,17 +39,11 @@ import ( "go.uber.org/zap" "golang.org/x/net/http2" "golang.org/x/net/http2/h2c" - "golang.org/x/sync/errgroup" "google.golang.org/grpc" "google.golang.org/grpc/admin" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/local" - svcv1 "github.com/cerbos/cerbos/api/genpb/cerbos/svc/v1" - "github.com/cerbos/cerbos/internal/audit" - "github.com/cerbos/cerbos/internal/telemetry" - "github.com/cerbos/cerbos/internal/validator" - // Import the default grpc encoding to ensure that it gets replaced by VT. _ "google.golang.org/grpc/encoding/proto" "google.golang.org/grpc/health" @@ -203,28 +203,30 @@ type Param struct { type Server struct { conf *Conf cancelFunc context.CancelFunc - group *errgroup.Group + pool *pool.ContextPool health *health.Server ocExporter *prometheus.Exporter + tlsConfig *tls.Config } func NewServer(conf *Conf) *Server { - ctx, cancelFunc := context.WithCancel(context.Background()) - - group, _ := errgroup.WithContext(ctx) - return &Server{ - conf: conf, - cancelFunc: cancelFunc, - group: group, - health: health.NewServer(), + conf: conf, + health: health.NewServer(), } } func (s *Server) Start(ctx context.Context, param Param) error { + ctx, cancelFunc := context.WithCancel(ctx) + s.pool = pool.New().WithContext(ctx).WithCancelOnError().WithFirstError() + s.cancelFunc = cancelFunc + defer s.cancelFunc() log := zap.L().Named("server") + if err := s.initializeTLSConfig(log); err != nil { + log.Error("Failed to initialize TLS configuration", zap.Error(err)) + } // It would be nice to have a single port to serve both gRPC and HTTP. Unfortunately, cmux // can't deal effectively with both gRPC and HTTP/2 when TLS is enabled (see https://github.com/soheilhy/cmux/issues/68). @@ -258,7 +260,7 @@ func (s *Server) Start(ctx context.Context, param Param) error { return err } - s.group.Go(func() error { + s.pool.Go(func(ctx context.Context) error { <-ctx.Done() log.Info("Shutting down") @@ -279,8 +281,7 @@ func (s *Server) Start(ctx context.Context, param Param) error { return nil }) - err = s.group.Wait() - if err != nil { + if err := s.pool.Wait(); err != nil { log.Error("Stopping server due to error", zap.Error(err)) return err } @@ -301,62 +302,68 @@ func (s *Server) Start(ctx context.Context, param Param) error { return nil } -func (s *Server) createListener(listenAddr string) (net.Listener, error) { - l, err := s.parseAndOpen(listenAddr) - if err != nil { - return nil, fmt.Errorf("failed to create listener at '%s': %w", listenAddr, err) - } - - tlsConf, err := s.getTLSConfig() - if err != nil { - return nil, err - } - - if tlsConf != nil { - l = tls.NewListener(l, tlsConf) - } - - return l, nil -} - -func (s *Server) getTLSConfig() (*tls.Config, error) { +func (s *Server) initializeTLSConfig(log *zap.Logger) error { if s.conf.TLS == nil || (s.conf.TLS.Cert == "" || s.conf.TLS.Key == "") { - return nil, nil + return nil } - // TODO (cell) Configure TLS with reloadable certificates - conf := s.conf.TLS - certificate, err := tls.LoadX509KeyPair(conf.Cert, conf.Key) + certinel, err := fswatcher.New(conf.Cert, conf.Key) if err != nil { - return nil, fmt.Errorf("failed to load certificate and key: %w", err) + return fmt.Errorf("failed to load certificate and key: %w", err) } - tlsConfig := util.DefaultTLSConfig() - tlsConfig.Certificates = []tls.Certificate{certificate} + s.pool.Go(func(ctx context.Context) (outErr error) { + log.Info("Starting certificate watcher") + if err := certinel.Start(ctx); err != nil { + if !errors.Is(err, context.Canceled) { + log.Error("Stopping certificate watcher due to error", zap.Error(err)) + return err + } + } + + log.Info("Stopping certificate watcher") + return nil + }) + + s.tlsConfig = util.DefaultTLSConfig() + s.tlsConfig.GetCertificate = certinel.GetCertificate if conf.CACert != "" { if _, err := os.Stat(conf.CACert); err != nil { //nolint:nilerr - return tlsConfig, nil + return nil } certPool := x509.NewCertPool() bs, err := os.ReadFile(conf.CACert) if err != nil { - return nil, fmt.Errorf("failed to load CA certificate: %w", err) + return fmt.Errorf("failed to load CA certificate: %w", err) } ok := certPool.AppendCertsFromPEM(bs) if !ok { - return nil, errors.New("failed to append certificates to the pool") + return errors.New("failed to append certificates to the pool") } - tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven - tlsConfig.ClientCAs = certPool + s.tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven + s.tlsConfig.ClientCAs = certPool + } + + return nil +} + +func (s *Server) createListener(listenAddr string) (net.Listener, error) { + l, err := s.parseAndOpen(listenAddr) + if err != nil { + return nil, fmt.Errorf("failed to create listener at '%s': %w", listenAddr, err) } - return tlsConfig, nil + if s.tlsConfig != nil { + l = tls.NewListener(l, s.tlsConfig) + } + + return l, nil } func (s *Server) startGRPCServer(l net.Listener, param Param) (*grpc.Server, error) { @@ -400,7 +407,7 @@ func (s *Server) startGRPCServer(l net.Listener, param Param) (*grpc.Server, err s.health.SetServingStatus(svcv1.CerbosPlaygroundService_ServiceDesc.ServiceName, healthpb.HealthCheckResponse_SERVING) } - s.group.Go(func() error { + s.pool.Go(func(_ context.Context) error { log.Info(fmt.Sprintf("Starting gRPC server at %s", s.conf.GRPCListenAddr)) cleanup, err := admin.Register(server) @@ -545,7 +552,7 @@ func (s *Server) startHTTPServer(ctx context.Context, l net.Listener, grpcSrv *g IdleTimeout: s.conf.Advanced.HTTP.IdleTimeout, } - s.group.Go(func() error { + s.pool.Go(func(ctx context.Context) error { log.Infof("Starting HTTP server at %s", s.conf.HTTPListenAddr) err := h.Serve(l) if err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -572,12 +579,8 @@ func defaultGRPCDialOpts() []grpc.DialOption { func (s *Server) mkGRPCConn(ctx context.Context) (*grpc.ClientConn, error) { opts := defaultGRPCDialOpts() - tlsConf, err := s.getTLSConfig() - if err != nil { - return nil, fmt.Errorf("failed to create TLS config: %w", err) - } - - if tlsConf != nil { + if s.tlsConfig != nil { + tlsConf := s.tlsConfig.Clone() tlsConf.InsecureSkipVerify = true // we are connecting as localhost which would differ from what the cert is issued for. opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf))) } else {