Skip to content

Commit

Permalink
feat: Reload certificates when they change on disk (#1841)
Browse files Browse the repository at this point in the history 
Automatically reload TLS certificates when they change on disk without
restarting the Cerbos server.

Signed-off-by: Charith Ellawala <charith@cerbos.dev>

Signed-off-by: Charith Ellawala <charith@cerbos.dev>
  • Loading branch information
@charithe
charithe committed Oct 24, 2023
1 parent 1295185 commit 111c4a3
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 58 deletions.
2 changes: 2 additions & 0 deletions go.mod
View file
Expand Up @@ -20,6 +20,7 @@ require (
github.com/cerbos/cloud-api v0.1.7
github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.2.0
github.com/cloudflare/certinel v0.4.0
github.com/dgraph-io/badger/v4 v4.2.0
github.com/doug-martin/goqu/v9 v9.18.0
github.com/fatih/color v1.15.0
Expand Down Expand Up @@ -171,6 +172,7 @@ require (
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
View file
Expand Up @@ -222,6 +222,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudflare/certinel v0.4.0 h1:gQMQlMcg38XAbHdBweB84jYX722eqDq3f+qG+B2UVks=
github.com/cloudflare/certinel v0.4.0/go.mod h1:Cyc0MG7cor0NIhmP1Xj8CpSkEub2YeCVN5A+/qPTYFc=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
Expand Down Expand Up @@ -308,6 +310,9 @@ github.com/fergusstrange/embedded-postgres v1.24.0/go.mod h1:wL562t1V+iuFwq0UcgM
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell/v2 v2.6.0 h1:OKbluoP9VYmJwZwq/iLb4BxwKcwGthaa1YNBJIyCySg=
Expand Down Expand Up @@ -573,6 +578,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
Expand Down Expand Up @@ -848,6 +854,7 @@ github.com/spf13/afero v1.10.0/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA=
github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs=
github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down Expand Up @@ -1159,6 +1166,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand Down Expand Up @@ -1206,6 +1214,7 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -1258,6 +1267,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
Expand Down Expand Up @@ -1456,6 +1466,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
gotest.tools/v3 v3.3.0 h1:MfDY1b1/0xN1CyMlQDac0ziEy9zJQd9CXBRRDHw2jJo=
gotest.tools/v3 v3.3.0/go.mod h1:Mcr9QNxkg0uMvy/YElmo4SpXgJKWgQvYrT7Kw5RzJ1A=
helm.sh/helm/v3 v3.13.1 h1:DG+XLGzBJeZvMLlMbm6bPDLV1dGaVW9eZsDoUd1/LM0=
Expand Down
119 changes: 61 additions & 58 deletions internal/server/server.go
View file
Expand Up @@ -18,13 +18,19 @@ import (
"time"

"contrib.go.opencensus.io/exporter/prometheus"
svcv1 "github.com/cerbos/cerbos/api/genpb/cerbos/svc/v1"
"github.com/cerbos/cerbos/internal/audit"
"github.com/cerbos/cerbos/internal/telemetry"
"github.com/cerbos/cerbos/internal/validator"
"github.com/cloudflare/certinel/fswatcher"
"github.com/gorilla/mux"
grpc_logging "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging"
grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/protovalidate"
grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery"
"github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
reuseport "github.com/kavu/go_reuseport"
prom "github.com/prometheus/client_golang/prometheus"
"github.com/sourcegraph/conc/pool"
"go.opencensus.io/plugin/ocgrpc"
"go.opencensus.io/plugin/ochttp"
"go.opencensus.io/stats/view"
Expand All @@ -33,17 +39,11 @@ import (
"go.uber.org/zap"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc"
"google.golang.org/grpc/admin"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/local"

svcv1 "github.com/cerbos/cerbos/api/genpb/cerbos/svc/v1"
"github.com/cerbos/cerbos/internal/audit"
"github.com/cerbos/cerbos/internal/telemetry"
"github.com/cerbos/cerbos/internal/validator"

// Import the default grpc encoding to ensure that it gets replaced by VT.
_ "google.golang.org/grpc/encoding/proto"
"google.golang.org/grpc/health"
Expand Down Expand Up @@ -203,28 +203,30 @@ type Param struct {
type Server struct {
conf *Conf
cancelFunc context.CancelFunc
group *errgroup.Group
pool *pool.ContextPool
health *health.Server
ocExporter *prometheus.Exporter
tlsConfig *tls.Config
}

func NewServer(conf *Conf) *Server {
ctx, cancelFunc := context.WithCancel(context.Background())

group, _ := errgroup.WithContext(ctx)

return &Server{
conf: conf,
cancelFunc: cancelFunc,
group: group,
health: health.NewServer(),
conf: conf,
health: health.NewServer(),
}
}

func (s *Server) Start(ctx context.Context, param Param) error {
ctx, cancelFunc := context.WithCancel(ctx)
s.pool = pool.New().WithContext(ctx).WithCancelOnError().WithFirstError()
s.cancelFunc = cancelFunc

defer s.cancelFunc()

log := zap.L().Named("server")
if err := s.initializeTLSConfig(log); err != nil {
log.Error("Failed to initialize TLS configuration", zap.Error(err))
}

// It would be nice to have a single port to serve both gRPC and HTTP. Unfortunately, cmux
// can't deal effectively with both gRPC and HTTP/2 when TLS is enabled (see https://github.com/soheilhy/cmux/issues/68).
Expand Down Expand Up @@ -258,7 +260,7 @@ func (s *Server) Start(ctx context.Context, param Param) error {
return err
}

s.group.Go(func() error {
s.pool.Go(func(ctx context.Context) error {
<-ctx.Done()
log.Info("Shutting down")

Expand All @@ -279,8 +281,7 @@ func (s *Server) Start(ctx context.Context, param Param) error {
return nil
})

err = s.group.Wait()
if err != nil {
if err := s.pool.Wait(); err != nil {
log.Error("Stopping server due to error", zap.Error(err))
return err
}
Expand All @@ -301,62 +302,68 @@ func (s *Server) Start(ctx context.Context, param Param) error {
return nil
}

func (s *Server) createListener(listenAddr string) (net.Listener, error) {
l, err := s.parseAndOpen(listenAddr)
if err != nil {
return nil, fmt.Errorf("failed to create listener at '%s': %w", listenAddr, err)
}

tlsConf, err := s.getTLSConfig()
if err != nil {
return nil, err
}

if tlsConf != nil {
l = tls.NewListener(l, tlsConf)
}

return l, nil
}

func (s *Server) getTLSConfig() (*tls.Config, error) {
func (s *Server) initializeTLSConfig(log *zap.Logger) error {
if s.conf.TLS == nil || (s.conf.TLS.Cert == "" || s.conf.TLS.Key == "") {
return nil, nil
return nil
}
// TODO (cell) Configure TLS with reloadable certificates

conf := s.conf.TLS

certificate, err := tls.LoadX509KeyPair(conf.Cert, conf.Key)
certinel, err := fswatcher.New(conf.Cert, conf.Key)
if err != nil {
return nil, fmt.Errorf("failed to load certificate and key: %w", err)
return fmt.Errorf("failed to load certificate and key: %w", err)
}

tlsConfig := util.DefaultTLSConfig()
tlsConfig.Certificates = []tls.Certificate{certificate}
s.pool.Go(func(ctx context.Context) (outErr error) {
log.Info("Starting certificate watcher")
if err := certinel.Start(ctx); err != nil {
if !errors.Is(err, context.Canceled) {
log.Error("Stopping certificate watcher due to error", zap.Error(err))
return err
}
}

log.Info("Stopping certificate watcher")
return nil
})

s.tlsConfig = util.DefaultTLSConfig()
s.tlsConfig.GetCertificate = certinel.GetCertificate

if conf.CACert != "" {
if _, err := os.Stat(conf.CACert); err != nil {
//nolint:nilerr
return tlsConfig, nil
return nil
}

certPool := x509.NewCertPool()
bs, err := os.ReadFile(conf.CACert)
if err != nil {
return nil, fmt.Errorf("failed to load CA certificate: %w", err)
return fmt.Errorf("failed to load CA certificate: %w", err)
}

ok := certPool.AppendCertsFromPEM(bs)
if !ok {
return nil, errors.New("failed to append certificates to the pool")
return errors.New("failed to append certificates to the pool")
}

tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven
tlsConfig.ClientCAs = certPool
s.tlsConfig.ClientAuth = tls.VerifyClientCertIfGiven
s.tlsConfig.ClientCAs = certPool
}

return nil
}

func (s *Server) createListener(listenAddr string) (net.Listener, error) {
l, err := s.parseAndOpen(listenAddr)
if err != nil {
return nil, fmt.Errorf("failed to create listener at '%s': %w", listenAddr, err)
}

return tlsConfig, nil
if s.tlsConfig != nil {
l = tls.NewListener(l, s.tlsConfig)
}

return l, nil
}

func (s *Server) startGRPCServer(l net.Listener, param Param) (*grpc.Server, error) {
Expand Down Expand Up @@ -400,7 +407,7 @@ func (s *Server) startGRPCServer(l net.Listener, param Param) (*grpc.Server, err
s.health.SetServingStatus(svcv1.CerbosPlaygroundService_ServiceDesc.ServiceName, healthpb.HealthCheckResponse_SERVING)
}

s.group.Go(func() error {
s.pool.Go(func(_ context.Context) error {
log.Info(fmt.Sprintf("Starting gRPC server at %s", s.conf.GRPCListenAddr))

cleanup, err := admin.Register(server)
Expand Down Expand Up @@ -545,7 +552,7 @@ func (s *Server) startHTTPServer(ctx context.Context, l net.Listener, grpcSrv *g
IdleTimeout: s.conf.Advanced.HTTP.IdleTimeout,
}

s.group.Go(func() error {
s.pool.Go(func(ctx context.Context) error {
log.Infof("Starting HTTP server at %s", s.conf.HTTPListenAddr)
err := h.Serve(l)
if err != nil && !errors.Is(err, http.ErrServerClosed) {
Expand All @@ -572,12 +579,8 @@ func defaultGRPCDialOpts() []grpc.DialOption {
func (s *Server) mkGRPCConn(ctx context.Context) (*grpc.ClientConn, error) {
opts := defaultGRPCDialOpts()

tlsConf, err := s.getTLSConfig()
if err != nil {
return nil, fmt.Errorf("failed to create TLS config: %w", err)
}

if tlsConf != nil {
if s.tlsConfig != nil {
tlsConf := s.tlsConfig.Clone()
tlsConf.InsecureSkipVerify = true // we are connecting as localhost which would differ from what the cert is issued for.
opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(tlsConf)))
} else {
Expand Down

0 comments on commit 111c4a3

Please sign in to comment.