[release-1.12] Make injector resilient to sentry unavailability (#7507)…

… (#7560)

Signed-off-by: yaron2 <schneider.yaron@live.com>
Signed-off-by: ItalyPaleAle <43508+ItalyPaleAle@users.noreply.github.com>
Co-authored-by: Yaron Schneider <schneider.yaron@live.com>

cmd/injector/main.go

@@ -131,6 +131,11 @@ func main() {
 				SentryID:      sentryID,
 				Security:      sec,
 			})
+			derr := requester.DialSentryConnection(ctx)
+			if derr != nil {
+				return derr
+			}
+
 			return inj.Run(ctx,
 				sec.TLSServerConfigNoClientAuth(),
 				requester.RequestCertificateFromSentry,

pkg/injector/sentry/sentry.go

@@ -23,7 +23,9 @@ import (
 	"encoding/pem"
 	"fmt"
 	"os"
+	"time"
 
+	grpcRetry "github.com/grpc-ecosystem/go-grpc-middleware/retry"
 	"github.com/spiffe/go-spiffe/v2/spiffeid"
 	"google.golang.org/grpc"
 
@@ -47,6 +49,7 @@ type Requester struct {
 	sentryID       spiffeid.ID
 	sec            security.Handler
 	kubernetesMode bool
+	sentryConn     *grpc.ClientConn
 }
 
 // New returns a new instance of the Requester.
@@ -60,6 +63,20 @@ func New(opts Options) *Requester {
 	}
 }
 
+// DialSentryConnection creates the gRPC connection to the Sentry service and blocks for 1 minute.
+func (r *Requester) DialSentryConnection(ctx context.Context) error {
+	connCtx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+
+	conn, err := grpc.DialContext(connCtx, r.sentryAddress, r.sec.GRPCDialOptionMTLS(r.sentryID), grpc.WithBlock())
+	if err != nil {
+		return fmt.Errorf("error establishing connection to sentry: %w", err)
+	}
+	r.sentryConn = conn
+
+	return nil
+}
+
 // RequestCertificateFromSentry requests a certificate from sentry for a
 // generic daprd identity in a namespace.
 // Returns the signed certificate chain and leaf private key as a PEM encoded
@@ -78,18 +95,12 @@ func (r *Requester) RequestCertificateFromSentry(ctx context.Context, namespace
 		return nil, nil, fmt.Errorf("failed to create sidecar csr: %w", err)
 	}
 
-	conn, err := grpc.DialContext(ctx, r.sentryAddress, r.sec.GRPCDialOptionMTLS(r.sentryID))
-	if err != nil {
-		return nil, nil, fmt.Errorf("error establishing connection to sentry: %w", err)
-	}
-	defer conn.Close()
-
 	token, tokenValidator, err := securitytoken.GetSentryToken(r.kubernetesMode)
 	if err != nil {
 		return nil, nil, fmt.Errorf("error obtaining token: %w", err)
 	}
 
-	resp, err := sentryv1pb.NewCAClient(conn).SignCertificate(ctx,
+	resp, err := sentryv1pb.NewCAClient(r.sentryConn).SignCertificate(ctx,
 		&sentryv1pb.SignCertificateRequest{
 			CertificateSigningRequest: pem.EncodeToMemory(&pem.Block{
 				Type: "CERTIFICATE REQUEST", Bytes: csrDER,
@@ -98,7 +109,7 @@ func (r *Requester) RequestCertificateFromSentry(ctx context.Context, namespace
 			Token:          token,
 			Namespace:      namespace,
 			TokenValidator: tokenValidator,
-		})
+		}, grpcRetry.WithMax(3), grpcRetry.WithPerRetryTimeout(time.Second*3))
 	if err != nil {
 		return nil, nil, fmt.Errorf("error from sentry SignCertificate: %w", err)
 	}