From 8ddcddb53ecdb1961bc078b6b1a2fb608f86764d Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Tue, 7 Jul 2020 12:31:20 -0700 Subject: [PATCH 01/10] Set auth header to localhost on unix target. --- clientconn.go | 3 + internal/grpcutil/target.go | 3 + test/end2end_test.go | 116 ++++++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+) diff --git a/clientconn.go b/clientconn.go index 3ed6034f1ad..9ded5816b7f 100644 --- a/clientconn.go +++ b/clientconn.go @@ -245,6 +245,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * // Determine the resolver to use. cc.parsedTarget = grpcutil.ParseTarget(cc.target) + unixScheme := strings.HasPrefix(cc.parsedTarget.Endpoint, "unix:") channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme) resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme) if resolverBuilder == nil { @@ -267,6 +268,8 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * cc.authority = creds.Info().ServerName } else if cc.dopts.insecure && cc.dopts.authority != "" { cc.authority = cc.dopts.authority + } else if unixScheme { + cc.authority = "localhost" } else { // Use endpoint from "scheme://authority/endpoint" as the default // authority for ClientConn. diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go index 80b33cdaf90..2e87f395b0a 100644 --- a/internal/grpcutil/target.go +++ b/internal/grpcutil/target.go @@ -43,6 +43,9 @@ func split2(s, sep string) (string, string, bool) { // target}. func ParseTarget(target string) (ret resolver.Target) { var ok bool + if strings.HasPrefix(target, "unix:") { + return resolver.Target{Endpoint: target} + } ret.Scheme, ret.Endpoint, ok = split2(target, "://") if !ok { return resolver.Target{Endpoint: target} diff --git a/test/end2end_test.go b/test/end2end_test.go index 3e129c3fc15..2722ef8d7b3 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -7179,3 +7179,119 @@ func (s) TestCanceledRPCCallOptionRace(t *testing.T) { } wg.Wait() } + +// unixServer is used to test servers listening over a unix socket. +type unixServer struct { + // Guarantees we satisfy this interface; panics if unimplemented methods are called. + testpb.TestServiceServer + + // Customizable implementations of server handlers. + emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) + + // A client connected to this service the test may use. Created in Start(). + client testpb.TestServiceClient + cc *grpc.ClientConn + s *grpc.Server + + cleanups []func() // Lambdas executed in Stop(); populated by Start(). +} + +func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return us.emptyCall(ctx, in) +} + +func (us *unixServer) Start(network, address, target string) error { + lis, err := net.Listen(network, address) + if err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { lis.Close() }) + + s := grpc.NewServer() + testpb.RegisterTestServiceServer(s, us) + go s.Serve(lis) + us.cleanups = append(us.cleanups, s.Stop) + us.s = s + + cc, err := grpc.Dial(target, grpc.WithInsecure()) + if err != nil { + return fmt.Errorf("grpc.Dial(%q) = %v", target, err) + } + us.cc = cc + if err := us.waitForReady(cc); err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { cc.Close() }) + + us.client = testpb.NewTestServiceClient(cc) + + return nil +} + +func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for { + s := cc.GetState() + if s == connectivity.Ready { + return nil + } + if !cc.WaitForStateChange(ctx, s) { + return ctx.Err() + } + } +} + +func (us *unixServer) Stop() { + for i := len(us.cleanups) - 1; i >= 0; i-- { + us.cleanups[i]() + } +} + +func runUnixTest(t *testing.T, address, target, expectedAuthority string) { + if err := os.RemoveAll(address); err != nil { + t.Fatalf("Error removing socket file %v: %v\n", address, err) + } + us := &unixServer{ + emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if auths, ok := md[":authority"]; ok { + if len(auths) < 1 { + return nil, status.Error(codes.Unauthenticated, "no authority header") + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.Unauthenticated, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + } else { + return nil, status.Error(codes.Unauthenticated, "no authority header") + } + } else { + return nil, status.Error(codes.Unauthenticated, "failed to parse metadata") + } + return &testpb.Empty{}, nil + }, + } + if err := us.Start("unix", address, target); err != nil { + t.Fatalf("Error starting endpoint server: %v\n", err) + return + } + defer us.Stop() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil\n", err) + } +} + +func (s) TestUnix1(t *testing.T) { + runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") +} + +func (s) TestUnix2(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") +} + +func (s) TestUnix3(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") +} From 34018d23c3e2faadeeb267182787e4cdc9f71acd Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Wed, 8 Jul 2020 10:41:29 -0700 Subject: [PATCH 02/10] Moved unix testing to new file --- test/end2end_test.go | 116 --------------------------------- test/unix_test.go | 151 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 116 deletions(-) create mode 100644 test/unix_test.go diff --git a/test/end2end_test.go b/test/end2end_test.go index 2722ef8d7b3..3e129c3fc15 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -7179,119 +7179,3 @@ func (s) TestCanceledRPCCallOptionRace(t *testing.T) { } wg.Wait() } - -// unixServer is used to test servers listening over a unix socket. -type unixServer struct { - // Guarantees we satisfy this interface; panics if unimplemented methods are called. - testpb.TestServiceServer - - // Customizable implementations of server handlers. - emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) - - // A client connected to this service the test may use. Created in Start(). - client testpb.TestServiceClient - cc *grpc.ClientConn - s *grpc.Server - - cleanups []func() // Lambdas executed in Stop(); populated by Start(). -} - -func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return us.emptyCall(ctx, in) -} - -func (us *unixServer) Start(network, address, target string) error { - lis, err := net.Listen(network, address) - if err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { lis.Close() }) - - s := grpc.NewServer() - testpb.RegisterTestServiceServer(s, us) - go s.Serve(lis) - us.cleanups = append(us.cleanups, s.Stop) - us.s = s - - cc, err := grpc.Dial(target, grpc.WithInsecure()) - if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", target, err) - } - us.cc = cc - if err := us.waitForReady(cc); err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { cc.Close() }) - - us.client = testpb.NewTestServiceClient(cc) - - return nil -} - -func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - return nil - } - if !cc.WaitForStateChange(ctx, s) { - return ctx.Err() - } - } -} - -func (us *unixServer) Stop() { - for i := len(us.cleanups) - 1; i >= 0; i-- { - us.cleanups[i]() - } -} - -func runUnixTest(t *testing.T, address, target, expectedAuthority string) { - if err := os.RemoveAll(address); err != nil { - t.Fatalf("Error removing socket file %v: %v\n", address, err) - } - us := &unixServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - if md, ok := metadata.FromIncomingContext(ctx); ok { - if auths, ok := md[":authority"]; ok { - if len(auths) < 1 { - return nil, status.Error(codes.Unauthenticated, "no authority header") - } - if auths[0] != expectedAuthority { - return nil, status.Error(codes.Unauthenticated, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) - } - } else { - return nil, status.Error(codes.Unauthenticated, "no authority header") - } - } else { - return nil, status.Error(codes.Unauthenticated, "failed to parse metadata") - } - return &testpb.Empty{}, nil - }, - } - if err := us.Start("unix", address, target); err != nil { - t.Fatalf("Error starting endpoint server: %v\n", err) - return - } - defer us.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) - if err != nil { - t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil\n", err) - } -} - -func (s) TestUnix1(t *testing.T) { - runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") -} - -func (s) TestUnix2(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") -} - -func (s) TestUnix3(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") -} diff --git a/test/unix_test.go b/test/unix_test.go new file mode 100644 index 00000000000..6a43e3c4478 --- /dev/null +++ b/test/unix_test.go @@ -0,0 +1,151 @@ +/* + * + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "net" + "os" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + testpb "google.golang.org/grpc/test/grpc_testing" +) + +// unixServer is used to test servers listening over a unix socket. +type unixServer struct { + // Guarantees we satisfy this interface; panics if unimplemented methods are called. + testpb.TestServiceServer + + // Customizable implementations of server handlers. + emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) + + // A client connected to this service the test may use. Created in Start(). + client testpb.TestServiceClient + cc *grpc.ClientConn + s *grpc.Server + + cleanups []func() // Lambdas executed in Stop(); populated by Start(). +} + +func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return us.emptyCall(ctx, in) +} + +func (us *unixServer) Start(network, address, target string) error { + lis, err := net.Listen(network, address) + if err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { lis.Close() }) + + s := grpc.NewServer() + testpb.RegisterTestServiceServer(s, us) + go s.Serve(lis) + us.cleanups = append(us.cleanups, s.Stop) + us.s = s + + cc, err := grpc.Dial(target, grpc.WithInsecure()) + if err != nil { + return fmt.Errorf("grpc.Dial(%q) = %v", target, err) + } + us.cc = cc + if err := us.waitForReady(cc); err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { cc.Close() }) + + us.client = testpb.NewTestServiceClient(cc) + + return nil +} + +func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for { + s := cc.GetState() + if s == connectivity.Ready { + return nil + } + if !cc.WaitForStateChange(ctx, s) { + return ctx.Err() + } + } +} + +func (us *unixServer) Stop() { + for i := len(us.cleanups) - 1; i >= 0; i-- { + us.cleanups[i]() + } +} + +func runUnixTest(t *testing.T, address, target, expectedAuthority string) { + if err := os.RemoveAll(address); err != nil { + t.Fatalf("Error removing socket file %v: %v\n", address, err) + } + us := &unixServer{ + emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if auths, ok := md[":authority"]; ok { + if len(auths) < 1 { + return nil, status.Error(codes.Unauthenticated, "no authority header") + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.Unauthenticated, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + } else { + return nil, status.Error(codes.Unauthenticated, "no authority header") + } + } else { + return nil, status.Error(codes.Unauthenticated, "failed to parse metadata") + } + return &testpb.Empty{}, nil + }, + } + if err := us.Start("unix", address, target); err != nil { + t.Fatalf("Error starting endpoint server: %v\n", err) + return + } + defer us.Stop() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil\n", err) + } +} + +func (s) TestUnix1(t *testing.T) { + runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") +} + +func (s) TestUnix2(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") +} + +func (s) TestUnix3(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") +} From e57680869dfe801246c05f8e26f27bf2a28ebfec Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Wed, 8 Jul 2020 11:30:46 -0700 Subject: [PATCH 03/10] Fixed license --- test/unix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unix_test.go b/test/unix_test.go index 6a43e3c4478..e29169b709d 100644 --- a/test/unix_test.go +++ b/test/unix_test.go @@ -1,6 +1,6 @@ /* * - * Copyright 2020 Google LLC + * Copyright 2020 gRPC authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. From 3c3dde6331cc835e6f4b2019d324da3e89d12322 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Wed, 15 Jul 2020 12:45:15 -0700 Subject: [PATCH 04/10] Using stubServer --- clientconn.go | 2 +- internal/grpcutil/target.go | 3 - test/authority_test.go | 81 +++++++++++++++++++ test/end2end_test.go | 18 ++++- test/unix_test.go | 151 ------------------------------------ 5 files changed, 96 insertions(+), 159 deletions(-) create mode 100644 test/authority_test.go delete mode 100644 test/unix_test.go diff --git a/clientconn.go b/clientconn.go index 2b7d17d8c50..15654b39ec5 100644 --- a/clientconn.go +++ b/clientconn.go @@ -245,7 +245,7 @@ func DialContext(ctx context.Context, target string, opts ...DialOption) (conn * // Determine the resolver to use. cc.parsedTarget = grpcutil.ParseTarget(cc.target) - unixScheme := strings.HasPrefix(cc.parsedTarget.Endpoint, "unix:") + unixScheme := strings.HasPrefix(cc.target, "unix:") channelz.Infof(logger, cc.channelzID, "parsed scheme: %q", cc.parsedTarget.Scheme) resolverBuilder := cc.getResolver(cc.parsedTarget.Scheme) if resolverBuilder == nil { diff --git a/internal/grpcutil/target.go b/internal/grpcutil/target.go index 2e87f395b0a..80b33cdaf90 100644 --- a/internal/grpcutil/target.go +++ b/internal/grpcutil/target.go @@ -43,9 +43,6 @@ func split2(s, sep string) (string, string, bool) { // target}. func ParseTarget(target string) (ret resolver.Target) { var ok bool - if strings.HasPrefix(target, "unix:") { - return resolver.Target{Endpoint: target} - } ret.Scheme, ret.Endpoint, ok = split2(target, "://") if !ok { return resolver.Target{Endpoint: target} diff --git a/test/authority_test.go b/test/authority_test.go new file mode 100644 index 00000000000..df6e1368a8b --- /dev/null +++ b/test/authority_test.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + testpb "google.golang.org/grpc/test/grpc_testing" +) + +func runUnixTest(t *testing.T, address, target, expectedAuthority string) { + if err := os.RemoveAll(address); err != nil { + t.Fatalf("Error removing socket file %v: %v\n", address, err) + } + us := &stubServer{ + emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + if md, ok := metadata.FromIncomingContext(ctx); ok { + if auths, ok := md[":authority"]; ok { + if len(auths) < 1 { + return nil, status.Error(codes.InvalidArgument, "no authority header") + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + return &testpb.Empty{}, nil + } + return nil, status.Error(codes.InvalidArgument, "no authority header") + } + return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") + }, + network: "unix", + address: address, + target: target, + } + if err := us.Start(nil); err != nil { + t.Fatalf("Error starting endpoint server: %v", err) + return + } + defer us.Stop() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) + if err != nil { + t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil", err) + } +} + +func (s) TestUnix1(t *testing.T) { + runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") +} + +func (s) TestUnix2(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") +} + +func (s) TestUnix3(t *testing.T) { + runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") +} diff --git a/test/end2end_test.go b/test/end2end_test.go index 3e129c3fc15..ea5ad8a3f44 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -4991,6 +4991,10 @@ type stubServer struct { cc *grpc.ClientConn s *grpc.Server + network string + address string + target string + addr string // address of listener cleanups []func() // Lambdas executed in Stop(); populated by Start(). @@ -5015,7 +5019,11 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) r := manual.NewBuilderWithScheme("whatever") ss.r = r - lis, err := net.Listen("tcp", "localhost:0") + if ss.network == "" && ss.address == "" { + ss.network = "tcp" + ss.address = "localhost:0" + } + lis, err := net.Listen(ss.network, ss.address) if err != nil { return fmt.Errorf(`net.Listen("tcp", "localhost:0") = %v`, err) } @@ -5028,12 +5036,14 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) ss.cleanups = append(ss.cleanups, s.Stop) ss.s = s - target := ss.r.Scheme() + ":///" + ss.addr + if ss.target == "" { + ss.target = ss.r.Scheme() + ":///" + ss.addr + } opts := append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(r)}, dopts...) - cc, err := grpc.Dial(target, opts...) + cc, err := grpc.Dial(ss.target, opts...) if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", target, err) + return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err) } ss.cc = cc ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}}) diff --git a/test/unix_test.go b/test/unix_test.go deleted file mode 100644 index e29169b709d..00000000000 --- a/test/unix_test.go +++ /dev/null @@ -1,151 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package test - -import ( - "context" - "fmt" - "net" - "os" - "testing" - "time" - - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" - testpb "google.golang.org/grpc/test/grpc_testing" -) - -// unixServer is used to test servers listening over a unix socket. -type unixServer struct { - // Guarantees we satisfy this interface; panics if unimplemented methods are called. - testpb.TestServiceServer - - // Customizable implementations of server handlers. - emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) - - // A client connected to this service the test may use. Created in Start(). - client testpb.TestServiceClient - cc *grpc.ClientConn - s *grpc.Server - - cleanups []func() // Lambdas executed in Stop(); populated by Start(). -} - -func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return us.emptyCall(ctx, in) -} - -func (us *unixServer) Start(network, address, target string) error { - lis, err := net.Listen(network, address) - if err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { lis.Close() }) - - s := grpc.NewServer() - testpb.RegisterTestServiceServer(s, us) - go s.Serve(lis) - us.cleanups = append(us.cleanups, s.Stop) - us.s = s - - cc, err := grpc.Dial(target, grpc.WithInsecure()) - if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", target, err) - } - us.cc = cc - if err := us.waitForReady(cc); err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { cc.Close() }) - - us.client = testpb.NewTestServiceClient(cc) - - return nil -} - -func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - return nil - } - if !cc.WaitForStateChange(ctx, s) { - return ctx.Err() - } - } -} - -func (us *unixServer) Stop() { - for i := len(us.cleanups) - 1; i >= 0; i-- { - us.cleanups[i]() - } -} - -func runUnixTest(t *testing.T, address, target, expectedAuthority string) { - if err := os.RemoveAll(address); err != nil { - t.Fatalf("Error removing socket file %v: %v\n", address, err) - } - us := &unixServer{ - emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - if md, ok := metadata.FromIncomingContext(ctx); ok { - if auths, ok := md[":authority"]; ok { - if len(auths) < 1 { - return nil, status.Error(codes.Unauthenticated, "no authority header") - } - if auths[0] != expectedAuthority { - return nil, status.Error(codes.Unauthenticated, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) - } - } else { - return nil, status.Error(codes.Unauthenticated, "no authority header") - } - } else { - return nil, status.Error(codes.Unauthenticated, "failed to parse metadata") - } - return &testpb.Empty{}, nil - }, - } - if err := us.Start("unix", address, target); err != nil { - t.Fatalf("Error starting endpoint server: %v\n", err) - return - } - defer us.Stop() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - _, err := us.client.EmptyCall(ctx, &testpb.Empty{}) - if err != nil { - t.Errorf("us.client.EmptyCall(_, _) = _, %v; want _, nil\n", err) - } -} - -func (s) TestUnix1(t *testing.T) { - runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") -} - -func (s) TestUnix2(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") -} - -func (s) TestUnix3(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") -} From 05a47d2041d2a32a1bbcdcc55aebbbb4363c3b93 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Wed, 15 Jul 2020 12:53:39 -0700 Subject: [PATCH 05/10] Merged tests --- test/authority_test.go | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/test/authority_test.go b/test/authority_test.go index df6e1368a8b..5b210b6f590 100644 --- a/test/authority_test.go +++ b/test/authority_test.go @@ -68,14 +68,35 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string) { } } -func (s) TestUnix1(t *testing.T) { - runUnixTest(t, "sock.sock", "unix:sock.sock", "localhost") -} - -func (s) TestUnix2(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:/tmp/sock.sock", "localhost") -} - -func (s) TestUnix3(t *testing.T) { - runUnixTest(t, "/tmp/sock.sock", "unix:///tmp/sock.sock", "localhost") +func (s) TestUnix(t *testing.T) { + tests := []struct { + name string + address string + target string + authority string + }{ + { + name: "Unix1", + address: "sock.sock", + target: "unix:sock.sock", + authority: "loclahost", + }, + { + name: "Unix2", + address: "/tmp/sock.sock", + target: "unix:/tmp/sock.sock", + authority: "loclahost", + }, + { + name: "Unix3", + address: "/tmp/sock.sock", + target: "unix:///tmp/sock.sock", + authority: "loclahost", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runUnixTest(t, test.address, test.target, test.authority) + }) + } } From 32ad8b7e8d8023f89b2e365d478eaf1e368e8918 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Thu, 16 Jul 2020 10:32:00 -0700 Subject: [PATCH 06/10] Moved back to unixServer --- test/authority_test.go | 108 +++++++++++++++++++++++++++++++++-------- test/end2end_test.go | 18 ++----- 2 files changed, 93 insertions(+), 33 deletions(-) diff --git a/test/authority_test.go b/test/authority_test.go index 5b210b6f590..721d545810d 100644 --- a/test/authority_test.go +++ b/test/authority_test.go @@ -21,41 +21,111 @@ package test import ( "context" "fmt" + "net" "os" "testing" "time" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" testpb "google.golang.org/grpc/test/grpc_testing" ) +// unixServer is used to test servers listening over a unix socket. +type unixServer struct { + // Guarantees we satisfy this interface; panics if unimplemented methods are called. + testpb.TestServiceServer + + // Customizable implementations of server handlers. + emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) + + // A client connected to this service the test may use. Created in Start(). + client testpb.TestServiceClient + cc *grpc.ClientConn + s *grpc.Server + + cleanups []func() // Lambdas executed in Stop(); populated by Start(). +} + +func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { + return us.emptyCall(ctx, in) +} + +func (us *unixServer) Start(address, target string) error { + lis, err := net.Listen("unix", address) + if err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { lis.Close() }) + + s := grpc.NewServer() + testpb.RegisterTestServiceServer(s, us) + go s.Serve(lis) + us.cleanups = append(us.cleanups, s.Stop) + us.s = s + + cc, err := grpc.Dial(target, grpc.WithInsecure()) + if err != nil { + return fmt.Errorf("grpc.Dial(%q) = %v", target, err) + } + us.cc = cc + if err := us.waitForReady(cc); err != nil { + return err + } + us.cleanups = append(us.cleanups, func() { cc.Close() }) + + us.client = testpb.NewTestServiceClient(cc) + + return nil +} + +func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for { + s := cc.GetState() + if s == connectivity.Ready { + return nil + } + if !cc.WaitForStateChange(ctx, s) { + return ctx.Err() + } + } +} + +func (us *unixServer) Stop() { + for i := len(us.cleanups) - 1; i >= 0; i-- { + us.cleanups[i]() + } +} + func runUnixTest(t *testing.T, address, target, expectedAuthority string) { if err := os.RemoveAll(address); err != nil { t.Fatalf("Error removing socket file %v: %v\n", address, err) } - us := &stubServer{ + us := &unixServer{ emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - if md, ok := metadata.FromIncomingContext(ctx); ok { - if auths, ok := md[":authority"]; ok { - if len(auths) < 1 { - return nil, status.Error(codes.InvalidArgument, "no authority header") - } - if auths[0] != expectedAuthority { - return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) - } - return &testpb.Empty{}, nil - } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") + } + auths, ok := md[":authority"] + if !ok { return nil, status.Error(codes.InvalidArgument, "no authority header") } - return nil, status.Error(codes.InvalidArgument, "failed to parse metadata") + if len(auths) < 1 { + return nil, status.Error(codes.InvalidArgument, "no authority header") + } + if auths[0] != expectedAuthority { + return nil, status.Error(codes.InvalidArgument, fmt.Sprintf("invalid authority header %v, expected %v", auths[0], expectedAuthority)) + } + return &testpb.Empty{}, nil }, - network: "unix", - address: address, - target: target, } - if err := us.Start(nil); err != nil { + if err := us.Start(address, target); err != nil { t.Fatalf("Error starting endpoint server: %v", err) return } @@ -79,19 +149,19 @@ func (s) TestUnix(t *testing.T) { name: "Unix1", address: "sock.sock", target: "unix:sock.sock", - authority: "loclahost", + authority: "localhost", }, { name: "Unix2", address: "/tmp/sock.sock", target: "unix:/tmp/sock.sock", - authority: "loclahost", + authority: "localhost", }, { name: "Unix3", address: "/tmp/sock.sock", target: "unix:///tmp/sock.sock", - authority: "loclahost", + authority: "localhost", }, } for _, test := range tests { diff --git a/test/end2end_test.go b/test/end2end_test.go index ea5ad8a3f44..3e129c3fc15 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -4991,10 +4991,6 @@ type stubServer struct { cc *grpc.ClientConn s *grpc.Server - network string - address string - target string - addr string // address of listener cleanups []func() // Lambdas executed in Stop(); populated by Start(). @@ -5019,11 +5015,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) r := manual.NewBuilderWithScheme("whatever") ss.r = r - if ss.network == "" && ss.address == "" { - ss.network = "tcp" - ss.address = "localhost:0" - } - lis, err := net.Listen(ss.network, ss.address) + lis, err := net.Listen("tcp", "localhost:0") if err != nil { return fmt.Errorf(`net.Listen("tcp", "localhost:0") = %v`, err) } @@ -5036,14 +5028,12 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) ss.cleanups = append(ss.cleanups, s.Stop) ss.s = s - if ss.target == "" { - ss.target = ss.r.Scheme() + ":///" + ss.addr - } + target := ss.r.Scheme() + ":///" + ss.addr opts := append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(r)}, dopts...) - cc, err := grpc.Dial(ss.target, opts...) + cc, err := grpc.Dial(target, opts...) if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err) + return fmt.Errorf("grpc.Dial(%q) = %v", target, err) } ss.cc = cc ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}}) From 05e2fc8c81c812e05ab032ed7c7a4a70b3239d56 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Thu, 16 Jul 2020 12:01:55 -0700 Subject: [PATCH 07/10] Moved to modified stubServer --- test/authority_test.go | 14 +++++++------- test/end2end_test.go | 36 ++++++++++++++++++++++++++---------- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/test/authority_test.go b/test/authority_test.go index 721d545810d..99a8bda1331 100644 --- a/test/authority_test.go +++ b/test/authority_test.go @@ -21,20 +21,17 @@ package test import ( "context" "fmt" - "net" "os" "testing" "time" - "google.golang.org/grpc" "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" testpb "google.golang.org/grpc/test/grpc_testing" ) -// unixServer is used to test servers listening over a unix socket. +/*// unixServer is used to test servers listening over a unix socket. type unixServer struct { // Guarantees we satisfy this interface; panics if unimplemented methods are called. testpb.TestServiceServer @@ -100,13 +97,13 @@ func (us *unixServer) Stop() { for i := len(us.cleanups) - 1; i >= 0; i-- { us.cleanups[i]() } -} +}*/ func runUnixTest(t *testing.T, address, target, expectedAuthority string) { if err := os.RemoveAll(address); err != nil { t.Fatalf("Error removing socket file %v: %v\n", address, err) } - us := &unixServer{ + us := &stubServer{ emptyCall: func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { md, ok := metadata.FromIncomingContext(ctx) if !ok { @@ -124,8 +121,11 @@ func runUnixTest(t *testing.T, address, target, expectedAuthority string) { } return &testpb.Empty{}, nil }, + network: "unix", + address: address, + target: target, } - if err := us.Start(address, target); err != nil { + if err := us.Start(nil); err != nil { t.Fatalf("Error starting endpoint server: %v", err) return } diff --git a/test/end2end_test.go b/test/end2end_test.go index 3e129c3fc15..0d6b0cc8ede 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -4991,6 +4991,10 @@ type stubServer struct { cc *grpc.ClientConn s *grpc.Server + network string + address string + target string + addr string // address of listener cleanups []func() // Lambdas executed in Stop(); populated by Start(). @@ -5012,12 +5016,15 @@ func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallSer // Start starts the server and creates a client connected to it. func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { - r := manual.NewBuilderWithScheme("whatever") - ss.r = r + if ss.network == "" && ss.address == "" && ss.target == "" { + ss.r = manual.NewBuilderWithScheme("whatever") + ss.network = "tcp" + ss.address = "localhost:0" + } - lis, err := net.Listen("tcp", "localhost:0") + lis, err := net.Listen(ss.network, ss.address) if err != nil { - return fmt.Errorf(`net.Listen("tcp", "localhost:0") = %v`, err) + return fmt.Errorf(`net.Listen("%v", "%v") = %v`, ss.network, ss.address, err) } ss.addr = lis.Addr().String() ss.cleanups = append(ss.cleanups, func() { lis.Close() }) @@ -5028,15 +5035,22 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) ss.cleanups = append(ss.cleanups, s.Stop) ss.s = s - target := ss.r.Scheme() + ":///" + ss.addr + var opts []grpc.DialOption + if ss.r != nil { + ss.target = ss.r.Scheme() + ":///" + ss.addr + opts = append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(ss.r)}, dopts...) + } else { + opts = append([]grpc.DialOption{grpc.WithInsecure()}, dopts...) + } - opts := append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(r)}, dopts...) - cc, err := grpc.Dial(target, opts...) + cc, err := grpc.Dial(ss.target, opts...) if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", target, err) + return fmt.Errorf("grpc.Dial(%q) = %v", ss.target, err) } ss.cc = cc - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}}) + if ss.r != nil { + ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}}) + } if err := ss.waitForReady(cc); err != nil { return err } @@ -5048,7 +5062,9 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) } func (ss *stubServer) newServiceConfig(sc string) { - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}, ServiceConfig: parseCfg(ss.r, sc)}) + if ss.r != nil { + ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}, ServiceConfig: parseCfg(ss.r, sc)}) + } } func (ss *stubServer) waitForReady(cc *grpc.ClientConn) error { From b54464c6fddada0f69ed244b79cf716f869c6fd3 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Thu, 16 Jul 2020 12:03:14 -0700 Subject: [PATCH 08/10] Removed comments --- test/authority_test.go | 68 ------------------------------------------ 1 file changed, 68 deletions(-) diff --git a/test/authority_test.go b/test/authority_test.go index 99a8bda1331..6cd5d82eec1 100644 --- a/test/authority_test.go +++ b/test/authority_test.go @@ -31,74 +31,6 @@ import ( testpb "google.golang.org/grpc/test/grpc_testing" ) -/*// unixServer is used to test servers listening over a unix socket. -type unixServer struct { - // Guarantees we satisfy this interface; panics if unimplemented methods are called. - testpb.TestServiceServer - - // Customizable implementations of server handlers. - emptyCall func(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) - - // A client connected to this service the test may use. Created in Start(). - client testpb.TestServiceClient - cc *grpc.ClientConn - s *grpc.Server - - cleanups []func() // Lambdas executed in Stop(); populated by Start(). -} - -func (us *unixServer) EmptyCall(ctx context.Context, in *testpb.Empty) (*testpb.Empty, error) { - return us.emptyCall(ctx, in) -} - -func (us *unixServer) Start(address, target string) error { - lis, err := net.Listen("unix", address) - if err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { lis.Close() }) - - s := grpc.NewServer() - testpb.RegisterTestServiceServer(s, us) - go s.Serve(lis) - us.cleanups = append(us.cleanups, s.Stop) - us.s = s - - cc, err := grpc.Dial(target, grpc.WithInsecure()) - if err != nil { - return fmt.Errorf("grpc.Dial(%q) = %v", target, err) - } - us.cc = cc - if err := us.waitForReady(cc); err != nil { - return err - } - us.cleanups = append(us.cleanups, func() { cc.Close() }) - - us.client = testpb.NewTestServiceClient(cc) - - return nil -} - -func (us *unixServer) waitForReady(cc *grpc.ClientConn) error { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for { - s := cc.GetState() - if s == connectivity.Ready { - return nil - } - if !cc.WaitForStateChange(ctx, s) { - return ctx.Err() - } - } -} - -func (us *unixServer) Stop() { - for i := len(us.cleanups) - 1; i >= 0; i-- { - us.cleanups[i]() - } -}*/ - func runUnixTest(t *testing.T, address, target, expectedAuthority string) { if err := os.RemoveAll(address); err != nil { t.Fatalf("Error removing socket file %v: %v\n", address, err) From bae44e37e4a2ec719427dede50eba4bc0a52e8d1 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Fri, 17 Jul 2020 11:29:26 -0700 Subject: [PATCH 09/10] Minor changes to stubServer unix behavior --- test/end2end_test.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/end2end_test.go b/test/end2end_test.go index 0d6b0cc8ede..746ec7f2558 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -4991,6 +4991,8 @@ type stubServer struct { cc *grpc.ClientConn s *grpc.Server + // Parameters for Listen and Dial. Defaults will be used if these are empty + // before Start. network string address string target string @@ -5016,15 +5018,19 @@ func (ss *stubServer) FullDuplexCall(stream testpb.TestService_FullDuplexCallSer // Start starts the server and creates a client connected to it. func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) error { - if ss.network == "" && ss.address == "" && ss.target == "" { - ss.r = manual.NewBuilderWithScheme("whatever") + if ss.network == "" { ss.network = "tcp" + } + if ss.address == "" { ss.address = "localhost:0" } + if ss.target == "" { + ss.r = manual.NewBuilderWithScheme("whatever") + } lis, err := net.Listen(ss.network, ss.address) if err != nil { - return fmt.Errorf(`net.Listen("%v", "%v") = %v`, ss.network, ss.address, err) + return fmt.Errorf("net.Listen(%q, %q) = %v", ss.network, ss.address, err) } ss.addr = lis.Addr().String() ss.cleanups = append(ss.cleanups, func() { lis.Close() }) @@ -5035,12 +5041,10 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) ss.cleanups = append(ss.cleanups, s.Stop) ss.s = s - var opts []grpc.DialOption + opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...) if ss.r != nil { ss.target = ss.r.Scheme() + ":///" + ss.addr - opts = append([]grpc.DialOption{grpc.WithInsecure(), grpc.WithResolvers(ss.r)}, dopts...) - } else { - opts = append([]grpc.DialOption{grpc.WithInsecure()}, dopts...) + opts = append(opts, grpc.WithResolvers(ss.r)) } cc, err := grpc.Dial(ss.target, opts...) From feb06507a5b5035b59687c99177b39fc3b24cc11 Mon Sep 17 00:00:00 2001 From: "gargut@google.com" Date: Fri, 17 Jul 2020 12:14:47 -0700 Subject: [PATCH 10/10] Removed addr from stubServer --- test/end2end_test.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/end2end_test.go b/test/end2end_test.go index 746ec7f2558..0b735c15984 100644 --- a/test/end2end_test.go +++ b/test/end2end_test.go @@ -4997,8 +4997,6 @@ type stubServer struct { address string target string - addr string // address of listener - cleanups []func() // Lambdas executed in Stop(); populated by Start(). r *manual.Resolver @@ -5032,7 +5030,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) if err != nil { return fmt.Errorf("net.Listen(%q, %q) = %v", ss.network, ss.address, err) } - ss.addr = lis.Addr().String() + ss.address = lis.Addr().String() ss.cleanups = append(ss.cleanups, func() { lis.Close() }) s := grpc.NewServer(sopts...) @@ -5043,7 +5041,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) opts := append([]grpc.DialOption{grpc.WithInsecure()}, dopts...) if ss.r != nil { - ss.target = ss.r.Scheme() + ":///" + ss.addr + ss.target = ss.r.Scheme() + ":///" + ss.address opts = append(opts, grpc.WithResolvers(ss.r)) } @@ -5053,7 +5051,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) } ss.cc = cc if ss.r != nil { - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}}) + ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}}) } if err := ss.waitForReady(cc); err != nil { return err @@ -5067,7 +5065,7 @@ func (ss *stubServer) Start(sopts []grpc.ServerOption, dopts ...grpc.DialOption) func (ss *stubServer) newServiceConfig(sc string) { if ss.r != nil { - ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.addr}}, ServiceConfig: parseCfg(ss.r, sc)}) + ss.r.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: ss.address}}, ServiceConfig: parseCfg(ss.r, sc)}) } }