Merge pull request #1178 from tonistiigi/1903-picks

[docker-19.03] docker 19.03 / v0.6.2 cherry picks

client/client_test.go

@@ -101,6 +101,8 @@ func TestClientIntegration(t *testing.T) {
 		testBridgeNetworking,
 		testCacheMountNoCache,
 		testExporterTargetExists,
+		testTarExporterWithSocket,
+		testMultipleRegistryCacheImportExport,
 	}, mirrors)
 
 	integration.Run(t, []integration.Test{
@@ -582,7 +584,8 @@ func testSecurityModeSysfs(t *testing.T, sb integration.Sandbox) {
 
 	if secMode == securitySandbox {
 		require.Error(t, err)
-		require.Contains(t, err.Error(), "exit code: 1")
+		require.Contains(t, err.Error(), "executor failed running")
+		require.Contains(t, err.Error(), "mkdir /sys/fs/cgroup/cpuset/securitytest")
 	} else {
 		require.NoError(t, err)
 	}
@@ -1431,6 +1434,30 @@ func testExporterTargetExists(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, dgst, mdDgst)
 }
 
+func testTarExporterWithSocket(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	alpine := llb.Image("docker.io/library/alpine:latest")
+	def, err := alpine.Run(llb.Args([]string{"sh", "-c", "nc -l -s local:/socket.sock & usleep 100000; kill %1"})).Marshal()
+	require.NoError(t, err)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:  ExporterTar,
+				Attrs: map[string]string{},
+				Output: func(m map[string]string) (io.WriteCloser, error) {
+					return nopWriteCloser{ioutil.Discard}, nil
+				},
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+}
+
 func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(context.TODO(), sb.Address())
@@ -1632,7 +1659,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
 	require.False(t, ok)
 }
 
-func testBasicCacheImportExport(t *testing.T, sb integration.Sandbox, cacheOptionsEntryImport, cacheOptionsEntryExport CacheOptionsEntry) {
+func testBasicCacheImportExport(t *testing.T, sb integration.Sandbox, cacheOptionsEntryImport, cacheOptionsEntryExport []CacheOptionsEntry) {
 	requiresLinux(t)
 	c, err := New(context.TODO(), sb.Address())
 	require.NoError(t, err)
@@ -1662,9 +1689,7 @@ func testBasicCacheImportExport(t *testing.T, sb integration.Sandbox, cacheOptio
 				OutputDir: destDir,
 			},
 		},
-		CacheExports: []CacheOptionsEntry{
-			cacheOptionsEntryExport,
-		},
+		CacheExports: cacheOptionsEntryExport,
 	}, nil)
 	require.NoError(t, err)
 
@@ -1690,9 +1715,7 @@ func testBasicCacheImportExport(t *testing.T, sb integration.Sandbox, cacheOptio
 				Type:      ExporterLocal,
 				OutputDir: destDir,
 			}},
-		CacheImports: []CacheOptionsEntry{
-			cacheOptionsEntryImport,
-		},
+		CacheImports: cacheOptionsEntryImport,
 	}, nil)
 	require.NoError(t, err)
 
@@ -1718,7 +1741,29 @@ func testBasicRegistryCacheImportExport(t *testing.T, sb integration.Sandbox) {
 			"ref": target,
 		},
 	}
-	testBasicCacheImportExport(t, sb, o, o)
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{o}, []CacheOptionsEntry{o})
+}
+
+func testMultipleRegistryCacheImportExport(t *testing.T, sb integration.Sandbox) {
+	registry, err := sb.NewRegistry()
+	if errors.Cause(err) == integration.ErrorRequirements {
+		t.Skip(err.Error())
+	}
+	require.NoError(t, err)
+	target := registry + "/buildkit/testexport:latest"
+	o := CacheOptionsEntry{
+		Type: "registry",
+		Attrs: map[string]string{
+			"ref": target,
+		},
+	}
+	o2 := CacheOptionsEntry{
+		Type: "registry",
+		Attrs: map[string]string{
+			"ref": target + "notexist",
+		},
+	}
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{o, o2}, []CacheOptionsEntry{o})
 }
 
 func testBasicLocalCacheImportExport(t *testing.T, sb integration.Sandbox) {
@@ -1737,7 +1782,7 @@ func testBasicLocalCacheImportExport(t *testing.T, sb integration.Sandbox) {
 			"dest": dir,
 		},
 	}
-	testBasicCacheImportExport(t, sb, im, ex)
+	testBasicCacheImportExport(t, sb, []CacheOptionsEntry{im}, []CacheOptionsEntry{ex})
 }
 
 func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {

executor/runcexecutor/executor.go

@@ -293,12 +293,11 @@ func (w *runcExecutor) Exec(ctx context.Context, meta executor.Meta, root cache.
 		NoPivot: w.noPivot,
 	})
 	close(done)
-	if err != nil {
-		return err
-	}
 
-	if status != 0 {
-		err := errors.Errorf("exit code: %d", status)
+	if status != 0 || err != nil {
+		if err == nil {
+			err = errors.Errorf("exit code: %d", status)
+		}
 		select {
 		case <-ctx.Done():
 			return errors.Wrapf(ctx.Err(), err.Error())

frontend/dockerfile/dockerfile_test.go

@@ -99,6 +99,7 @@ var fileOpTests = []integration.Test{
 	testCopyChownCreateDest,
 	testCopyThroughSymlinkContext,
 	testCopyThroughSymlinkMultiStage,
+	testCopySocket,
 	testContextChangeDirToFile,
 	testNoSnapshotLeak,
 	testCopySymlinks,
@@ -1013,6 +1014,53 @@ COPY --from=build /sub2/foo bar
 	require.Equal(t, "data", string(dt))
 }
 
+func testCopySocket(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+	isFileOp := getFileOp(t, sb)
+
+	dockerfile := []byte(`
+FROM scratch
+COPY . /
+`)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", dockerfile, 0600),
+		fstest.CreateSocket("socket.sock", 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = f.Solve(context.TODO(), c, client.SolveOpt{
+		Exports: []client.ExportEntry{
+			{
+				Type:      client.ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		FrontendAttrs: map[string]string{
+			"build-arg:BUILDKIT_DISABLE_FILEOP": strconv.FormatBool(!isFileOp),
+		},
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	fi, err := os.Lstat(filepath.Join(destDir, "socket.sock"))
+	require.NoError(t, err)
+	// make sure socket is converted to regular file.
+	require.Equal(t, fi.Mode().IsRegular(), true)
+}
+
 func testIgnoreEntrypoint(t *testing.T, sb integration.Sandbox) {
 	f := getFrontend(t, sb)
 

go.mod

@@ -10,10 +10,10 @@ require (
 	github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50
 	github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
+	github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6
 	github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 // indirect
 	github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
+	github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda
 	github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 // indirect
 	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd // indirect
 	github.com/containernetworking/cni v0.6.1-0.20180218032124-142cde0c766c // indirect
@@ -53,7 +53,7 @@ require (
 	github.com/sirupsen/logrus v1.3.0
 	github.com/stretchr/testify v1.3.0
 	github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
-	github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76
+	github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
 	github.com/uber/jaeger-client-go v0.0.0-20180103221425-e02c85f9069e
 	github.com/uber/jaeger-lib v1.2.1 // indirect

go.sum

@@ -19,14 +19,14 @@ github.com/containerd/containerd v1.2.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX
 github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0 h1:enps1EZBEgR8QxwdrpsoSxcsCXWnMKchIQ/0dzC0eKw=
 github.com/containerd/containerd v1.3.0-0.20190507210959-7c1e88399ec0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20181001140422-bd77b46c8352/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
-github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 h1:NmTXa/uVnDyp0TY5MKi197+3HWcnYWfnHGyaFthlnGw=
+github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260 h1:XGyg7oTtD0DoRFhbpV6x1WfV0flKC4UxXU7ab1zC08U=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645 h1:CCxW+4asjsbhMjWAznv/rTLYYI7Mcm6LovkzaPtD3rU=
 github.com/containerd/go-cni v0.0.0-20190610170741-5a4663dad645/go.mod h1:2wlRxCQdiBY+OcjNg5x8kI+5mEL1fGt25L4IzQHYJsM=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3 h1:esQOJREg8nw8aXj6uCN5dfW5cKUBiEJ/+nni1Q/D/sw=
-github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda h1:3LYDkJtHAZGbWD75PoTBJuxldc+njsz9uSfjwIoOR6c=
+github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7 h1:SKDlsIhYxNE1LO0xwuOR+3QWj3zRibVQu5jWIMQmOfU=
 github.com/containerd/ttrpc v0.0.0-20190411181408-699c4e40d1e7/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd h1:JNn81o/xG+8NEo3bC/vx9pbi/g2WI8mtP2/nXzu297Y=
@@ -134,8 +134,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76 h1:eGfgYrNUSD448sa4mxH6nQpyZfN39QH0mLB7QaKIjus=
-github.com/tonistiigi/fsutil v0.0.0-20190327153851-3bbb99cdbd76/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
+github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d h1:HJg27yqwTV7vFG9dWPDbUi373o/bmSDYGN9mZgVwdH0=
+github.com/tonistiigi/fsutil v0.0.0-20190819224149-3d2716dd0a4d/go.mod h1:pzh7kdwkDRh+Bx8J30uqaKJ1M4QrSH/um8fcIXeM8rc=
 github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe h1:pd7hrFSqUPxYS9IB+UMG1AB/8EXGXo17ssx0bSQ5L6Y=
 github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe/go.mod h1:/+MCh11CJf2oz0BXmlmqyopK/ad1rKkcOXPoYuPCJYU=
 github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=

session/sshforward/copy.go

@@ -9,17 +9,17 @@ import (
 	"google.golang.org/grpc"
 )
 
-func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) error {
+func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream, closeStream func() error) error {
 	g, ctx := errgroup.WithContext(ctx)
 
 	g.Go(func() (retErr error) {
 		p := &BytesMessage{}
 		for {
 			if err := stream.RecvMsg(p); err != nil {
+				conn.Close()
 				if err == io.EOF {
 					return nil
 				}
-				conn.Close()
 				return errors.WithStack(err)
 			}
 			select {
@@ -42,6 +42,9 @@ func Copy(ctx context.Context, conn io.ReadWriteCloser, stream grpc.Stream) erro
 			n, err := conn.Read(buf)
 			switch {
 			case err == io.EOF:
+				if closeStream != nil {
+					closeStream()
+				}
 				return nil
 			case err != nil:
 				return errors.WithStack(err)

session/sshforward/ssh.go

@@ -49,7 +49,7 @@ func (s *server) run(ctx context.Context, l net.Listener, id string) error {
 				return err
 			}
 
-			go Copy(ctx, conn, stream)
+			go Copy(ctx, conn, stream, stream.CloseSend)
 		}
 	})
 

session/sshforward/sshprovider/agentprovider.go

@@ -114,7 +114,7 @@ func (sp *socketProvider) ForwardAgent(stream sshforward.SSH_ForwardAgentServer)
 
 	eg.Go(func() error {
 		defer s1.Close()
-		return sshforward.Copy(ctx, s2, stream)
+		return sshforward.Copy(ctx, s2, stream, nil)
 	})
 
 	return eg.Wait()

solver/edge.go

@@ -177,6 +177,9 @@ func (e *edge) finishIncoming(req pipe.Sender) {
 
 // updateIncoming updates the current value of incoming pipe request
 func (e *edge) updateIncoming(req pipe.Sender) {
+	if debugScheduler {
+		logrus.Debugf("updateIncoming %s %#v desired=%s", e.edge.Vertex.Name(), e.edgeState, req.Request().Payload.(*edgeRequest).desiredState)
+	}
 	req.Update(&e.edgeState)
 }
 

solver/internal/pipe/pipe.go

@@ -11,23 +11,27 @@ import (
 type channel struct {
 	OnSendCompletion func()
 	value            atomic.Value
-	lastValue        interface{}
+	lastValue        *wrappedValue
+}
+
+type wrappedValue struct {
+	value interface{}
 }
 
 func (c *channel) Send(v interface{}) {
-	c.value.Store(v)
+	c.value.Store(&wrappedValue{value: v})
 	if c.OnSendCompletion != nil {
 		c.OnSendCompletion()
 	}
 }
 
 func (c *channel) Receive() (interface{}, bool) {
 	v := c.value.Load()
-	if c.lastValue == v {
+	if v == nil || v.(*wrappedValue) == c.lastValue {
 		return nil, false
 	}
-	c.lastValue = v
-	return v, true
+	c.lastValue = v.(*wrappedValue)
+	return v.(*wrappedValue).value, true
 }
 
 type Pipe struct {

solver/llbsolver/bridge.go

@@ -54,7 +54,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
 			}
 		}
 		if prevCm, ok := b.cms[cmId]; !ok {
-			func(cmId string) {
+			func(cmId string, im gw.CacheOptionsEntry) {
 				cm = newLazyCacheManager(cmId, func() (solver.CacheManager, error) {
 					var cmNew solver.CacheManager
 					if err := inVertexContext(b.builder.Context(ctx), "importing cache manifest from "+cmId, "", func(ctx context.Context) error {
@@ -74,7 +74,7 @@ func (b *llbBridge) Solve(ctx context.Context, req frontend.SolveRequest) (res *
 					}
 					return cmNew, nil
 				})
-			}(cmId)
+			}(cmId, im)
 			b.cms[cmId] = cm
 		} else {
 			cm = prevCm