From a83fbb989881e6c6ef7667dd4aa823192c1433c7 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Thu, 8 Sep 2022 15:54:40 +0900 Subject: [PATCH] dockerfile: implement `ADD --checksum= ` e.g., ADD --checksum=sha256:24454f830cdb571e2c4ad15481119c43b3cafd48dd869a9b2945d1036d1dc68d https://mirrors.edge.kernel.org/pub/linux/kernel/Historic/linux-0.01.tar.gz / Fix issue 975 Signed-off-by: Akihiro Suda --- .golangci.yml | 1 + frontend/dockerfile/dockerfile2llb/convert.go | 26 ++- .../dockerfile2llb/convert_addchecksum.go | 6 + .../dockerfile2llb/convert_noaddchecksum.go | 6 + .../dockerfile/dockerfile_addchecksum_test.go | 155 ++++++++++++++++++ frontend/dockerfile/docs/reference.md | 16 +- frontend/dockerfile/instructions/commands.go | 2 + frontend/dockerfile/instructions/parse.go | 11 ++ frontend/dockerfile/release/labs/tags | 2 +- 9 files changed, 221 insertions(+), 4 deletions(-) create mode 100644 frontend/dockerfile/dockerfile2llb/convert_addchecksum.go create mode 100644 frontend/dockerfile/dockerfile2llb/convert_noaddchecksum.go create mode 100644 frontend/dockerfile/dockerfile_addchecksum_test.go diff --git a/.golangci.yml b/.golangci.yml index 17848dc286ab..ce39a8968e65 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -8,6 +8,7 @@ run: build-tags: - dfrunsecurity - dfaddgit + - dfaddchecksum linters: enable: diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index c5ace3e220ad..427d13341569 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -35,6 +35,7 @@ import ( "github.com/moby/buildkit/util/suggest" "github.com/moby/buildkit/util/system" "github.com/moby/sys/signal" + digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -727,6 +728,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error { chmod: c.Chmod, link: c.Link, keepGitDir: c.KeepGitDir, + checksum: c.Checksum, location: c.Location(), opt: opt, }) @@ -1073,6 +1075,21 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } } + if cfg.checksum != "" { + if !cfg.isAddCommand { + return errors.New("checksum can't be specified for COPY") + } + if !addChecksumEnabled { + return errors.New("instruction 'ADD --checksum=' requires the labs channel") + } + if len(cfg.params.SourcePaths) != 1 { + return errors.New("checksum can't be specified for multiple sources") + } + if !isHTTPSource(cfg.params.SourcePaths[0]) { + return errors.New("checksum can't be specified for non-HTTP sources") + } + } + commitMessage := bytes.NewBufferString("") if cfg.isAddCommand { commitMessage.WriteString("ADD") @@ -1111,7 +1128,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } else { a = a.Copy(st, "/", dest, opts...) } - } else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") { + } else if isHTTPSource(src) { if !cfg.isAddCommand { return errors.New("source can't be a URL for COPY") } @@ -1129,7 +1146,7 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { } } - st := llb.HTTP(src, llb.Filename(f), dfCmd(cfg.params)) + st := llb.HTTP(src, llb.Filename(f), llb.Checksum(cfg.checksum), dfCmd(cfg.params)) opts := append([]llb.CopyOption{&llb.CopyInfo{ Mode: mode, @@ -1235,6 +1252,7 @@ type copyConfig struct { chmod string link bool keepGitDir bool + checksum digest.Digest location []parser.Range opt dispatchOpt } @@ -1752,3 +1770,7 @@ func clampTimes(img Image, tm *time.Time) Image { } return img } + +func isHTTPSource(src string) bool { + return strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") +} diff --git a/frontend/dockerfile/dockerfile2llb/convert_addchecksum.go b/frontend/dockerfile/dockerfile2llb/convert_addchecksum.go new file mode 100644 index 000000000000..4506baeb8ba8 --- /dev/null +++ b/frontend/dockerfile/dockerfile2llb/convert_addchecksum.go @@ -0,0 +1,6 @@ +//go:build dfaddchecksum +// +build dfaddchecksum + +package dockerfile2llb + +const addChecksumEnabled = true diff --git a/frontend/dockerfile/dockerfile2llb/convert_noaddchecksum.go b/frontend/dockerfile/dockerfile2llb/convert_noaddchecksum.go new file mode 100644 index 000000000000..8de035297c1b --- /dev/null +++ b/frontend/dockerfile/dockerfile2llb/convert_noaddchecksum.go @@ -0,0 +1,6 @@ +//go:build !dfaddchecksum +// +build !dfaddchecksum + +package dockerfile2llb + +const addChecksumEnabled = false diff --git a/frontend/dockerfile/dockerfile_addchecksum_test.go b/frontend/dockerfile/dockerfile_addchecksum_test.go new file mode 100644 index 000000000000..d205414e7733 --- /dev/null +++ b/frontend/dockerfile/dockerfile_addchecksum_test.go @@ -0,0 +1,155 @@ +//go:build dfaddchecksum +// +build dfaddchecksum + +package dockerfile + +import ( + "fmt" + "testing" + + "github.com/containerd/continuity/fs/fstest" + "github.com/moby/buildkit/client" + "github.com/moby/buildkit/frontend/dockerfile/builder" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/util/testutil/httpserver" + "github.com/moby/buildkit/util/testutil/integration" + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +var addChecksumTests = integration.TestFuncs( + testAddChecksum, +) + +func init() { + allTests = append(allTests, addChecksumTests...) +} + +func testAddChecksum(t *testing.T, sb integration.Sandbox) { + f := getFrontend(t, sb) + f.RequiresBuildctl(t) + + resp := httpserver.Response{ + Etag: identity.NewID(), + Content: []byte("content1"), + } + server := httpserver.NewTestServer(map[string]httpserver.Response{ + "/foo": resp, + }) + defer server.Close() + + c, err := client.New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + t.Run("Valid", func(t *testing.T) { + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=%s %s /tmp/foo +`, digest.FromBytes(resp.Content).String(), server.URL+"/foo")) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.NoError(t, err) + }) + t.Run("DigestMismatch", func(t *testing.T) { + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=%s %s /tmp/foo +`, digest.FromBytes(nil).String(), server.URL+"/foo")) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.Error(t, err, "digest mismatch") + }) + t.Run("DigestWithKnownButUnsupportedAlgoName", func(t *testing.T) { + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=md5:7e55db001d319a94b0b713529a756623 %s /tmp/foo +`, server.URL+"/foo")) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.Error(t, err, "unsupported digest algorithm") + }) + t.Run("DigestWithUnknownAlgoName", func(t *testing.T) { + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=unknown:%s %s /tmp/foo +`, digest.FromBytes(resp.Content).Encoded(), server.URL+"/foo")) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.Error(t, err, "unsupported digest algorithm") + }) + t.Run("DigestWithoutAlgoName", func(t *testing.T) { + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=%s %s /tmp/foo +`, digest.FromBytes(resp.Content).Encoded(), server.URL+"/foo")) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.Error(t, err, "invalid checksum digest format") + }) + t.Run("NonHTTPSource", func(t *testing.T) { + foo := []byte("local file") + dockerfile := []byte(fmt.Sprintf(` +FROM scratch +ADD --checksum=%s foo /tmp/foo +`, digest.FromBytes(foo).String())) + dir, err := integration.Tmpdir( + t, + fstest.CreateFile("foo", foo, 0600), + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + }, nil) + require.Error(t, err, "checksum can't be specified for non-HTTP sources") + }) +} diff --git a/frontend/dockerfile/docs/reference.md b/frontend/dockerfile/docs/reference.md index 4508d561d029..1a8087ebab9f 100644 --- a/frontend/dockerfile/docs/reference.md +++ b/frontend/dockerfile/docs/reference.md @@ -1336,7 +1336,7 @@ RUN apt-get update && apt-get install -y ... ADD has two forms: ```dockerfile -ADD [--chown=:] ... +ADD [--chown=:] [--checksum=] ... ADD [--chown=:] ["",... ""] ``` @@ -1507,6 +1507,20 @@ guide – Leverage build cache](https://docs.docker.com/develop/develop-images/d - If `` doesn't exist, it is created along with all missing directories in its path. +### Verifying a remote file checksum `ADD --checksum= ` +> **Note** +> +> Available in [`docker/dockerfile-upstream:master-labs`](#syntax). +> Will be included in `docker/dockerfile:1.5-labs`. + +The checksum of a remote file can be verified with the `--checksum` flag: + +```dockerfile +ADD --checksum=sha256:24454f830cdb571e2c4ad15481119c43b3cafd48dd869a9b2945d1036d1dc68d https://mirrors.edge.kernel.org/pub/linux/kernel/Historic/linux-0.01.tar.gz / +``` + +The `--checksum` flag only supports HTTP sources currently. + ### Adding a git repository `ADD ` > **Note** diff --git a/frontend/dockerfile/instructions/commands.go b/frontend/dockerfile/instructions/commands.go index cbb111a235ed..a82938afb689 100644 --- a/frontend/dockerfile/instructions/commands.go +++ b/frontend/dockerfile/instructions/commands.go @@ -6,6 +6,7 @@ import ( "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/strslice" "github.com/moby/buildkit/frontend/dockerfile/parser" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -226,6 +227,7 @@ type AddCommand struct { Chmod string Link bool KeepGitDir bool // whether to keep .git dir, only meaningful for git sources + Checksum digest.Digest } // Expand variables diff --git a/frontend/dockerfile/instructions/parse.go b/frontend/dockerfile/instructions/parse.go index 9694592267e7..29ee2d220f47 100644 --- a/frontend/dockerfile/instructions/parse.go +++ b/frontend/dockerfile/instructions/parse.go @@ -13,6 +13,7 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/command" "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/moby/buildkit/util/suggest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -282,6 +283,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) { flChmod := req.flags.AddString("chmod", "") flLink := req.flags.AddBool("link", false) flKeepGitDir := req.flags.AddBool("keep-git-dir", false) + flChecksum := req.flags.AddString("checksum", "") if err := req.flags.Parse(); err != nil { return nil, err } @@ -291,6 +293,14 @@ func parseAdd(req parseRequest) (*AddCommand, error) { return nil, err } + var checksum digest.Digest + if flChecksum.Value != "" { + checksum, err = digest.Parse(flChecksum.Value) + if err != nil { + return nil, err + } + } + return &AddCommand{ withNameAndCode: newWithNameAndCode(req), SourcesAndDest: *sourcesAndDest, @@ -298,6 +308,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) { Chmod: flChmod.Value, Link: flLink.Value == "true", KeepGitDir: flKeepGitDir.Value == "true", + Checksum: checksum, }, nil } diff --git a/frontend/dockerfile/release/labs/tags b/frontend/dockerfile/release/labs/tags index 7f2ea1ceef88..71a17cb3926a 100644 --- a/frontend/dockerfile/release/labs/tags +++ b/frontend/dockerfile/release/labs/tags @@ -1 +1 @@ -dfrunsecurity dfaddgit +dfrunsecurity dfaddgit dfaddchecksum