Merge pull request #2918 from tonistiigi/source-date-epoch

Add support for SOURCE_DATE_EPOCH

control/control.go

@@ -12,6 +12,7 @@ import (
 	"github.com/moby/buildkit/client"
 	controlgateway "github.com/moby/buildkit/control/gateway"
 	"github.com/moby/buildkit/exporter"
+	"github.com/moby/buildkit/exporter/util/epoch"
 	"github.com/moby/buildkit/frontend"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/grpchijack"
@@ -267,6 +268,17 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 	if err != nil {
 		return nil, err
 	}
+
+	// if SOURCE_DATE_EPOCH is set, enable it for the exporter
+	if epochVal, ok := req.FrontendAttrs["build-arg:SOURCE_DATE_EPOCH"]; ok {
+		if _, ok := req.ExporterAttrs[epoch.KeySourceDateEpoch]; !ok {
+			if req.ExporterAttrs == nil {
+				req.ExporterAttrs = make(map[string]string)
+			}
+			req.ExporterAttrs[epoch.KeySourceDateEpoch] = epochVal
+		}
+	}
+
 	if req.Exporter != "" {
 		exp, err := w.Exporter(req.Exporter, c.opt.SessionManager)
 		if err != nil {

exporter/containerimage/exptypes/types.go

@@ -13,6 +13,7 @@ const (
 	ExporterInlineCache          = "containerimage.inlinecache"
 	ExporterBuildInfo            = "containerimage.buildinfo"
 	ExporterPlatformsKey         = "refs.platforms"
+	ExporterEpochKey             = "source.date.epoch"
 )
 
 type Platforms struct {

exporter/containerimage/opts.go

@@ -2,8 +2,10 @@ package containerimage
 
 import (
 	"strconv"
+	"time"
 
 	cacheconfig "github.com/moby/buildkit/cache/config"
+	"github.com/moby/buildkit/exporter/util/epoch"
 	"github.com/moby/buildkit/util/compression"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -31,6 +33,7 @@ type ImageCommitOpts struct {
 	BuildInfo      bool
 	BuildInfoAttrs bool
 	Annotations    AnnotationsGroup
+	Epoch          *time.Time
 }
 
 func (c *ImageCommitOpts) Load(opt map[string]string) (map[string]string, error) {
@@ -44,6 +47,11 @@ func (c *ImageCommitOpts) Load(opt map[string]string) (map[string]string, error)
 	}
 	opt = toStringMap(optb)
 
+	c.Epoch, opt, err = epoch.ParseAttr(opt)
+	if err != nil {
+		return nil, err
+	}
+
 	for k, v := range opt {
 		var err error
 		switch k {

exporter/containerimage/writer.go

@@ -19,6 +19,7 @@ import (
 	cacheconfig "github.com/moby/buildkit/cache/config"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/exporter/util/epoch"
 	gatewaypb "github.com/moby/buildkit/frontend/gateway/pb"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/snapshot"
@@ -71,6 +72,14 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
 		}
 	}
 
+	if opts.Epoch == nil {
+		if tm, ok, err := epoch.ParseSource(inp); err != nil {
+			return nil, err
+		} else if ok {
+			opts.Epoch = tm
+		}
+	}
+
 	if len(inp.Refs) == 0 {
 		remotes, err := ic.exportLayers(ctx, opts.RefCfg, session.NewGroup(sessionID), inp.Ref)
 		if err != nil {
@@ -86,7 +95,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
 			}
 		}
 
-		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], opts.Annotations.Platform(nil), inp.Metadata[exptypes.ExporterInlineCache], dtbi)
+		mfstDesc, configDesc, err := ic.commitDistributionManifest(ctx, opts, inp.Ref, inp.Metadata[exptypes.ExporterImageConfigKey], &remotes[0], opts.Annotations.Platform(nil), inp.Metadata[exptypes.ExporterInlineCache], dtbi, opts.Epoch)
 		if err != nil {
 			return nil, err
 		}
@@ -166,7 +175,7 @@ func (ic *ImageWriter) Commit(ctx context.Context, inp *exporter.Source, session
 			}
 		}
 
-		desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, &remotes[remotesMap[p.ID]], opts.Annotations.Platform(&p.Platform), inlineCache, dtbi)
+		desc, _, err := ic.commitDistributionManifest(ctx, opts, r, config, &remotes[remotesMap[p.ID]], opts.Annotations.Platform(&p.Platform), inlineCache, dtbi, opts.Epoch)
 		if err != nil {
 			return nil, err
 		}
@@ -353,7 +362,7 @@ func (ic *ImageWriter) extractAttestations(ctx context.Context, opts *ImageCommi
 	return statements, nil
 }
 
-func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *ImageCommitOpts, ref cache.ImmutableRef, config []byte, remote *solver.Remote, annotations *Annotations, inlineCache []byte, buildInfo []byte) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) {
+func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *ImageCommitOpts, ref cache.ImmutableRef, config []byte, remote *solver.Remote, annotations *Annotations, inlineCache []byte, buildInfo []byte, epoch *time.Time) (*ocispecs.Descriptor, *ocispecs.Descriptor, error) {
 	if len(config) == 0 {
 		var err error
 		config, err = defaultImageConfig()
@@ -375,7 +384,7 @@ func (ic *ImageWriter) commitDistributionManifest(ctx context.Context, opts *Ima
 
 	remote, history = normalizeLayersAndHistory(ctx, remote, history, ref, opts.OCITypes)
 
-	config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, buildInfo)
+	config, err = patchImageConfig(config, remote.Descriptors, history, inlineCache, buildInfo, epoch)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -619,7 +628,7 @@ func parseHistoryFromConfig(dt []byte) ([]ocispecs.History, error) {
 	return config.History, nil
 }
 
-func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, buildInfo []byte) ([]byte, error) {
+func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs.History, cache []byte, buildInfo []byte, epoch *time.Time) ([]byte, error) {
 	m := map[string]json.RawMessage{}
 	if err := json.Unmarshal(dt, &m); err != nil {
 		return nil, errors.Wrap(err, "failed to parse image config for patch")
@@ -636,12 +645,35 @@ func patchImageConfig(dt []byte, descs []ocispecs.Descriptor, history []ocispecs
 	}
 	m["rootfs"] = dt
 
+	if epoch != nil {
+		for i, h := range history {
+			if h.Created == nil || h.Created.After(*epoch) {
+				history[i].Created = epoch
+			}
+		}
+	}
+
 	dt, err = json.Marshal(history)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to marshal history")
 	}
 	m["history"] = dt
 
+	// if epoch is set then clamp creation time
+	if v, ok := m["created"]; ok && epoch != nil {
+		var tm time.Time
+		if err := json.Unmarshal(v, &tm); err != nil {
+			return nil, errors.Wrapf(err, "failed to unmarshal creation time %q", m["created"])
+		}
+		if tm.After(*epoch) {
+			dt, err = json.Marshal(&epoch)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to marshal creation time")
+			}
+			m["created"] = dt
+		}
+	}
+
 	if _, ok := m["created"]; !ok {
 		var tm *time.Time
 		for _, h := range history {

exporter/local/export.go

@@ -11,6 +11,7 @@ import (
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/exporter/util/epoch"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
@@ -37,11 +38,17 @@ func New(opt Opt) (exporter.Exporter, error) {
 }
 
 func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
-	return &localExporterInstance{localExporter: e}, nil
+	tm, _, err := epoch.ParseAttr(opt)
+	if err != nil {
+		return nil, err
+	}
+
+	return &localExporterInstance{localExporter: e, epoch: tm}, nil
 }
 
 type localExporterInstance struct {
 	*localExporter
+	epoch *time.Time
 }
 
 func (e *localExporterInstance) Name() string {
@@ -56,6 +63,14 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 	timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
 	defer cancel()
 
+	if e.epoch == nil {
+		if tm, ok, err := epoch.ParseSource(inp); err != nil {
+			return nil, err
+		} else if ok {
+			e.epoch = tm
+		}
+	}
+
 	caller, err := e.opt.SessionManager.Get(timeoutCtx, sessionID, false)
 	if err != nil {
 		return nil, err
@@ -105,9 +120,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 			}
 
 			walkOpt := &fsutil.WalkOpt{}
+			var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
 
 			if idmap != nil {
-				walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+				idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult {
 					uid, gid, err := idmap.ToContainer(idtools.Identity{
 						UID: int(st.Uid),
 						GID: int(st.Gid),
@@ -121,14 +137,29 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 				}
 			}
 
+			walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+				res := fsutil.MapResultKeep
+				if idMapFunc != nil {
+					res = idMapFunc(p, st)
+				}
+				if e.epoch != nil {
+					st.ModTime = e.epoch.UnixNano()
+				}
+				return res
+			}
+
 			fs := fsutil.NewFS(src, walkOpt)
 			lbl := "copying files"
 			if isMap {
 				lbl += " " + k
-				fs, err = fsutil.SubDirFS([]fsutil.Dir{{FS: fs, Stat: fstypes.Stat{
+				st := fstypes.Stat{
 					Mode: uint32(os.ModeDir | 0755),
 					Path: strings.Replace(k, "/", "_", -1),
-				}}})
+				}
+				if e.epoch != nil {
+					st.ModTime = e.epoch.UnixNano()
+				}
+				fs, err = fsutil.SubDirFS([]fsutil.Dir{{FS: fs, Stat: st}})
 				if err != nil {
 					return err
 				}

exporter/tar/export.go

@@ -12,6 +12,7 @@ import (
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/exporter"
 	"github.com/moby/buildkit/exporter/containerimage/exptypes"
+	"github.com/moby/buildkit/exporter/util/epoch"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/filesync"
 	"github.com/moby/buildkit/snapshot"
@@ -45,6 +46,12 @@ func New(opt Opt) (exporter.Exporter, error) {
 func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exporter.ExporterInstance, error) {
 	li := &localExporterInstance{localExporter: e}
 
+	tm, _, err := epoch.ParseAttr(opt)
+	if err != nil {
+		return nil, err
+	}
+	li.epoch = tm
+
 	v, ok := opt[preferNondistLayersKey]
 	if ok {
 		b, err := strconv.ParseBool(v)
@@ -60,6 +67,7 @@ func (e *localExporter) Resolve(ctx context.Context, opt map[string]string) (exp
 type localExporterInstance struct {
 	*localExporter
 	preferNonDist bool
+	epoch         *time.Time
 }
 
 func (e *localExporterInstance) Name() string {
@@ -79,6 +87,14 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 		}
 	}()
 
+	if e.epoch == nil {
+		if tm, ok, err := epoch.ParseSource(inp); err != nil {
+			return nil, err
+		} else if ok {
+			e.epoch = tm
+		}
+	}
+
 	getDir := func(ctx context.Context, k string, ref cache.ImmutableRef) (*fsutil.Dir, error) {
 		var src string
 		var err error
@@ -108,9 +124,10 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 		}
 
 		walkOpt := &fsutil.WalkOpt{}
+		var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
 
 		if idmap != nil {
-			walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+			idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult {
 				uid, gid, err := idmap.ToContainer(idtools.Identity{
 					UID: int(st.Uid),
 					GID: int(st.Gid),
@@ -124,12 +141,28 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 			}
 		}
 
+		walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+			res := fsutil.MapResultKeep
+			if idMapFunc != nil {
+				res = idMapFunc(p, st)
+			}
+			if e.epoch != nil {
+				st.ModTime = e.epoch.UnixNano()
+			}
+			return res
+		}
+
+		st := fstypes.Stat{
+			Mode: uint32(os.ModeDir | 0755),
+			Path: strings.Replace(k, "/", "_", -1),
+		}
+		if e.epoch != nil {
+			st.ModTime = e.epoch.UnixNano()
+		}
+
 		return &fsutil.Dir{
-			FS: fsutil.NewFS(src, walkOpt),
-			Stat: fstypes.Stat{
-				Mode: uint32(os.ModeDir | 0755),
-				Path: strings.Replace(k, "/", "_", -1),
-			},
+			FS:   fsutil.NewFS(src, walkOpt),
+			Stat: st,
 		}, nil
 	}