Merge pull request #869 from zeripath/graph-generation-2

plumbing: commitgraph, Add generation v2 support

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/gliderlabs/ssh v0.3.5
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376
 	github.com/go-git/go-billy/v5 v5.5.0
-	github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389
+	github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da
 	github.com/google/go-cmp v0.5.9
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99

go.sum

@@ -32,8 +32,8 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D
 github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
 github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
 github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389 h1:AlfdJ8f+G+4a4fXeHmAlKfyR3Yup4sVGCXlh+e+TrE8=
-github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231007200033-41cf6f1b6389/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
+github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=

plumbing/format/commitgraph/v2/chunk.go

@@ -3,7 +3,7 @@ package v2
 import "bytes"
 
 const (
-	chunkSigLen    = 4 // Length of a chunk signature
+	szChunkSig     = 4 // Length of a chunk signature
 	chunkSigOffset = 4 // Offset of each chunk signature in chunkSignatures
 )
 
@@ -28,14 +28,15 @@ const (
 	BaseGraphsListChunk                          // "BASE"
 	ZeroChunk                                    // "\000\000\000\000"
 )
+const lenChunks = int(ZeroChunk) // ZeroChunk is not a valid chunk type, but it is used to determine the length of the chunk type list.
 
 // Signature returns the byte signature for the chunk type.
 func (ct ChunkType) Signature() []byte {
 	if ct >= BaseGraphsListChunk || ct < 0 { // not a valid chunk type just return ZeroChunk
-		return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+chunkSigLen]
+		return chunkSignatures[ZeroChunk*chunkSigOffset : ZeroChunk*chunkSigOffset+szChunkSig]
 	}
 
-	return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+chunkSigLen]
+	return chunkSignatures[ct*chunkSigOffset : ct*chunkSigOffset+szChunkSig]
 }
 
 // ChunkTypeFromBytes returns the chunk type for the given byte signature.

plumbing/format/commitgraph/v2/commitgraph.go

@@ -2,6 +2,7 @@ package v2
 
 import (
 	"io"
+	"math"
 	"time"
 
 	"github.com/go-git/go-git/v5/plumbing"
@@ -19,10 +20,22 @@ type CommitData struct {
 	// Generation number is the pre-computed generation in the commit graph
 	// or zero if not available.
 	Generation uint64
+	// GenerationV2 stores the corrected commit date for the commits
+	// It combines the contents of the GDA2 and GDO2 sections of the commit-graph
+	// with the commit time portion of the CDAT section.
+	GenerationV2 uint64
 	// When is the timestamp of the commit.
 	When time.Time
 }
 
+// GenerationV2Data returns the corrected commit date for the commits
+func (c *CommitData) GenerationV2Data() uint64 {
+	if c.GenerationV2 == 0 || c.GenerationV2 == math.MaxUint64 {
+		return 0
+	}
+	return c.GenerationV2 - uint64(c.When.Unix())
+}
+
 // Index represents a representation of commit graph that allows indexed
 // access to the nodes using commit object hash
 type Index interface {
@@ -35,6 +48,10 @@ type Index interface {
 	GetCommitDataByIndex(i uint32) (*CommitData, error)
 	// Hashes returns all the hashes that are available in the index
 	Hashes() []plumbing.Hash
+	// HasGenerationV2 returns true if the commit graph has the corrected commit date data
+	HasGenerationV2() bool
+	// MaximumNumberOfHashes returns the maximum number of hashes within the index
+	MaximumNumberOfHashes() uint32
 
 	io.Closer
 }

plumbing/format/commitgraph/v2/commitgraph_test.go

@@ -7,7 +7,11 @@ import (
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/util"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/cache"
 	commitgraph "github.com/go-git/go-git/v5/plumbing/format/commitgraph/v2"
+	"github.com/go-git/go-git/v5/plumbing/format/packfile"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/storage/filesystem"
 
 	fixtures "github.com/go-git/go-git-fixtures/v4"
 	. "gopkg.in/check.v1"
@@ -76,6 +80,37 @@ func testDecodeHelper(c *C, index commitgraph.Index) {
 	c.Assert(hashes[10].String(), Equals, "e713b52d7e13807e87a002e812041f248db3f643")
 }
 
+func (s *CommitgraphSuite) TestDecodeMultiChain(c *C) {
+	fixtures.ByTag("commit-graph-chain-2").Test(c, func(f *fixtures.Fixture) {
+		dotgit := f.DotGit()
+		index, err := commitgraph.OpenChainOrFileIndex(dotgit)
+		c.Assert(err, IsNil)
+		defer index.Close()
+		storer := filesystem.NewStorage(f.DotGit(), cache.NewObjectLRUDefault())
+		p := f.Packfile()
+		defer p.Close()
+		packfile.UpdateObjectStorage(storer, p)
+
+		for idx, hash := range index.Hashes() {
+			idx2, err := index.GetIndexByHash(hash)
+			c.Assert(err, IsNil)
+			c.Assert(idx2, Equals, uint32(idx))
+			hash2, err := index.GetHashByIndex(idx2)
+			c.Assert(err, IsNil)
+			c.Assert(hash2.String(), Equals, hash.String())
+
+			commitData, err := index.GetCommitDataByIndex(uint32(idx))
+			c.Assert(err, IsNil)
+			commit, err := object.GetCommit(storer, hash)
+			c.Assert(err, IsNil)
+
+			for i, parent := range commit.ParentHashes {
+				c.Assert(hash.String()+":"+parent.String(), Equals, hash.String()+":"+commitData.ParentHashes[i].String())
+			}
+		}
+	})
+}
+
 func (s *CommitgraphSuite) TestDecode(c *C) {
 	fixtures.ByTag("commit-graph").Test(c, func(f *fixtures.Fixture) {
 		dotgit := f.DotGit()

plumbing/format/commitgraph/v2/encoder.go

@@ -3,6 +3,7 @@ package v2
 import (
 	"crypto"
 	"io"
+	"math"
 
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/hash"
@@ -28,13 +29,21 @@ func (e *Encoder) Encode(idx Index) error {
 	hashes := idx.Hashes()
 
 	// Sort the inout and prepare helper structures we'll need for encoding
-	hashToIndex, fanout, extraEdgesCount := e.prepare(idx, hashes)
+	hashToIndex, fanout, extraEdgesCount, generationV2OverflowCount := e.prepare(idx, hashes)
 
 	chunkSignatures := [][]byte{OIDFanoutChunk.Signature(), OIDLookupChunk.Signature(), CommitDataChunk.Signature()}
-	chunkSizes := []uint64{4 * 256, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + commitDataSize)}
+	chunkSizes := []uint64{szUint32 * lenFanout, uint64(len(hashes)) * hash.Size, uint64(len(hashes)) * (hash.Size + szCommitData)}
 	if extraEdgesCount > 0 {
 		chunkSignatures = append(chunkSignatures, ExtraEdgeListChunk.Signature())
-		chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*4)
+		chunkSizes = append(chunkSizes, uint64(extraEdgesCount)*szUint32)
+	}
+	if idx.HasGenerationV2() {
+		chunkSignatures = append(chunkSignatures, GenerationDataChunk.Signature())
+		chunkSizes = append(chunkSizes, uint64(len(hashes))*szUint32)
+		if generationV2OverflowCount > 0 {
+			chunkSignatures = append(chunkSignatures, GenerationDataOverflowChunk.Signature())
+			chunkSizes = append(chunkSizes, uint64(generationV2OverflowCount)*szUint64)
+		}
 	}
 
 	if err := e.encodeFileHeader(len(chunkSignatures)); err != nil {
@@ -49,38 +58,52 @@ func (e *Encoder) Encode(idx Index) error {
 	if err := e.encodeOidLookup(hashes); err != nil {
 		return err
 	}
-	if extraEdges, err := e.encodeCommitData(hashes, hashToIndex, idx); err == nil {
-		if err = e.encodeExtraEdges(extraEdges); err != nil {
+
+	extraEdges, generationV2Data, err := e.encodeCommitData(hashes, hashToIndex, idx)
+	if err != nil {
+		return err
+	}
+	if err = e.encodeExtraEdges(extraEdges); err != nil {
+		return err
+	}
+	if idx.HasGenerationV2() {
+		overflows, err := e.encodeGenerationV2Data(generationV2Data)
+		if err != nil {
+			return err
+		}
+		if err = e.encodeGenerationV2Overflow(overflows); err != nil {
 			return err
 		}
-	} else {
-		return err
 	}
 
 	return e.encodeChecksum()
 }
 
-func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount uint32) {
+func (e *Encoder) prepare(idx Index, hashes []plumbing.Hash) (hashToIndex map[plumbing.Hash]uint32, fanout []uint32, extraEdgesCount uint32, generationV2OverflowCount uint32) {
 	// Sort the hashes and build our index
 	plumbing.HashesSort(hashes)
 	hashToIndex = make(map[plumbing.Hash]uint32)
-	fanout = make([]uint32, 256)
+	fanout = make([]uint32, lenFanout)
 	for i, hash := range hashes {
 		hashToIndex[hash] = uint32(i)
 		fanout[hash[0]]++
 	}
 
 	// Convert the fanout to cumulative values
-	for i := 1; i <= 0xff; i++ {
+	for i := 1; i < lenFanout; i++ {
 		fanout[i] += fanout[i-1]
 	}
 
+	hasGenerationV2 := idx.HasGenerationV2()
+
 	// Find out if we will need extra edge table
 	for i := 0; i < len(hashes); i++ {
 		v, _ := idx.GetCommitDataByIndex(uint32(i))
 		if len(v.ParentHashes) > 2 {
 			extraEdgesCount += uint32(len(v.ParentHashes) - 1)
-			break
+		}
+		if hasGenerationV2 && v.GenerationV2Data() > math.MaxUint32 {
+			generationV2OverflowCount++
 		}
 	}
 
@@ -100,7 +123,7 @@ func (e *Encoder) encodeFileHeader(chunkCount int) (err error) {
 
 func (e *Encoder) encodeChunkHeaders(chunkSignatures [][]byte, chunkSizes []uint64) (err error) {
 	// 8 bytes of file header, 12 bytes for each chunk header and 12 byte for terminator
-	offset := uint64(8 + len(chunkSignatures)*12 + 12)
+	offset := uint64(szSignature + szHeader + (len(chunkSignatures)+1)*(szChunkSig+szUint64))
 	for i, signature := range chunkSignatures {
 		if _, err = e.Write(signature); err == nil {
 			err = binary.WriteUint64(e, offset)
@@ -134,7 +157,10 @@ func (e *Encoder) encodeOidLookup(hashes []plumbing.Hash) (err error) {
 	return
 }
 
-func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, err error) {
+func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumbing.Hash]uint32, idx Index) (extraEdges []uint32, generationV2Data []uint64, err error) {
+	if idx.HasGenerationV2() {
+		generationV2Data = make([]uint64, 0, len(hashes))
+	}
 	for _, hash := range hashes {
 		origIndex, _ := idx.GetIndexByHash(hash)
 		commitData, _ := idx.GetCommitDataByIndex(origIndex)
@@ -173,6 +199,9 @@ func (e *Encoder) encodeCommitData(hashes []plumbing.Hash, hashToIndex map[plumb
 		if err = binary.WriteUint64(e, unixTime); err != nil {
 			return
 		}
+		if generationV2Data != nil {
+			generationV2Data = append(generationV2Data, commitData.GenerationV2Data())
+		}
 	}
 	return
 }
@@ -186,6 +215,35 @@ func (e *Encoder) encodeExtraEdges(extraEdges []uint32) (err error) {
 	return
 }
 
+func (e *Encoder) encodeGenerationV2Data(generationV2Data []uint64) (overflows []uint64, err error) {
+	head := 0
+	for _, data := range generationV2Data {
+		if data >= 0x80000000 {
+			// overflow
+			if err = binary.WriteUint32(e, uint32(head)|0x80000000); err != nil {
+				return nil, err
+			}
+			generationV2Data[head] = data
+			head++
+			continue
+		}
+		if err = binary.WriteUint32(e, uint32(data)); err != nil {
+			return nil, err
+		}
+	}
+
+	return generationV2Data[:head], nil
+}
+
+func (e *Encoder) encodeGenerationV2Overflow(overflows []uint64) (err error) {
+	for _, overflow := range overflows {
+		if err = binary.WriteUint64(e, overflow); err != nil {
+			return
+		}
+	}
+	return
+}
+
 func (e *Encoder) encodeChecksum() error {
 	_, err := e.Write(e.hash.Sum(nil)[:hash.Size])
 	return err