diff --git a/plumbing/object/rename.go b/plumbing/object/rename.go
index 7fed72c2f..0394613ff 100644
--- a/plumbing/object/rename.go
+++ b/plumbing/object/rename.go
@@ -403,10 +403,16 @@ func min(a, b int) int {
 	return b
 }
 
+const maxMatrixSize = 10000
+
 func buildSimilarityMatrix(srcs, dsts []*Change, renameScore int) (similarityMatrix, error) {
 	// Allocate for the worst-case scenario where every pair has a score
 	// that we need to consider. We might not need that many.
-	matrix := make(similarityMatrix, 0, len(srcs)*len(dsts))
+	matrixSize := len(srcs) * len(dsts)
+	if matrixSize > maxMatrixSize {
+		matrixSize = maxMatrixSize
+	}
+	matrix := make(similarityMatrix, 0, matrixSize)
 	srcSizes := make([]int64, len(srcs))
 	dstSizes := make([]int64, len(dsts))
 	dstTooLarge := make(map[int]bool)