.github/workflows/benchmark.yml

@@ -0,0 +1,32 @@
+name: Run benchmarks
+on:
+  pull_request:
+    branches: [master]
+
+permissions:
+  contents: read
+
+jobs:
+  benchmark:
+    runs-on: ubuntu-latest
+    steps:
+    # Base for comparison is master branch.
+    - name: Checkout code
+      uses: actions/checkout@v4.2.1
+      with:
+        ref: master
+    - name: Install Go
+      uses: actions/setup-go@v5.0.2
+      with:
+        go-version-file: 'go.mod'
+
+    # 30 runs with 100ms benchtime seems to result in acceptable p-values
+    # When I tried with count=10, it would be unreliable because of the actions
+    # runner is in a shared environment and CPU and mem would be affected by others. (or so I think)
+    - run: go test -run=none -bench=. -count=30 -benchtime=100ms -timeout=20m > /tmp/prev
+    - name: Checkout code
+      uses: actions/checkout@v4.2.1
+    - run: go test -run=none -bench=. -count=30 -benchtime=100ms -timeout=20m > /tmp/curr
+
+    - run: go install golang.org/x/perf/cmd/benchstat@latest
+    - run: benchstat /tmp/prev /tmp/curr

.github/workflows/codeql.yml

@@ -20,13 +20,13 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.2.1
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3.25.14
+        uses: github/codeql-action/init@v3.26.12
         with:
           languages: go
           queries: security-and-quality
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3.25.14
+        uses: github/codeql-action/analyze@v3.26.12

.github/workflows/go.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4.1.7
+      uses: actions/checkout@v4.2.1
     - name: Install Go
       uses: actions/setup-go@v5.0.2
       with:
@@ -24,13 +24,10 @@ jobs:
         version: "v1.58"
 
   test:
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, windows-latest]
-    runs-on: ${{ matrix.platform }}
+    runs-on: ubuntu-latest
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4.1.7
+      uses: actions/checkout@v4.2.1
     - name: Install Go
       if: success()
       uses: actions/setup-go@v5.0.2

go.mod

@@ -2,4 +2,7 @@ module github.com/gabriel-vasile/mimetype
 
 go 1.20
 
-require golang.org/x/net v0.27.0
+require golang.org/x/net v0.30.0
+
+// v1.4.4 had a test file detected as malicious by antivirus software. #575
+retract v1.4.4

go.sum

@@ -1,2 +1,2 @@
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
+golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=

internal/magic/binary.go

@@ -21,6 +21,8 @@ var (
 	SWF = prefix([]byte("CWS"), []byte("FWS"), []byte("ZWS"))
 	// Torrent has bencoded text in the beginning.
 	Torrent = prefix([]byte("d8:announce"))
+	// PAR1 matches a parquet file.
+	Par1 = prefix([]byte{0x50, 0x41, 0x52, 0x31})
 )
 
 // Java bytecode and Mach-O binaries share the same magic number.

internal/magic/ftyp.go

@@ -1,22 +1,14 @@
 package magic
 
-import "bytes"
+import (
+	"bytes"
+)
 
 var (
 	// AVIF matches an AV1 Image File Format still or animated.
 	// Wikipedia page seems outdated listing image/avif-sequence for animations.
 	// https://github.com/AOMediaCodec/av1-avif/issues/59
 	AVIF = ftyp([]byte("avif"), []byte("avis"))
-	// Mp4 matches an MP4 file.
-	Mp4 = ftyp(
-		[]byte("avc1"), []byte("dash"), []byte("iso2"), []byte("iso3"),
-		[]byte("iso4"), []byte("iso5"), []byte("iso6"), []byte("isom"),
-		[]byte("mmp4"), []byte("mp41"), []byte("mp42"), []byte("mp4v"),
-		[]byte("mp71"), []byte("MSNV"), []byte("NDAS"), []byte("NDSC"),
-		[]byte("NSDC"), []byte("NSDH"), []byte("NDSM"), []byte("NDSP"),
-		[]byte("NDSS"), []byte("NDXC"), []byte("NDXH"), []byte("NDXM"),
-		[]byte("NDXP"), []byte("NDXS"), []byte("F4V "), []byte("F4P "),
-	)
 	// ThreeGP matches a 3GPP file.
 	ThreeGP = ftyp(
 		[]byte("3gp1"), []byte("3gp2"), []byte("3gp3"), []byte("3gp4"),
@@ -53,6 +45,17 @@ var (
 	Heif = ftyp([]byte("mif1"), []byte("heim"), []byte("heis"), []byte("avic"))
 	// HeifSequence matches a High Efficiency Image File Format (HEIF) file sequence.
 	HeifSequence = ftyp([]byte("msf1"), []byte("hevm"), []byte("hevs"), []byte("avcs"))
+	// Mj2 matches a Motion JPEG 2000 file: https://en.wikipedia.org/wiki/Motion_JPEG_2000.
+	Mj2 = ftyp([]byte("mj2s"), []byte("mjp2"), []byte("MFSM"), []byte("MGSV"))
+	// Dvb matches a Digital Video Broadcasting file: https://dvb.org.
+	// https://cconcolato.github.io/mp4ra/filetype.html
+	// https://github.com/file/file/blob/512840337ead1076519332d24fefcaa8fac36e06/magic/Magdir/animation#L135-L154
+	Dvb = ftyp(
+		[]byte("dby1"), []byte("dsms"), []byte("dts1"), []byte("dts2"),
+		[]byte("dts3"), []byte("dxo "), []byte("dmb1"), []byte("dmpf"),
+		[]byte("drc1"), []byte("dv1a"), []byte("dv1b"), []byte("dv2a"),
+		[]byte("dv2b"), []byte("dv3a"), []byte("dv3b"), []byte("dvr1"),
+		[]byte("dvt1"), []byte("emsg"))
 	// TODO: add support for remaining video formats at ftyps.com.
 )
 
@@ -86,3 +89,21 @@ func QuickTime(raw []byte, _ uint32) bool {
 	}
 	return bytes.Equal(raw[:8], []byte("\x00\x00\x00\x08wide"))
 }
+
+// Mp4 detects an .mp4 file. Mp4 detections only does a basic ftyp check.
+// Mp4 has many registered and unregistered code points so it's hard to keep track
+// of all. Detection will default on video/mp4 for all ftyp files.
+// ISO_IEC_14496-12 is the specification for the iso container.
+func Mp4(raw []byte, _ uint32) bool {
+	if len(raw) < 12 {
+		return false
+	}
+	// ftyps are made out of boxes. The first 4 bytes of the box represent
+	// its size in big-endian uint32. First box is the ftyp box and it is small
+	// in size. Check most significant byte is 0 to filter out false positive
+	// text files that happen to contain the string "ftyp" at index 4.
+	if raw[0] != 0 {
+		return false
+	}
+	return bytes.Equal(raw[4:8], []byte("ftyp"))
+}

internal/magic/magic.go

@@ -153,9 +153,6 @@ func ftyp(sigs ...[]byte) Detector {
 		if len(raw) < 12 {
 			return false
 		}
-		if !bytes.Equal(raw[4:8], []byte("ftyp")) {
-			return false
-		}
 		for _, s := range sigs {
 			if bytes.Equal(raw[8:12], s) {
 				return true
@@ -242,3 +239,13 @@ func min(a, b int) int {
 	}
 	return b
 }
+
+type readBuf []byte
+
+func (b *readBuf) advance(n int) bool {
+	if n < 0 || len(*b) < n {
+		return false
+	}
+	*b = (*b)[n:]
+	return true
+}

internal/magic/ms_office.go

@@ -5,58 +5,19 @@ import (
 	"encoding/binary"
 )
 
-var (
-	xlsxSigFiles = [][]byte{
-		[]byte("xl/worksheets/"),
-		[]byte("xl/drawings/"),
-		[]byte("xl/theme/"),
-		[]byte("xl/_rels/"),
-		[]byte("xl/styles.xml"),
-		[]byte("xl/workbook.xml"),
-		[]byte("xl/sharedStrings.xml"),
-	}
-	docxSigFiles = [][]byte{
-		[]byte("word/media/"),
-		[]byte("word/_rels/document.xml.rels"),
-		[]byte("word/document.xml"),
-		[]byte("word/styles.xml"),
-		[]byte("word/fontTable.xml"),
-		[]byte("word/settings.xml"),
-		[]byte("word/numbering.xml"),
-		[]byte("word/header"),
-		[]byte("word/footer"),
-	}
-	pptxSigFiles = [][]byte{
-		[]byte("ppt/slides/"),
-		[]byte("ppt/media/"),
-		[]byte("ppt/slideLayouts/"),
-		[]byte("ppt/theme/"),
-		[]byte("ppt/slideMasters/"),
-		[]byte("ppt/tags/"),
-		[]byte("ppt/notesMasters/"),
-		[]byte("ppt/_rels/"),
-		[]byte("ppt/handoutMasters/"),
-		[]byte("ppt/notesSlides/"),
-		[]byte("ppt/presentation.xml"),
-		[]byte("ppt/tableStyles.xml"),
-		[]byte("ppt/presProps.xml"),
-		[]byte("ppt/viewProps.xml"),
-	}
-)
-
 // Xlsx matches a Microsoft Excel 2007 file.
 func Xlsx(raw []byte, limit uint32) bool {
-	return zipContains(raw, xlsxSigFiles...)
+	return zipContains(raw, []byte("xl/"), true)
 }
 
 // Docx matches a Microsoft Word 2007 file.
 func Docx(raw []byte, limit uint32) bool {
-	return zipContains(raw, docxSigFiles...)
+	return zipContains(raw, []byte("word/"), true)
 }
 
 // Pptx matches a Microsoft PowerPoint 2007 file.
 func Pptx(raw []byte, limit uint32) bool {
-	return zipContains(raw, pptxSigFiles...)
+	return zipContains(raw, []byte("ppt/"), true)
 }
 
 // Ole matches an Open Linking and Embedding file.

internal/magic/text_csv.go

@@ -1,12 +1,28 @@
 package magic
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/csv"
 	"errors"
 	"io"
+	"sync"
 )
 
+// A bufio.Reader pool to alleviate problems with memory allocations.
+var readerPool = sync.Pool{
+	New: func() any {
+		// Initiate with empty source reader.
+		return bufio.NewReader(nil)
+	},
+}
+
+func newReader(r io.Reader) *bufio.Reader {
+	br := readerPool.Get().(*bufio.Reader)
+	br.Reset(r)
+	return br
+}
+
 // Csv matches a comma-separated values file.
 func Csv(raw []byte, limit uint32) bool {
 	return sv(raw, ',', limit)
@@ -18,7 +34,11 @@ func Tsv(raw []byte, limit uint32) bool {
 }
 
 func sv(in []byte, comma rune, limit uint32) bool {
-	r := csv.NewReader(bytes.NewReader(dropLastLine(in, limit)))
+	in = dropLastLine(in, limit)
+
+	br := newReader(bytes.NewReader(in))
+	defer readerPool.Put(br)
+	r := csv.NewReader(br)
 	r.Comma = comma
 	r.ReuseRecord = true
 	r.LazyQuotes = true

internal/magic/text_csv_test.go

@@ -0,0 +1,20 @@
+package magic
+
+import (
+	"io"
+	"math/rand"
+	"testing"
+)
+
+func BenchmarkCsv(b *testing.B) {
+	r := rand.New(rand.NewSource(0))
+	data := make([]byte, 4096)
+	if _, err := io.ReadFull(r, data); err != io.ErrUnexpectedEOF && err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		Csv(data, 0)
+	}
+}