diff --git a/codecs/vp9/bits.go b/codecs/vp9/bits.go new file mode 100644 index 0000000..a6a3c1f --- /dev/null +++ b/codecs/vp9/bits.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +package vp9 + +import "errors" + +var errNotEnoughBits = errors.New("not enough bits") + +func hasSpace(buf []byte, pos int, n int) error { + if n > ((len(buf) * 8) - pos) { + return errNotEnoughBits + } + return nil +} + +func readFlag(buf []byte, pos *int) (bool, error) { + err := hasSpace(buf, *pos, 1) + if err != nil { + return false, err + } + + return readFlagUnsafe(buf, pos), nil +} + +func readFlagUnsafe(buf []byte, pos *int) bool { + b := (buf[*pos>>0x03] >> (7 - (*pos & 0x07))) & 0x01 + *pos++ + return b == 1 +} + +func readBits(buf []byte, pos *int, n int) (uint64, error) { + err := hasSpace(buf, *pos, n) + if err != nil { + return 0, err + } + + return readBitsUnsafe(buf, pos, n), nil +} + +func readBitsUnsafe(buf []byte, pos *int, n int) uint64 { + res := 8 - (*pos & 0x07) + if n < res { + v := uint64((buf[*pos>>0x03] >> (res - n)) & (1<>0x03] & (1<= 8 { + v = (v << 8) | uint64(buf[*pos>>0x03]) + *pos += 8 + n -= 8 + } + + if n > 0 { + v = (v << n) | uint64(buf[*pos>>0x03]>>(8-n)) + *pos += n + } + + return v +} diff --git a/codecs/vp9/header.go b/codecs/vp9/header.go new file mode 100644 index 0000000..30e15bc --- /dev/null +++ b/codecs/vp9/header.go @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +// Package vp9 contains a VP9 header parser. +package vp9 + +import ( + "errors" +) + +var ( + errInvalidFrameMarker = errors.New("invalid frame marker") + errWrongFrameSyncByte0 = errors.New("wrong frame_sync_byte_0") + errWrongFrameSyncByte1 = errors.New("wrong frame_sync_byte_1") + errWrongFrameSyncByte2 = errors.New("wrong frame_sync_byte_2") +) + +// HeaderColorConfig is the color_config member of an header. +type HeaderColorConfig struct { + TenOrTwelveBit bool + BitDepth uint8 + ColorSpace uint8 + ColorRange bool + SubsamplingX bool + SubsamplingY bool +} + +func (c *HeaderColorConfig) unmarshal(profile uint8, buf []byte, pos *int) error { + if profile >= 2 { + var err error + c.TenOrTwelveBit, err = readFlag(buf, pos) + if err != nil { + return err + } + + if c.TenOrTwelveBit { + c.BitDepth = 12 + } else { + c.BitDepth = 10 + } + } else { + c.BitDepth = 8 + } + + tmp, err := readBits(buf, pos, 3) + if err != nil { + return err + } + c.ColorSpace = uint8(tmp) + + if c.ColorSpace != 7 { + var err error + c.ColorRange, err = readFlag(buf, pos) + if err != nil { + return err + } + + if profile == 1 || profile == 3 { + err := hasSpace(buf, *pos, 3) + if err != nil { + return err + } + + c.SubsamplingX = readFlagUnsafe(buf, pos) + c.SubsamplingY = readFlagUnsafe(buf, pos) + *pos++ + } else { + c.SubsamplingX = true + c.SubsamplingY = true + } + } else { + c.ColorRange = true + + if profile == 1 || profile == 3 { + c.SubsamplingX = false + c.SubsamplingY = false + + err := hasSpace(buf, *pos, 1) + if err != nil { + return err + } + *pos++ + } + } + + return nil +} + +// HeaderFrameSize is the frame_size member of an header. +type HeaderFrameSize struct { + FrameWidthMinus1 uint16 + FrameHeightMinus1 uint16 +} + +func (s *HeaderFrameSize) unmarshal(buf []byte, pos *int) error { + err := hasSpace(buf, *pos, 32) + if err != nil { + return err + } + + s.FrameWidthMinus1 = uint16(readBitsUnsafe(buf, pos, 16)) + s.FrameHeightMinus1 = uint16(readBitsUnsafe(buf, pos, 16)) + return nil +} + +// Header is a VP9 Frame header. +// Specification: +// https://storage.googleapis.com/downloads.webmproject.org/docs/vp9/vp9-bitstream-specification-v0.6-20160331-draft.pdf +type Header struct { + Profile uint8 + ShowExistingFrame bool + FrameToShowMapIdx uint8 + NonKeyFrame bool + ShowFrame bool + ErrorResilientMode bool + ColorConfig *HeaderColorConfig + FrameSize *HeaderFrameSize +} + +// Unmarshal decodes a Header. +func (h *Header) Unmarshal(buf []byte) error { + pos := 0 + + err := hasSpace(buf, pos, 4) + if err != nil { + return err + } + + frameMarker := readBitsUnsafe(buf, &pos, 2) + if frameMarker != 2 { + return errInvalidFrameMarker + } + + profileLowBit := uint8(readBitsUnsafe(buf, &pos, 1)) + profileHighBit := uint8(readBitsUnsafe(buf, &pos, 1)) + h.Profile = profileHighBit<<1 + profileLowBit + + if h.Profile == 3 { + err = hasSpace(buf, pos, 1) + if err != nil { + return err + } + pos++ + } + + h.ShowExistingFrame, err = readFlag(buf, &pos) + if err != nil { + return err + } + + if h.ShowExistingFrame { + var tmp uint64 + tmp, err = readBits(buf, &pos, 3) + if err != nil { + return err + } + h.FrameToShowMapIdx = uint8(tmp) + return nil + } + + err = hasSpace(buf, pos, 3) + if err != nil { + return err + } + + h.NonKeyFrame = readFlagUnsafe(buf, &pos) + h.ShowFrame = readFlagUnsafe(buf, &pos) + h.ErrorResilientMode = readFlagUnsafe(buf, &pos) + + if !h.NonKeyFrame { + err := hasSpace(buf, pos, 24) + if err != nil { + return err + } + + frameSyncByte0 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte0 != 0x49 { + return errWrongFrameSyncByte0 + } + + frameSyncByte1 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte1 != 0x83 { + return errWrongFrameSyncByte1 + } + + frameSyncByte2 := uint8(readBitsUnsafe(buf, &pos, 8)) + if frameSyncByte2 != 0x42 { + return errWrongFrameSyncByte2 + } + + h.ColorConfig = &HeaderColorConfig{} + err = h.ColorConfig.unmarshal(h.Profile, buf, &pos) + if err != nil { + return err + } + + h.FrameSize = &HeaderFrameSize{} + err = h.FrameSize.unmarshal(buf, &pos) + if err != nil { + return err + } + } + + return nil +} + +// Width returns the video width. +func (h Header) Width() uint16 { + if h.FrameSize == nil { + return 0 + } + return h.FrameSize.FrameWidthMinus1 + 1 +} + +// Height returns the video height. +func (h Header) Height() uint16 { + if h.FrameSize == nil { + return 0 + } + return h.FrameSize.FrameHeightMinus1 + 1 +} diff --git a/codecs/vp9/header_test.go b/codecs/vp9/header_test.go new file mode 100644 index 0000000..41fb46a --- /dev/null +++ b/codecs/vp9/header_test.go @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2023 The Pion community +// SPDX-License-Identifier: MIT + +package vp9 + +import ( + "reflect" + "testing" +) + +func TestHeaderUnmarshal(t *testing.T) { + cases := []struct { + name string + byts []byte + sh Header + width uint16 + height uint16 + }{ + { + "chrome webrtc", + []byte{ + 0x82, 0x49, 0x83, 0x42, 0x00, 0x77, 0xf0, 0x32, + 0x34, 0x30, 0x38, 0x24, 0x1c, 0x19, 0x40, 0x18, + 0x03, 0x40, 0x5f, 0xb4, + }, + Header{ + ShowFrame: true, + ColorConfig: &HeaderColorConfig{ + BitDepth: 8, + SubsamplingX: true, + SubsamplingY: true, + }, + FrameSize: &HeaderFrameSize{ + FrameWidthMinus1: 1919, + FrameHeightMinus1: 803, + }, + }, + 1920, + 804, + }, + { + "vp9 sample", + []byte{ + 0x82, 0x49, 0x83, 0x42, 0x40, 0xef, 0xf0, 0x86, + 0xf4, 0x04, 0x21, 0xa0, 0xe0, 0x00, 0x30, 0x70, + 0x00, 0x00, 0x00, 0x01, + }, + Header{ + ShowFrame: true, + ColorConfig: &HeaderColorConfig{ + BitDepth: 8, + ColorSpace: 2, + SubsamplingX: true, + SubsamplingY: true, + }, + FrameSize: &HeaderFrameSize{ + FrameWidthMinus1: 3839, + FrameHeightMinus1: 2159, + }, + }, + 3840, + 2160, + }, + } + + for _, ca := range cases { + t.Run(ca.name, func(t *testing.T) { + var sh Header + err := sh.Unmarshal(ca.byts) + if err != nil { + t.Fatal("unexpected error") + } + + if !reflect.DeepEqual(ca.sh, sh) { + t.Fatalf("expected %#+v, got %#+v", ca.sh, sh) + } + if ca.width != sh.Width() { + t.Fatalf("unexpected width") + } + if ca.height != sh.Height() { + t.Fatalf("unexpected height") + } + }) + } +} diff --git a/codecs/vp9_packet.go b/codecs/vp9_packet.go index 6a6e0b0..98920be 100644 --- a/codecs/vp9_packet.go +++ b/codecs/vp9_packet.go @@ -5,6 +5,7 @@ package codecs import ( "github.com/pion/randutil" + "github.com/pion/rtp/codecs/vp9" ) // Use global random generator to properly seed by crypto grade random. @@ -12,24 +13,50 @@ var globalMathRandomGenerator = randutil.NewMathRandomGenerator() // nolint:goch // VP9Payloader payloads VP9 packets type VP9Payloader struct { - pictureID uint16 - initialized bool + // whether to use flexible mode or non-flexible mode. + FlexibleMode bool // InitialPictureIDFn is a function that returns random initial picture ID. InitialPictureIDFn func() uint16 + + pictureID uint16 + initialized bool } const ( - vp9HeaderSize = 3 // Flexible mode 15 bit picture ID maxSpatialLayers = 5 maxVP9RefPics = 3 ) // Payload fragments an VP9 packet across one or more byte arrays func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { + if !p.initialized { + if p.InitialPictureIDFn == nil { + p.InitialPictureIDFn = func() uint16 { + return uint16(globalMathRandomGenerator.Intn(0x7FFF)) + } + } + p.pictureID = p.InitialPictureIDFn() & 0x7FFF + p.initialized = true + } + + var payloads [][]byte + if p.FlexibleMode { + payloads = p.payloadFlexible(mtu, payload) + } else { + payloads = p.payloadNonFlexible(mtu, payload) + } + + p.pictureID++ + if p.pictureID >= 0x8000 { + p.pictureID = 0 + } + + return payloads +} + +func (p *VP9Payloader) payloadFlexible(mtu uint16, payload []byte) [][]byte { /* - * https://www.ietf.org/id/draft-ietf-payload-vp9-13.txt - * * Flexible mode (F=1) * 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+ @@ -46,7 +73,45 @@ func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { * V: | SS | * | .. | * +-+-+-+-+-+-+-+-+ - * + */ + + headerSize := 3 + maxFragmentSize := int(mtu) - headerSize + payloadDataRemaining := len(payload) + payloadDataIndex := 0 + var payloads [][]byte + + if min(maxFragmentSize, payloadDataRemaining) <= 0 { + return [][]byte{} + } + + for payloadDataRemaining > 0 { + currentFragmentSize := min(maxFragmentSize, payloadDataRemaining) + out := make([]byte, headerSize+currentFragmentSize) + + out[0] = 0x90 // F=1, I=1 + if payloadDataIndex == 0 { + out[0] |= 0x08 // B=1 + } + if payloadDataRemaining == currentFragmentSize { + out[0] |= 0x04 // E=1 + } + + out[1] = byte(p.pictureID>>8) | 0x80 + out[2] = byte(p.pictureID) + + copy(out[headerSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) + payloads = append(payloads, out) + + payloadDataRemaining -= currentFragmentSize + payloadDataIndex += currentFragmentSize + } + + return payloads +} + +func (p *VP9Payloader) payloadNonFlexible(mtu uint16, payload []byte) [][]byte { + /* * Non-flexible mode (F=0) * 0 1 2 3 4 5 6 7 * +-+-+-+-+-+-+-+-+ @@ -65,51 +130,80 @@ func (p *VP9Payloader) Payload(mtu uint16, payload []byte) [][]byte { * +-+-+-+-+-+-+-+-+ */ - if !p.initialized { - if p.InitialPictureIDFn == nil { - p.InitialPictureIDFn = func() uint16 { - return uint16(globalMathRandomGenerator.Intn(0x7FFF)) - } - } - p.pictureID = p.InitialPictureIDFn() & 0x7FFF - p.initialized = true - } - if payload == nil { + var h vp9.Header + err := h.Unmarshal(payload) + if err != nil { return [][]byte{} } - maxFragmentSize := int(mtu) - vp9HeaderSize payloadDataRemaining := len(payload) payloadDataIndex := 0 - - if min(maxFragmentSize, payloadDataRemaining) <= 0 { - return [][]byte{} - } - var payloads [][]byte + for payloadDataRemaining > 0 { + var headerSize int + if !h.NonKeyFrame && payloadDataIndex == 0 { + headerSize = 3 + 8 + } else { + headerSize = 3 + } + + maxFragmentSize := int(mtu) - headerSize currentFragmentSize := min(maxFragmentSize, payloadDataRemaining) - out := make([]byte, vp9HeaderSize+currentFragmentSize) + if currentFragmentSize <= 0 { + return [][]byte{} + } + + out := make([]byte, headerSize+currentFragmentSize) + + out[0] = 0x80 | 0x01 // I=1, Z=1 - out[0] = 0x90 // F=1 I=1 + if h.NonKeyFrame { + out[0] |= 0x40 // P=1 + } if payloadDataIndex == 0 { out[0] |= 0x08 // B=1 } if payloadDataRemaining == currentFragmentSize { out[0] |= 0x04 // E=1 } + out[1] = byte(p.pictureID>>8) | 0x80 out[2] = byte(p.pictureID) - copy(out[vp9HeaderSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) + off := 3 + + if !h.NonKeyFrame && payloadDataIndex == 0 { + out[0] |= 0x02 // V=1 + out[off] = 0x10 | 0x08 // N_S=0, Y=1, G=1 + off++ + + width := h.Width() + out[off] = byte(width >> 8) + off++ + out[off] = byte(width & 0xFF) + off++ + + height := h.Height() + out[off] = byte(height >> 8) + off++ + out[off] = byte(height & 0xFF) + off++ + + out[off] = 0x01 // N_G=1 + off++ + + out[off] = 1<<4 | 1<<2 // TID=0, U=1, R=1 + off++ + + out[off] = 0x01 // P_DIFF=1 + } + + copy(out[headerSize:], payload[payloadDataIndex:payloadDataIndex+currentFragmentSize]) payloads = append(payloads, out) payloadDataRemaining -= currentFragmentSize payloadDataIndex += currentFragmentSize } - p.pictureID++ - if p.pictureID >= 0x8000 { - p.pictureID = 0 - } return payloads } diff --git a/codecs/vp9_packet_test.go b/codecs/vp9_packet_test.go index 97e176b..9591f22 100644 --- a/codecs/vp9_packet_test.go +++ b/codecs/vp9_packet_test.go @@ -5,7 +5,6 @@ package codecs import ( "errors" - "fmt" "math/rand" "reflect" "testing" @@ -223,62 +222,134 @@ func TestVP9Payloader_Payload(t *testing.T) { } cases := map[string]struct { - b [][]byte - mtu uint16 - res [][]byte + b [][]byte + flexible bool + mtu uint16 + res [][]byte }{ - "NilPayload": { - b: [][]byte{nil}, - mtu: 100, - res: [][]byte{}, + "flexible NilPayload": { + b: [][]byte{nil}, + flexible: true, + mtu: 100, + res: [][]byte{}, }, - "SmallMTU": { - b: [][]byte{{0x00, 0x00}}, - mtu: 1, - res: [][]byte{}, + "flexible SmallMTU": { + b: [][]byte{{0x00, 0x00}}, + flexible: true, + mtu: 1, + res: [][]byte{}, }, - "OnePacket": { - b: [][]byte{{0x01, 0x02}}, - mtu: 10, + "flexible OnePacket": { + b: [][]byte{{0x01, 0x02}}, + flexible: true, + mtu: 10, res: [][]byte{ {0x9C, rands[0][0], rands[0][1], 0x01, 0x02}, }, }, - "TwoPackets": { - b: [][]byte{{0x01, 0x02}}, - mtu: 4, + "flexible TwoPackets": { + b: [][]byte{{0x01, 0x02}}, + flexible: true, + mtu: 4, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01}, {0x94, rands[0][0], rands[0][1], 0x02}, }, }, - "ThreePackets": { - b: [][]byte{{0x01, 0x02, 0x03}}, - mtu: 4, + "flexible ThreePackets": { + b: [][]byte{{0x01, 0x02, 0x03}}, + flexible: true, + mtu: 4, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01}, {0x90, rands[0][0], rands[0][1], 0x02}, {0x94, rands[0][0], rands[0][1], 0x03}, }, }, - "TwoFramesFourPackets": { - b: [][]byte{{0x01, 0x02, 0x03}, {0x04}}, - mtu: 5, + "flexible TwoFramesFourPackets": { + b: [][]byte{{0x01, 0x02, 0x03}, {0x04}}, + flexible: true, + mtu: 5, res: [][]byte{ {0x98, rands[0][0], rands[0][1], 0x01, 0x02}, {0x94, rands[0][0], rands[0][1], 0x03}, {0x9C, rands[1][0], rands[1][1], 0x04}, }, }, + "non-flexible NilPayload": { + b: [][]byte{nil}, + mtu: 100, + res: [][]byte{}, + }, + "non-flexible SmallMTU": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 1, + res: [][]byte{}, + }, + "non-flexible OnePacket key frame": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 20, + res: [][]byte{{ + 0x8f, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, 0x49, 0x83, 0x42, 0x00, + 0x77, 0xf0, 0x32, 0x34, + }}, + }, + "non-flexible TwoPackets key frame": { + b: [][]byte{{0x82, 0x49, 0x83, 0x42, 0x0, 0x77, 0xf0, 0x32, 0x34}}, + mtu: 12, + res: [][]byte{ + { + 0x8b, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, + }, + { + 0x85, 0xa1, 0xf4, 0x49, 0x83, 0x42, 0x00, 0x77, + 0xf0, 0x32, 0x34, + }, + }, + }, + "non-flexible ThreePackets key frame": { + b: [][]byte{{ + 0x82, 0x49, 0x83, 0x42, 0x00, 0x77, 0xf0, 0x32, + 0x34, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, + }}, + mtu: 12, + res: [][]byte{ + { + 0x8b, 0xa1, 0xf4, 0x18, 0x07, 0x80, 0x03, 0x24, + 0x01, 0x14, 0x01, 0x82, + }, + { + 0x81, 0xa1, 0xf4, 0x49, 0x83, 0x42, 0x00, 0x77, + 0xf0, 0x32, 0x34, 0x01, + }, + { + 0x85, 0xa1, 0xf4, 0x02, 0x03, 0x04, 0x05, 0x06, + 0x07, 0x08, + }, + }, + }, + "non-flexible OnePacket non key frame": { + b: [][]byte{{0x86, 0x0, 0x40, 0x92, 0xe1, 0x31, 0x42, 0x8c, 0xc0, 0x40}}, + mtu: 20, + res: [][]byte{{ + 0xcd, 0xa1, 0xf4, 0x86, 0x00, 0x40, 0x92, 0xe1, + 0x31, 0x42, 0x8c, 0xc0, 0x40, + }}, + }, } + for name, c := range cases { - pck := VP9Payloader{ - InitialPictureIDFn: func() uint16 { - return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec - }, - } - c := c - t.Run(fmt.Sprintf("%s_MTU%d", name, c.mtu), func(t *testing.T) { + t.Run(name, func(t *testing.T) { + pck := VP9Payloader{ + FlexibleMode: c.flexible, + InitialPictureIDFn: func() uint16 { + return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec + }, + } + res := [][]byte{} for _, b := range c.b { res = append(res, pck.Payload(c.mtu, b)...) @@ -288,8 +359,10 @@ func TestVP9Payloader_Payload(t *testing.T) { } }) } + t.Run("PictureIDOverflow", func(t *testing.T) { pck := VP9Payloader{ + FlexibleMode: true, InitialPictureIDFn: func() uint16 { return uint16(rand.New(rand.NewSource(0)).Int31n(0x7FFF)) //nolint:gosec },